Make config-schema extensible for handling of unknown fields (#156214)

Related issue https://github.com/elastic/kibana/issues/155764.

In this POC, I'm adding an `extendsDeep` function to the schema object.
This feature allows you to create a copy of an existing schema
definition and recursively modify options without mutating them. With
`extendsDeep`, you can specify whether unknown attributes on objects
should be allowed, forbidden or ignored.

This new function is particularly useful for alerting scenarios where we
need to drop unknown fields when reading from Elasticsearch without
modifying the schema object. Since we don't control the schema
definition in some areas, `extendsDeep` provides a convenient way to set
the `unknowns` option to all objects recursively. By doing so, we can
validate and drop unknown properties using the same defined schema, just
with `unknowns: forbid` extension.

Usage:
```
// Single, shared type definition
const type = schema.object({ foo: schema.string() });

// Drop unknown fields (bar in this case)
const savedObject = { foo: 'test', bar: 'test' };
const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
ignoreSchema.validate(savedObject);

// Prevent unknown fields (bar in this case)
const soToUpdate = { foo: 'test', bar: 'test' };
const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
forbidSchema.validate(soToUpdate);
```

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-config-schema/src/types/array_type.test.ts

@@ -188,3 +188,30 @@ describe('#maxSize', () => {
     ).toThrowErrorMatchingInlineSnapshot(`"array size is [2], but cannot be greater than [1]"`);
   });
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.arrayOf(
+    schema.object({
+      foo: schema.string(),
+    })
+  );
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate([{ foo: 'test', bar: 'test' }]);
+    expect(result).toEqual([{ foo: 'test', bar: 'test' }]);
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate([{ foo: 'test', bar: 'test' }]);
+    expect(result).toEqual([{ foo: 'test' }]);
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate([{ foo: 'test', bar: 'test' }])
+    ).toThrowErrorMatchingInlineSnapshot(`"[0.bar]: definition for this key is missing"`);
+  });
+});

packages/kbn-config-schema/src/types/array_type.ts

@@ -8,7 +8,7 @@
 
 import typeDetect from 'type-detect';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions } from './type';
 
 export type ArrayOptions<T> = TypeOptions<T[]> & {
   minSize?: number;
@@ -16,6 +16,9 @@ export type ArrayOptions<T> = TypeOptions<T[]> & {
 };
 
 export class ArrayType<T> extends Type<T[]> {
+  private readonly arrayType: Type<T>;
+  private readonly arrayOptions: ArrayOptions<T>;
+
   constructor(type: Type<T>, options: ArrayOptions<T> = {}) {
     let schema = internals.array().items(type.getSchema().optional()).sparse(false);
 
@@ -28,6 +31,12 @@ export class ArrayType<T> extends Type<T[]> {
     }
 
     super(schema, options);
+    this.arrayType = type;
+    this.arrayOptions = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new ArrayType(this.arrayType.extendsDeep(options), this.arrayOptions);
   }
 
   protected handleError(type: string, { limit, reason, value }: Record<string, any>) {

packages/kbn-config-schema/src/types/conditional_type.test.ts

@@ -395,3 +395,84 @@ describe('#validate', () => {
     });
   });
 });
+
+describe('#extendsDeep', () => {
+  describe('#equalType', () => {
+    const type = schema.object({
+      foo: schema.string(),
+      test: schema.conditional(
+        schema.siblingRef('foo'),
+        'test',
+        schema.object({
+          bar: schema.string(),
+        }),
+        schema.string()
+      ),
+    });
+
+    test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+      const result = type
+        .extendsDeep({ unknowns: 'allow' })
+        .validate({ foo: 'test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'test',
+        test: { bar: 'test', baz: 'test' },
+      });
+    });
+
+    test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+      const result = type
+        .extendsDeep({ unknowns: 'ignore' })
+        .validate({ foo: 'test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'test',
+        test: { bar: 'test' },
+      });
+    });
+    test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+      expect(() =>
+        type
+          .extendsDeep({ unknowns: 'forbid' })
+          .validate({ foo: 'test', test: { bar: 'test', baz: 'test' } })
+      ).toThrowErrorMatchingInlineSnapshot(`"[test.baz]: definition for this key is missing"`);
+    });
+  });
+
+  describe('#notEqualType', () => {
+    const type = schema.object({
+      foo: schema.string(),
+      test: schema.conditional(
+        schema.siblingRef('foo'),
+        'test',
+        schema.string(),
+        schema.object({
+          bar: schema.string(),
+        })
+      ),
+    });
+
+    test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+      const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+      const result = allowSchema.validate({ foo: 'not-test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'not-test',
+        test: { bar: 'test', baz: 'test' },
+      });
+    });
+
+    test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+      const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+      const result = ignoreSchema.validate({ foo: 'not-test', test: { bar: 'test', baz: 'test' } });
+      expect(result).toEqual({
+        foo: 'not-test',
+        test: { bar: 'test' },
+      });
+    });
+    test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+      const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+      expect(() =>
+        forbidSchema.validate({ foo: 'not-test', test: { bar: 'test', baz: 'test' } })
+      ).toThrowErrorMatchingInlineSnapshot(`"[test.baz]: definition for this key is missing"`);
+    });
+  });
+});

packages/kbn-config-schema/src/types/conditional_type.ts

@@ -9,11 +9,17 @@
 import typeDetect from 'type-detect';
 import { internals } from '../internals';
 import { Reference } from '../references';
-import { Type, TypeOptions } from './type';
+import { ExtendsDeepOptions, Type, TypeOptions } from './type';
 
 export type ConditionalTypeValue = string | number | boolean | object | null;
 
 export class ConditionalType<A extends ConditionalTypeValue, B, C> extends Type<B | C> {
+  private readonly leftOperand: Reference<A>;
+  private readonly rightOperand: Reference<A> | A | Type<unknown>;
+  private readonly equalType: Type<B>;
+  private readonly notEqualType: Type<C>;
+  private readonly options?: TypeOptions<B | C>;
+
   constructor(
     leftOperand: Reference<A>,
     rightOperand: Reference<A> | A | Type<unknown>,
@@ -31,6 +37,21 @@ export class ConditionalType<A extends ConditionalTypeValue, B, C> extends Type<
     });
 
     super(schema, options);
+    this.leftOperand = leftOperand;
+    this.rightOperand = rightOperand;
+    this.equalType = equalType;
+    this.notEqualType = notEqualType;
+    this.options = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new ConditionalType(
+      this.leftOperand,
+      this.rightOperand,
+      this.equalType.extendsDeep(options),
+      this.notEqualType.extendsDeep(options),
+      this.options
+    );
   }
 
   protected handleError(type: string, { value }: Record<string, any>) {

packages/kbn-config-schema/src/types/map_of_type.test.ts

@@ -185,3 +185,28 @@ test('error preserves full path', () => {
     `"[grandParentKey.parentKey.ab]: expected value of type [number] but got [string]"`
   );
 });
+
+describe('#extendsDeep', () => {
+  describe('#keyType', () => {
+    const type = schema.mapOf(schema.string(), schema.object({ foo: schema.string() }));
+
+    test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+      const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+      const result = allowSchema.validate({ key: { foo: 'test', bar: 'test' } });
+      expect(result.get('key')).toEqual({ foo: 'test', bar: 'test' });
+    });
+
+    test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+      const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+      const result = ignoreSchema.validate({ key: { foo: 'test', bar: 'test' } });
+      expect(result.get('key')).toEqual({ foo: 'test' });
+    });
+
+    test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+      const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+      expect(() =>
+        forbidSchema.validate({ key: { foo: 'test', bar: 'test' } })
+      ).toThrowErrorMatchingInlineSnapshot(`"[key.bar]: definition for this key is missing"`);
+    });
+  });
+});

packages/kbn-config-schema/src/types/map_type.ts

@@ -9,11 +9,15 @@
 import typeDetect from 'type-detect';
 import { SchemaTypeError, SchemaTypesError } from '../errors';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions } from './type';
 
 export type MapOfOptions<K, V> = TypeOptions<Map<K, V>>;
 
 export class MapOfType<K, V> extends Type<Map<K, V>> {
+  private readonly keyType: Type<K>;
+  private readonly valueType: Type<V>;
+  private readonly mapOptions: MapOfOptions<K, V>;
+
   constructor(keyType: Type<K>, valueType: Type<V>, options: MapOfOptions<K, V> = {}) {
     const defaultValue = options.defaultValue;
     const schema = internals.map().entries(keyType.getSchema(), valueType.getSchema());
@@ -26,6 +30,17 @@ export class MapOfType<K, V> extends Type<Map<K, V>> {
       // default value instead.
       defaultValue: defaultValue instanceof Map ? () => defaultValue : defaultValue,
     });
+    this.keyType = keyType;
+    this.valueType = valueType;
+    this.mapOptions = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new MapOfType(
+      this.keyType.extendsDeep(options),
+      this.valueType.extendsDeep(options),
+      this.mapOptions
+    );
   }
 
   protected handleError(

packages/kbn-config-schema/src/types/maybe_type.test.ts

@@ -95,3 +95,26 @@ describe('maybe + object', () => {
     expect(type.validate({})).toEqual({});
   });
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.maybe(schema.object({ foo: schema.string() }));
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate({ foo: 'test', bar: 'test' });
+    expect(result).toEqual({ foo: 'test', bar: 'test' });
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate({ foo: 'test', bar: 'test' });
+    expect(result).toEqual({ foo: 'test' });
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate({ foo: 'test', bar: 'test' })
+    ).toThrowErrorMatchingInlineSnapshot(`"[bar]: definition for this key is missing"`);
+  });
+});

packages/kbn-config-schema/src/types/maybe_type.ts

@@ -6,9 +6,11 @@
  * Side Public License, v 1.
  */
 
-import { Type } from './type';
+import { Type, ExtendsDeepOptions } from './type';
 
 export class MaybeType<V> extends Type<V | undefined> {
+  private readonly maybeType: Type<V>;
+
   constructor(type: Type<V>) {
     super(
       type
@@ -16,5 +18,10 @@ export class MaybeType<V> extends Type<V | undefined> {
         .optional()
         .default(() => undefined)
     );
+    this.maybeType = type;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new MaybeType(this.maybeType.extendsDeep(options));
   }
 }

packages/kbn-config-schema/src/types/object_type.test.ts

@@ -563,3 +563,26 @@ test('returns schema structure', () => {
     { path: ['nested', 'uri'], type: 'string' },
   ]);
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.object({ test: schema.object({ foo: schema.string() }) });
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate({ test: { foo: 'test', bar: 'test' } });
+    expect(result).toEqual({ test: { foo: 'test', bar: 'test' } });
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate({ test: { foo: 'test', bar: 'test' } });
+    expect(result).toEqual({ test: { foo: 'test' } });
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate({ test: { foo: 'test', bar: 'test' } })
+    ).toThrowErrorMatchingInlineSnapshot(`"[test.bar]: definition for this key is missing"`);
+  });
+});

packages/kbn-config-schema/src/types/object_type.ts

@@ -9,7 +9,7 @@
 import type { AnySchema } from 'joi';
 import typeDetect from 'type-detect';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions, OptionsForUnknowns } from './type';
 import { ValidationError } from '../errors';
 
 export type Props = Record<string, Type<any>>;
@@ -60,13 +60,7 @@ type ExtendedObjectTypeOptions<P extends Props, NP extends NullableProps> = Obje
 >;
 
 interface UnknownOptions {
-  /**
-   * Options for dealing with unknown keys:
-   * - allow: unknown keys will be permitted
-   * - ignore: unknown keys will not fail validation, but will be stripped out
-   * - forbid (default): unknown keys will fail validation
-   */
-  unknowns?: 'allow' | 'ignore' | 'forbid';
+  unknowns?: OptionsForUnknowns;
 }
 
 export type ObjectTypeOptions<P extends Props = any> = TypeOptions<ObjectResultType<P>> &
@@ -181,6 +175,25 @@ export class ObjectType<P extends Props = any> extends Type<ObjectResultType<P>>
     return new ObjectType(extendedProps, extendedOptions);
   }
 
+  public extendsDeep(options: ExtendsDeepOptions) {
+    const extendedProps = Object.entries(this.props).reduce((memo, [key, value]) => {
+      if (value !== null && value !== undefined) {
+        return {
+          ...memo,
+          [key]: value.extendsDeep(options),
+        };
+      }
+      return memo;
+    }, {} as P);
+
+    const extendedOptions: ObjectTypeOptions<P> = {
+      ...this.options,
+      ...(options.unknowns ? { unknowns: options.unknowns } : {}),
+    };
+
+    return new ObjectType(extendedProps, extendedOptions);
+  }
+
   protected handleError(type: string, { reason, value }: Record<string, any>) {
     switch (type) {
       case 'any.required':

packages/kbn-config-schema/src/types/record_type.test.ts

@@ -185,3 +185,26 @@ test('error preserves full path', () => {
     `"[grandParentKey.parentKey.ab]: expected value of type [number] but got [string]"`
   );
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.recordOf(schema.string(), schema.object({ foo: schema.string() }));
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate({ key: { foo: 'test', bar: 'test' } });
+    expect(result).toEqual({ key: { foo: 'test', bar: 'test' } });
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate({ key: { foo: 'test', bar: 'test' } });
+    expect(result).toEqual({ key: { foo: 'test' } });
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate({ key: { foo: 'test', bar: 'test' } })
+    ).toThrowErrorMatchingInlineSnapshot(`"[key.bar]: definition for this key is missing"`);
+  });
+});

packages/kbn-config-schema/src/types/record_type.ts

@@ -9,15 +9,30 @@
 import typeDetect from 'type-detect';
 import { SchemaTypeError, SchemaTypesError } from '../errors';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions } from './type';
 
 export type RecordOfOptions<K extends string, V> = TypeOptions<Record<K, V>>;
 
 export class RecordOfType<K extends string, V> extends Type<Record<K, V>> {
+  private readonly keyType: Type<K>;
+  private readonly valueType: Type<V>;
+  private readonly options: RecordOfOptions<K, V>;
+
   constructor(keyType: Type<K>, valueType: Type<V>, options: RecordOfOptions<K, V> = {}) {
     const schema = internals.record().entries(keyType.getSchema(), valueType.getSchema());
 
     super(schema, options);
+    this.keyType = keyType;
+    this.valueType = valueType;
+    this.options = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new RecordOfType(
+      this.keyType.extendsDeep(options),
+      this.valueType.extendsDeep(options),
+      this.options
+    );
   }
 
   protected handleError(

packages/kbn-config-schema/src/types/type.ts

@@ -20,6 +20,18 @@ export interface SchemaStructureEntry {
   type: string;
 }
 
+/**
+ * Options for dealing with unknown keys:
+ * - allow: unknown keys will be permitted
+ * - ignore: unknown keys will not fail validation, but will be stripped out
+ * - forbid (default): unknown keys will fail validation
+ */
+export type OptionsForUnknowns = 'allow' | 'ignore' | 'forbid';
+
+export interface ExtendsDeepOptions {
+  unknowns?: OptionsForUnknowns;
+}
+
 export const convertValidationFunction = <T = unknown>(
   validate: (value: T) => string | void
 ): CustomValidator<T> => {
@@ -83,6 +95,10 @@ export abstract class Type<V> {
     this.internalSchema = schema;
   }
 
+  public extendsDeep(newOptions: ExtendsDeepOptions): Type<V> {
+    return this;
+  }
+
   public validate(value: any, context: Record<string, any> = {}, namespace?: string): V {
     const { value: validatedValue, error } = this.internalSchema.validate(value, {
       context,

packages/kbn-config-schema/src/types/union_type.test.ts

@@ -168,3 +168,26 @@ test('fails if nested union type fail', () => {
      - [1]: expected value of type [number] but got [string]"
   `);
 });
+
+describe('#extendsDeep', () => {
+  const type = schema.oneOf([schema.object({ foo: schema.string() })]);
+
+  test('objects with unknown attributes are kept when extending with unknowns=allow', () => {
+    const allowSchema = type.extendsDeep({ unknowns: 'allow' });
+    const result = allowSchema.validate({ foo: 'test', bar: 'test' });
+    expect(result).toEqual({ foo: 'test', bar: 'test' });
+  });
+
+  test('objects with unknown attributes are dropped when extending with unknowns=ignore', () => {
+    const ignoreSchema = type.extendsDeep({ unknowns: 'ignore' });
+    const result = ignoreSchema.validate({ foo: 'test', bar: 'test' });
+    expect(result).toEqual({ foo: 'test' });
+  });
+
+  test('objects with unknown attributes fail validation when extending with unknowns=forbid', () => {
+    const forbidSchema = type.extendsDeep({ unknowns: 'forbid' });
+    expect(() =>
+      forbidSchema.validate({ foo: 'test', bar: 'test' })
+    ).toThrowErrorMatchingInlineSnapshot(`"[bar]: definition for this key is missing"`);
+  });
+});

packages/kbn-config-schema/src/types/union_type.ts

@@ -9,13 +9,25 @@
 import typeDetect from 'type-detect';
 import { SchemaTypeError, SchemaTypesError } from '../errors';
 import { internals } from '../internals';
-import { Type, TypeOptions } from './type';
+import { Type, TypeOptions, ExtendsDeepOptions } from './type';
 
 export class UnionType<RTS extends Array<Type<any>>, T> extends Type<T> {
+  private readonly unionTypes: RTS;
+  private readonly typeOptions?: TypeOptions<T>;
+
   constructor(types: RTS, options?: TypeOptions<T>) {
     const schema = internals.alternatives(types.map((type) => type.getSchema())).match('any');
 
     super(schema, options);
+    this.unionTypes = types;
+    this.typeOptions = options;
+  }
+
+  public extendsDeep(options: ExtendsDeepOptions) {
+    return new UnionType(
+      this.unionTypes.map((t) => t.extendsDeep(options)),
+      this.typeOptions
+    );
   }
 
   protected handleError(type: string, { value, details }: Record<string, any>, path: string[]) {