mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
[ZDT] DocumentMigrator: support higher version documents (#157895)
## Summary Part of https://github.com/elastic/kibana/issues/150312 (next steps depend on https://github.com/elastic/kibana/pull/153117) **This PR does two things:** - introduce the concept of version persistence schema - adapt the document migrator to support downward migrations for documents of an higher version. In the follow-up, we will then update the calls from the SOR to the document migrator to allow downward conversions when we're using the ZDT migration algorithm (which requires https://github.com/elastic/kibana/pull/153117 to be merged) ### Model version persistence schema. *(This is what has also been named 'eviction schema' or 'known fields schema'.)* A new `SavedObjectsModelVersion.schemas.backwardConversion` property was added to the model version definition. This 'schema' can either be an arbitrary function, or a `schema.object` from `@kbn/config-schema` ```ts type SavedObjectModelVersionBackwardConversionSchema< InAttrs = unknown, OutAttrs = unknown > = ObjectType | SavedObjectModelVersionBackwardConversionFn<InAttrs, OutAttrs>; ``` When specified for a version, the document's attributes will go thought this schema during down conversions by the document migrator. ### Adapt the document migrator to support downward migrations for documents of an higher version. Add an `allowDowngrade` option to `DocumentMigrator.migrate` and `KibanaMigrator.migrateDocument`. When this option is set to `true`, the document migration will accept to 'downgrade' the document if necessary, instead of throwing an error as done when the option is `false` or unspecified (which was the only behavior prior to this PR's changes) --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Pierre Gayvallet
GitHub
parent
cc7c2812af
commit
c24dc357fd
30 changed files with 996 additions and 89 deletions
|
|
@ -39,6 +39,7 @@ export type {
|
|||
KibanaMigratorStatus,
|
||||
MigrationResult,
|
||||
MigrationStatus,
|
||||
MigrateDocumentOptions,
|
||||
} from './src/migration';
|
||||
export { parseObjectKey, getObjectKey, getIndexForType } from './src/utils';
|
||||
export {
|
||||
|
|
@ -64,4 +65,5 @@ export {
|
|||
getModelVersionDelta,
|
||||
buildModelVersionTransformFn,
|
||||
aggregateMappingAdditions,
|
||||
convertModelVersionBackwardConversionSchema,
|
||||
} from './src/model_version';
|
||||
|
|
|
|||
|
|
@ -11,4 +11,5 @@ export type {
|
|||
KibanaMigratorStatus,
|
||||
MigrationStatus,
|
||||
MigrationResult,
|
||||
MigrateDocumentOptions,
|
||||
} from './kibana_migrator';
|
||||
|
|
|
|||
|
|
@ -49,7 +49,22 @@ export interface IKibanaMigrator {
|
|||
* @param doc - The saved object to migrate
|
||||
* @returns `doc` with all registered migrations applied.
|
||||
*/
|
||||
migrateDocument(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc;
|
||||
migrateDocument(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
options?: MigrateDocumentOptions
|
||||
): SavedObjectUnsanitizedDoc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for {@link IKibanaMigrator.migrateDocument}
|
||||
* @internal
|
||||
*/
|
||||
export interface MigrateDocumentOptions {
|
||||
/**
|
||||
* Defines whether it is allowed to convert documents from an higher version or not.
|
||||
* Defaults to `false`.
|
||||
*/
|
||||
allowDowngrade?: boolean;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
|
|
|
|||
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { schema } from '@kbn/config-schema';
|
||||
import { convertModelVersionBackwardConversionSchema } from './backward_conversion_schema';
|
||||
import type {
|
||||
SavedObjectUnsanitizedDoc,
|
||||
SavedObjectModelVersionForwardCompatibilityFn,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
|
||||
describe('convertModelVersionBackwardConversionSchema', () => {
|
||||
const createDoc = (
|
||||
parts: Partial<SavedObjectUnsanitizedDoc<any>>
|
||||
): SavedObjectUnsanitizedDoc<any> => ({
|
||||
id: 'id',
|
||||
type: 'type',
|
||||
attributes: {},
|
||||
...parts,
|
||||
});
|
||||
|
||||
describe('using functions', () => {
|
||||
it('converts the schema', () => {
|
||||
const conversionSchema: jest.MockedFunction<SavedObjectModelVersionForwardCompatibilityFn> =
|
||||
jest.fn();
|
||||
conversionSchema.mockImplementation((attrs) => attrs);
|
||||
|
||||
const doc = createDoc({ attributes: { foo: 'bar' } });
|
||||
const converted = convertModelVersionBackwardConversionSchema(conversionSchema);
|
||||
|
||||
const output = converted(doc);
|
||||
|
||||
expect(conversionSchema).toHaveBeenCalledTimes(1);
|
||||
expect(conversionSchema).toHaveBeenCalledWith({ foo: 'bar' });
|
||||
expect(output).toEqual(doc);
|
||||
});
|
||||
|
||||
it('returns the document with the updated properties', () => {
|
||||
const conversionSchema: jest.MockedFunction<
|
||||
SavedObjectModelVersionForwardCompatibilityFn<any, any>
|
||||
> = jest.fn();
|
||||
conversionSchema.mockImplementation((attrs) => ({ foo: attrs.foo }));
|
||||
|
||||
const doc = createDoc({ attributes: { foo: 'bar', hello: 'dolly' } });
|
||||
const converted = convertModelVersionBackwardConversionSchema(conversionSchema);
|
||||
|
||||
const output = converted(doc);
|
||||
|
||||
expect(output).toEqual({
|
||||
...doc,
|
||||
attributes: { foo: 'bar' },
|
||||
});
|
||||
});
|
||||
|
||||
it('throws if the function throws', () => {
|
||||
const conversionSchema: jest.MockedFunction<
|
||||
SavedObjectModelVersionForwardCompatibilityFn<any, any>
|
||||
> = jest.fn();
|
||||
conversionSchema.mockImplementation(() => {
|
||||
throw new Error('dang');
|
||||
});
|
||||
|
||||
const doc = createDoc({});
|
||||
const converted = convertModelVersionBackwardConversionSchema(conversionSchema);
|
||||
|
||||
expect(() => converted(doc)).toThrowErrorMatchingInlineSnapshot(`"dang"`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('using config-schemas', () => {
|
||||
it('converts the schema', () => {
|
||||
const conversionSchema = schema.object(
|
||||
{
|
||||
foo: schema.maybe(schema.string()),
|
||||
},
|
||||
{ unknowns: 'ignore' }
|
||||
);
|
||||
const validateSpy = jest.spyOn(conversionSchema, 'validate');
|
||||
|
||||
const doc = createDoc({ attributes: { foo: 'bar' } });
|
||||
const converted = convertModelVersionBackwardConversionSchema(conversionSchema);
|
||||
|
||||
const output = converted(doc);
|
||||
|
||||
expect(validateSpy).toHaveBeenCalledTimes(1);
|
||||
expect(validateSpy).toHaveBeenCalledWith({ foo: 'bar' }, {});
|
||||
expect(output).toEqual(doc);
|
||||
});
|
||||
|
||||
it('returns the document with the updated properties', () => {
|
||||
const conversionSchema = schema.object(
|
||||
{
|
||||
foo: schema.maybe(schema.string()),
|
||||
},
|
||||
{ unknowns: 'ignore' }
|
||||
);
|
||||
|
||||
const doc = createDoc({ attributes: { foo: 'bar', hello: 'dolly' } });
|
||||
const converted = convertModelVersionBackwardConversionSchema(conversionSchema);
|
||||
|
||||
const output = converted(doc);
|
||||
|
||||
expect(output).toEqual({
|
||||
...doc,
|
||||
attributes: {
|
||||
foo: 'bar',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('throws if the validation throws', () => {
|
||||
const conversionSchema = schema.object(
|
||||
{
|
||||
foo: schema.maybe(schema.string()),
|
||||
},
|
||||
{ unknowns: 'forbid' }
|
||||
);
|
||||
|
||||
const doc = createDoc({ attributes: { foo: 'bar', hello: 'dolly' } });
|
||||
const converted = convertModelVersionBackwardConversionSchema(conversionSchema);
|
||||
|
||||
expect(() => converted(doc)).toThrowErrorMatchingInlineSnapshot(
|
||||
`"[hello]: definition for this key is missing"`
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { isConfigSchema, type ObjectType } from '@kbn/config-schema';
|
||||
import type {
|
||||
SavedObjectUnsanitizedDoc,
|
||||
SavedObjectModelVersionForwardCompatibilitySchema,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
|
||||
function isObjectType(
|
||||
schema: SavedObjectModelVersionForwardCompatibilitySchema
|
||||
): schema is ObjectType {
|
||||
return isConfigSchema(schema);
|
||||
}
|
||||
|
||||
export type ConvertedSchema = (doc: SavedObjectUnsanitizedDoc) => SavedObjectUnsanitizedDoc;
|
||||
|
||||
export const convertModelVersionBackwardConversionSchema = (
|
||||
schema: SavedObjectModelVersionForwardCompatibilitySchema
|
||||
): ConvertedSchema => {
|
||||
if (isObjectType(schema)) {
|
||||
return (doc) => {
|
||||
const attrs = schema.validate(doc.attributes, {});
|
||||
return {
|
||||
...doc,
|
||||
attributes: attrs,
|
||||
};
|
||||
};
|
||||
} else {
|
||||
return (doc) => {
|
||||
const attrs = schema(doc.attributes);
|
||||
return {
|
||||
...doc,
|
||||
attributes: attrs,
|
||||
};
|
||||
};
|
||||
}
|
||||
};
|
||||
|
|
@ -36,3 +36,4 @@ export {
|
|||
export { getModelVersionDelta } from './get_version_delta';
|
||||
export { buildModelVersionTransformFn } from './build_transform_fn';
|
||||
export { aggregateMappingAdditions } from './aggregate_model_changes';
|
||||
export { convertModelVersionBackwardConversionSchema } from './backward_conversion_schema';
|
||||
|
|
|
|||
|
|
@ -17,9 +17,11 @@ jest.doMock('./internal_transforms', () => ({
|
|||
}));
|
||||
|
||||
export const getModelVersionTransformsMock = jest.fn();
|
||||
export const getModelVersionSchemasMock = jest.fn();
|
||||
|
||||
jest.doMock('./model_version', () => ({
|
||||
getModelVersionTransforms: getModelVersionTransformsMock,
|
||||
getModelVersionSchemas: getModelVersionSchemasMock,
|
||||
}));
|
||||
|
||||
export const validateTypeMigrationsMock = jest.fn();
|
||||
|
|
@ -33,5 +35,6 @@ export const resetAllMocks = () => {
|
|||
getReferenceTransformsMock.mockReset().mockReturnValue([]);
|
||||
getConversionTransformsMock.mockReset().mockReturnValue([]);
|
||||
getModelVersionTransformsMock.mockReset().mockReturnValue([]);
|
||||
getModelVersionSchemasMock.mockReset().mockReturnValue({});
|
||||
validateTypeMigrationsMock.mockReset();
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
getCoreTransformsMock,
|
||||
getConversionTransformsMock,
|
||||
getModelVersionTransformsMock,
|
||||
getModelVersionSchemasMock,
|
||||
getReferenceTransformsMock,
|
||||
resetAllMocks,
|
||||
validateTypeMigrationsMock,
|
||||
|
|
@ -195,6 +196,60 @@ describe('buildActiveMigrations', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('model version schemas', () => {
|
||||
it('calls getModelVersionSchemas with the correct parameters', () => {
|
||||
const foo = createType({ name: 'foo' });
|
||||
const bar = createType({ name: 'bar' });
|
||||
|
||||
addType(foo);
|
||||
addType(bar);
|
||||
|
||||
buildMigrations();
|
||||
|
||||
expect(getModelVersionSchemasMock).toHaveBeenCalledTimes(2);
|
||||
expect(getModelVersionSchemasMock).toHaveBeenNthCalledWith(1, {
|
||||
typeDefinition: foo,
|
||||
});
|
||||
expect(getModelVersionSchemasMock).toHaveBeenNthCalledWith(2, {
|
||||
typeDefinition: bar,
|
||||
});
|
||||
});
|
||||
|
||||
it('adds the schemas from getModelVersionSchemas to each type', () => {
|
||||
const foo = createType({ name: 'foo' });
|
||||
const bar = createType({ name: 'bar' });
|
||||
|
||||
addType(foo);
|
||||
addType(bar);
|
||||
|
||||
getModelVersionSchemasMock.mockImplementation(
|
||||
({ typeDefinition }: { typeDefinition: SavedObjectsType }) => {
|
||||
if (typeDefinition.name === 'foo') {
|
||||
return {
|
||||
'7.10.0': jest.fn(),
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
'8.3.0': jest.fn(),
|
||||
'8.4.0': jest.fn(),
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const migrations = buildMigrations();
|
||||
|
||||
expect(Object.keys(migrations).sort()).toEqual(['bar', 'foo']);
|
||||
expect(migrations.foo.versionSchemas).toEqual({
|
||||
'7.10.0': expect.any(Function),
|
||||
});
|
||||
expect(migrations.bar.versionSchemas).toEqual({
|
||||
'8.3.0': expect.any(Function),
|
||||
'8.4.0': expect.any(Function),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('internal transforms', () => {
|
||||
it('calls getReferenceTransforms with the correct parameters', () => {
|
||||
const foo = createType({ name: 'foo' });
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import {
|
|||
} from './internal_transforms';
|
||||
import { validateTypeMigrations } from './validate_migrations';
|
||||
import { transformComparator, convertMigrationFunction } from './utils';
|
||||
import { getModelVersionTransforms } from './model_version';
|
||||
import { getModelVersionTransforms, getModelVersionSchemas } from './model_version';
|
||||
|
||||
/**
|
||||
* Converts migrations from a format that is convenient for callers to a format that
|
||||
|
|
@ -48,7 +48,7 @@ export function buildActiveMigrations({
|
|||
referenceTransforms,
|
||||
});
|
||||
|
||||
if (typeTransforms.transforms.length) {
|
||||
if (typeTransforms.transforms.length || Object.keys(typeTransforms.versionSchemas).length) {
|
||||
migrations[type.name] = typeTransforms;
|
||||
}
|
||||
|
||||
|
|
@ -90,6 +90,8 @@ const buildTypeTransforms = ({
|
|||
...modelVersionTransforms,
|
||||
].sort(transformComparator);
|
||||
|
||||
const modelVersionSchemas = getModelVersionSchemas({ typeDefinition: type });
|
||||
|
||||
return {
|
||||
immediateVersion: _.chain(transforms)
|
||||
.groupBy('transformType')
|
||||
|
|
@ -106,5 +108,8 @@ const buildTypeTransforms = ({
|
|||
.mapValues((items) => _.last(items)?.version)
|
||||
.value() as Record<TransformType, string>,
|
||||
transforms,
|
||||
versionSchemas: {
|
||||
...modelVersionSchemas,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -25,18 +25,18 @@ const mockLoggerFactory = loggingSystemMock.create();
|
|||
const mockLogger = mockLoggerFactory.get('mock logger');
|
||||
const kibanaVersion = '25.2.3';
|
||||
|
||||
const createType = (parts: Partial<SavedObjectsType>): SavedObjectsType => ({
|
||||
name: 'unknown',
|
||||
namespaceType: 'single',
|
||||
hidden: false,
|
||||
mappings: { properties: {} },
|
||||
migrations: {},
|
||||
...parts,
|
||||
});
|
||||
|
||||
const createRegistry = (...types: Array<Partial<SavedObjectsType>>) => {
|
||||
const registry = new SavedObjectTypeRegistry();
|
||||
types.forEach((type) =>
|
||||
registry.registerType({
|
||||
name: 'unknown',
|
||||
namespaceType: 'single',
|
||||
hidden: false,
|
||||
mappings: { properties: {} },
|
||||
migrations: {},
|
||||
...type,
|
||||
})
|
||||
);
|
||||
types.forEach((type) => registry.registerType(createType(type)));
|
||||
registry.registerType({
|
||||
name: LEGACY_URL_ALIAS_TYPE,
|
||||
namespaceType: 'agnostic',
|
||||
|
|
@ -1377,6 +1377,95 @@ describe('DocumentMigrator', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('down migration', () => {
|
||||
it('accepts to downgrade the document if `allowDowngrade` is true', () => {
|
||||
const registry = createRegistry({});
|
||||
|
||||
const fooType = createType({
|
||||
name: 'foo',
|
||||
switchToModelVersionAt: '8.5.0',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: (attrs: any) => {
|
||||
return {
|
||||
foo: attrs.foo,
|
||||
};
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
registry.registerType(fooType);
|
||||
|
||||
const migrator = new DocumentMigrator({
|
||||
...testOpts(),
|
||||
typeRegistry: registry,
|
||||
});
|
||||
migrator.prepareMigrations();
|
||||
|
||||
const document: SavedObjectUnsanitizedDoc = {
|
||||
id: 'smelly',
|
||||
type: 'foo',
|
||||
attributes: {
|
||||
foo: 'bar',
|
||||
hello: 'dolly',
|
||||
},
|
||||
typeMigrationVersion: '10.2.0',
|
||||
};
|
||||
|
||||
const migrated = migrator.migrate(document, { allowDowngrade: true });
|
||||
|
||||
expect(migrated).toHaveProperty('typeMigrationVersion', '10.1.0');
|
||||
expect(migrated.attributes).toEqual({ foo: 'bar' });
|
||||
});
|
||||
|
||||
it('throws when trying to downgrade if `allowDowngrade` is false', () => {
|
||||
const registry = createRegistry({});
|
||||
|
||||
const fooType = createType({
|
||||
name: 'foo',
|
||||
switchToModelVersionAt: '8.5.0',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: (attrs: any) => {
|
||||
return {
|
||||
foo: attrs.foo,
|
||||
};
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
registry.registerType(fooType);
|
||||
|
||||
const migrator = new DocumentMigrator({
|
||||
...testOpts(),
|
||||
typeRegistry: registry,
|
||||
});
|
||||
migrator.prepareMigrations();
|
||||
|
||||
const document: SavedObjectUnsanitizedDoc = {
|
||||
id: 'smelly',
|
||||
type: 'foo',
|
||||
attributes: {
|
||||
foo: 'bar',
|
||||
hello: 'dolly',
|
||||
},
|
||||
typeMigrationVersion: '10.2.0',
|
||||
};
|
||||
|
||||
expect(() =>
|
||||
migrator.migrate(document, { allowDowngrade: false })
|
||||
).toThrowErrorMatchingInlineSnapshot(
|
||||
`"Document \\"smelly\\" belongs to a more recent version of Kibana [10.2.0] when the last known version is [10.1.0]."`
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function renameAttr(path: string, newPath: string) {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Boom from '@hapi/boom';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { SavedObjectsMigrationVersion } from '@kbn/core-saved-objects-common';
|
||||
import type {
|
||||
|
|
@ -16,9 +17,25 @@ import type { ActiveMigrations } from './types';
|
|||
import { maxVersion } from './pipelines/utils';
|
||||
import { buildActiveMigrations } from './build_active_migrations';
|
||||
import { DocumentUpgradePipeline, DocumentDowngradePipeline } from './pipelines';
|
||||
import { downgradeRequired } from './utils';
|
||||
import { TransformType } from './types';
|
||||
|
||||
/**
|
||||
* Options for {@link VersionedTransformer.migrate}
|
||||
*/
|
||||
export interface DocumentMigrateOptions {
|
||||
/**
|
||||
* Defines whether it is allowed to convert documents from an higher version or not.
|
||||
* - If `true`, documents from higher versions will go though the downgrade pipeline.
|
||||
* - If `false`, an error will be thrown when trying to process a document with an higher type version.
|
||||
* Defaults to `false`.
|
||||
*/
|
||||
allowDowngrade?: boolean;
|
||||
}
|
||||
|
||||
interface TransformOptions {
|
||||
convertNamespaceTypes?: boolean;
|
||||
allowDowngrade?: boolean;
|
||||
}
|
||||
|
||||
interface DocumentMigratorOptions {
|
||||
|
|
@ -49,23 +66,23 @@ export interface VersionedTransformer {
|
|||
/**
|
||||
* Migrates a document to its latest version.
|
||||
*/
|
||||
migrate(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc;
|
||||
migrate(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
options?: DocumentMigrateOptions
|
||||
): SavedObjectUnsanitizedDoc;
|
||||
|
||||
/**
|
||||
* Migrates a document to the latest version and applies type conversions if applicable.
|
||||
* Also returns any additional document(s) that may have been created during the transformation process.
|
||||
*
|
||||
* @remark This only be used by the savedObject migration during upgrade. For all other scenarios,
|
||||
* {@link VersionedTransformer#migrate} should be used instead.
|
||||
*/
|
||||
migrateAndConvert(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc[];
|
||||
/**
|
||||
* Converts a document down to the specified version.
|
||||
*/
|
||||
transformDown(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
options: { targetTypeVersion: string }
|
||||
): SavedObjectUnsanitizedDoc;
|
||||
}
|
||||
|
||||
/**
|
||||
* A concrete implementation of the VersionedTransformer interface.
|
||||
* A concrete implementation of the {@link VersionedTransformer} interface.
|
||||
*/
|
||||
export class DocumentMigrator implements VersionedTransformer {
|
||||
private options: DocumentMigratorOptions;
|
||||
|
|
@ -130,9 +147,13 @@ export class DocumentMigrator implements VersionedTransformer {
|
|||
/**
|
||||
* Migrates a document to the latest version.
|
||||
*/
|
||||
public migrate(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc {
|
||||
const { document } = this.transform(doc);
|
||||
|
||||
public migrate(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
{ allowDowngrade = false }: DocumentMigrateOptions = {}
|
||||
): SavedObjectUnsanitizedDoc {
|
||||
const { document } = this.transform(doc, {
|
||||
allowDowngrade,
|
||||
});
|
||||
return document;
|
||||
}
|
||||
|
||||
|
|
@ -141,32 +162,38 @@ export class DocumentMigrator implements VersionedTransformer {
|
|||
* have been created during the transformation process.
|
||||
*/
|
||||
public migrateAndConvert(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc[] {
|
||||
const { document, additionalDocs } = this.transform(doc, { convertNamespaceTypes: true });
|
||||
|
||||
return [document, ...additionalDocs];
|
||||
}
|
||||
|
||||
public transformDown(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
options: { targetTypeVersion: string }
|
||||
): SavedObjectUnsanitizedDoc {
|
||||
if (!this.migrations) {
|
||||
throw new Error('Migrations are not ready. Make sure prepareMigrations is called first.');
|
||||
}
|
||||
|
||||
const pipeline = new DocumentDowngradePipeline({
|
||||
document: doc,
|
||||
typeTransforms: this.migrations[doc.type],
|
||||
kibanaVersion: this.options.kibanaVersion,
|
||||
targetTypeVersion: options.targetTypeVersion,
|
||||
const { document, additionalDocs } = this.transform(doc, {
|
||||
convertNamespaceTypes: true,
|
||||
allowDowngrade: false,
|
||||
});
|
||||
const { document } = pipeline.run();
|
||||
return document;
|
||||
return [document, ...additionalDocs];
|
||||
}
|
||||
|
||||
private transform(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
{ convertNamespaceTypes = false }: TransformOptions = {}
|
||||
{ convertNamespaceTypes = false, allowDowngrade = false }: TransformOptions = {}
|
||||
) {
|
||||
if (!this.migrations) {
|
||||
throw new Error('Migrations are not ready. Make sure prepareMigrations is called first.');
|
||||
}
|
||||
const typeMigrations = this.migrations[doc.type];
|
||||
if (downgradeRequired(doc, typeMigrations?.latestVersion ?? {})) {
|
||||
const currentVersion = doc.typeMigrationVersion ?? doc.migrationVersion?.[doc.type];
|
||||
const latestVersion = this.migrations[doc.type].latestVersion[TransformType.Migrate];
|
||||
if (!allowDowngrade) {
|
||||
throw Boom.badData(
|
||||
`Document "${doc.id}" belongs to a more recent version of Kibana [${currentVersion}] when the last known version is [${latestVersion}].`
|
||||
);
|
||||
}
|
||||
return this.transformDown(doc, { targetTypeVersion: latestVersion! });
|
||||
} else {
|
||||
return this.transformUp(doc, { convertNamespaceTypes });
|
||||
}
|
||||
}
|
||||
|
||||
private transformUp(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
{ convertNamespaceTypes }: { convertNamespaceTypes: boolean }
|
||||
) {
|
||||
if (!this.migrations) {
|
||||
throw new Error('Migrations are not ready. Make sure prepareMigrations is called first.');
|
||||
|
|
@ -182,4 +209,23 @@ export class DocumentMigrator implements VersionedTransformer {
|
|||
|
||||
return { document, additionalDocs };
|
||||
}
|
||||
|
||||
private transformDown = (
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
{ targetTypeVersion }: { targetTypeVersion: string }
|
||||
) => {
|
||||
if (!this.migrations) {
|
||||
throw new Error('Migrations are not ready. Make sure prepareMigrations is called first.');
|
||||
}
|
||||
|
||||
const pipeline = new DocumentDowngradePipeline({
|
||||
document: doc,
|
||||
targetTypeVersion,
|
||||
typeTransforms: this.migrations[doc.type],
|
||||
kibanaVersion: this.options.kibanaVersion,
|
||||
});
|
||||
const { document } = pipeline.run();
|
||||
const additionalDocs: SavedObjectUnsanitizedDoc[] = [];
|
||||
return { document, additionalDocs };
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export const convertModelVersionBackwardConversionSchemaMock = jest.fn();
|
||||
|
||||
jest.doMock('@kbn/core-saved-objects-base-server-internal', () => {
|
||||
const actual = jest.requireActual('@kbn/core-saved-objects-base-server-internal');
|
||||
return {
|
||||
...actual,
|
||||
convertModelVersionBackwardConversionSchema: convertModelVersionBackwardConversionSchemaMock,
|
||||
};
|
||||
});
|
||||
|
|
@ -6,6 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { convertModelVersionBackwardConversionSchemaMock } from './model_version.test.mocks';
|
||||
import { loggerMock, MockedLogger } from '@kbn/logging-mocks';
|
||||
import type {
|
||||
SavedObjectsType,
|
||||
|
|
@ -15,7 +16,19 @@ import type {
|
|||
} from '@kbn/core-saved-objects-server';
|
||||
import { modelVersionToVirtualVersion } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { Transform, TransformType } from './types';
|
||||
import { getModelVersionTransforms, convertModelVersionTransformFn } from './model_version';
|
||||
import {
|
||||
getModelVersionTransforms,
|
||||
convertModelVersionTransformFn,
|
||||
getModelVersionSchemas,
|
||||
} from './model_version';
|
||||
|
||||
const createType = (parts: Partial<SavedObjectsType>): SavedObjectsType => ({
|
||||
name: 'test',
|
||||
hidden: false,
|
||||
namespaceType: 'single',
|
||||
mappings: { properties: {} },
|
||||
...parts,
|
||||
});
|
||||
|
||||
describe('getModelVersionTransforms', () => {
|
||||
let log: MockedLogger;
|
||||
|
|
@ -26,14 +39,6 @@ describe('getModelVersionTransforms', () => {
|
|||
transform: expect.any(Function),
|
||||
});
|
||||
|
||||
const createType = (parts: Partial<SavedObjectsType>): SavedObjectsType => ({
|
||||
name: 'test',
|
||||
hidden: false,
|
||||
namespaceType: 'single',
|
||||
mappings: { properties: {} },
|
||||
...parts,
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
log = loggerMock.create();
|
||||
});
|
||||
|
|
@ -208,3 +213,77 @@ describe('convertModelVersionTransformFn', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getModelVersionSchemas', () => {
|
||||
beforeEach(() => {
|
||||
convertModelVersionBackwardConversionSchemaMock.mockReset();
|
||||
convertModelVersionBackwardConversionSchemaMock.mockImplementation(() => jest.fn());
|
||||
});
|
||||
|
||||
it('calls convertModelVersionBackwardConversionSchema with the correct parameters', () => {
|
||||
const schema1 = jest.fn();
|
||||
const schema3 = jest.fn();
|
||||
|
||||
const typeDefinition = createType({
|
||||
name: 'foo',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: schema1,
|
||||
},
|
||||
},
|
||||
2: {
|
||||
changes: [],
|
||||
schemas: {},
|
||||
},
|
||||
3: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: schema3,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
getModelVersionSchemas({ typeDefinition });
|
||||
|
||||
expect(convertModelVersionBackwardConversionSchemaMock).toHaveBeenCalledTimes(2);
|
||||
expect(convertModelVersionBackwardConversionSchemaMock).toHaveBeenCalledWith(schema1);
|
||||
expect(convertModelVersionBackwardConversionSchemaMock).toHaveBeenCalledWith(schema3);
|
||||
});
|
||||
|
||||
it('generate schemas for correct model versions', () => {
|
||||
const schema1 = jest.fn();
|
||||
const schema3 = jest.fn();
|
||||
|
||||
const typeDefinition = createType({
|
||||
name: 'foo',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: schema1,
|
||||
},
|
||||
},
|
||||
2: {
|
||||
changes: [],
|
||||
schemas: {},
|
||||
},
|
||||
3: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: schema3,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const schemas = getModelVersionSchemas({ typeDefinition });
|
||||
|
||||
expect(schemas).toEqual({
|
||||
[modelVersionToVirtualVersion(1)]: expect.any(Function),
|
||||
[modelVersionToVirtualVersion(3)]: expect.any(Function),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -17,9 +17,31 @@ import {
|
|||
modelVersionToVirtualVersion,
|
||||
assertValidModelVersion,
|
||||
buildModelVersionTransformFn,
|
||||
convertModelVersionBackwardConversionSchema,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { TransformSavedObjectDocumentError } from '../core';
|
||||
import { type Transform, type TransformFn, TransformType } from './types';
|
||||
import { type Transform, type TransformFn, TransformType, type TypeVersionSchema } from './types';
|
||||
|
||||
export const getModelVersionSchemas = ({
|
||||
typeDefinition,
|
||||
}: {
|
||||
typeDefinition: SavedObjectsType;
|
||||
}): Record<string, TypeVersionSchema> => {
|
||||
const modelVersionMap =
|
||||
typeof typeDefinition.modelVersions === 'function'
|
||||
? typeDefinition.modelVersions()
|
||||
: typeDefinition.modelVersions ?? {};
|
||||
|
||||
return Object.entries(modelVersionMap).reduce((map, [rawModelVersion, versionDefinition]) => {
|
||||
const schema = versionDefinition.schemas?.forwardCompatibility;
|
||||
if (schema) {
|
||||
const modelVersion = assertValidModelVersion(rawModelVersion);
|
||||
const virtualVersion = modelVersionToVirtualVersion(modelVersion);
|
||||
map[virtualVersion] = convertModelVersionBackwardConversionSchema(schema);
|
||||
}
|
||||
return map;
|
||||
}, {} as Record<string, TypeVersionSchema>);
|
||||
};
|
||||
|
||||
export const getModelVersionTransforms = ({
|
||||
typeDefinition,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
import _ from 'lodash';
|
||||
import type { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server';
|
||||
import { Transform, TransformType, TypeTransforms, TransformFn } from '../types';
|
||||
import { Transform, TransformType, TypeTransforms, TransformFn, TypeVersionSchema } from '../types';
|
||||
import { DocumentDowngradePipeline } from './downgrade_pipeline';
|
||||
|
||||
// snake case is way better for migration function names in this very specific scenario.
|
||||
|
|
@ -28,6 +28,10 @@ describe('DocumentMigratorPipeline', () => {
|
|||
...parts,
|
||||
});
|
||||
|
||||
const createSchema = (): jest.MockedFunction<TypeVersionSchema> => {
|
||||
return jest.fn().mockImplementation((doc: unknown) => doc);
|
||||
};
|
||||
|
||||
const latestVersions = (
|
||||
parts: Partial<Record<TransformType, string>> = {}
|
||||
): Record<TransformType, string> => ({
|
||||
|
|
@ -48,6 +52,7 @@ describe('DocumentMigratorPipeline', () => {
|
|||
transforms,
|
||||
immediateVersion: latestVersions(versions),
|
||||
latestVersion: latestVersions(versions),
|
||||
versionSchemas: {},
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -180,7 +185,7 @@ describe('DocumentMigratorPipeline', () => {
|
|||
expect(outputDoc.typeMigrationVersion).toEqual('8.7.0');
|
||||
});
|
||||
|
||||
it('throws trying to apply a transform without down fn', () => {
|
||||
it('skips transforms without down fn', () => {
|
||||
const document = createDoc({
|
||||
id: 'foo-1',
|
||||
type: 'foo',
|
||||
|
|
@ -220,9 +225,10 @@ describe('DocumentMigratorPipeline', () => {
|
|||
targetTypeVersion: '8.5.0',
|
||||
});
|
||||
|
||||
expect(() => pipeline.run()).toThrowErrorMatchingInlineSnapshot(
|
||||
`"Could not apply transformation migrate:8.7.0: no down conversion registered"`
|
||||
);
|
||||
pipeline.run();
|
||||
|
||||
expect(migrate8_6_0_down).toHaveBeenCalledTimes(1);
|
||||
expect(migrate8_8_0_down).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('throws trying to downgrade to a higher version', () => {
|
||||
|
|
@ -604,4 +610,113 @@ describe('DocumentMigratorPipeline', () => {
|
|||
|
||||
expect(outputDoc.coreMigrationVersion).toEqual('8.7.0');
|
||||
});
|
||||
|
||||
it('accepts converting documents from higher versions than the last known', () => {
|
||||
const document = createDoc({
|
||||
id: 'foo-1',
|
||||
type: 'foo',
|
||||
typeMigrationVersion: '8.10.0',
|
||||
});
|
||||
|
||||
const migrate8_8_0_up = createTransformFn();
|
||||
const migrate8_8_0_down = createTransformFn();
|
||||
|
||||
const fooTransforms = getTypeTransforms([
|
||||
{
|
||||
transformType: TransformType.Migrate,
|
||||
version: '8.8.0',
|
||||
transform: migrate8_8_0_up,
|
||||
transformDown: migrate8_8_0_down,
|
||||
},
|
||||
]);
|
||||
|
||||
const pipeline = new DocumentDowngradePipeline({
|
||||
document,
|
||||
kibanaVersion: '8.8.0',
|
||||
typeTransforms: fooTransforms,
|
||||
targetTypeVersion: '8.7.0',
|
||||
});
|
||||
|
||||
const { document: outputDoc } = pipeline.run();
|
||||
|
||||
expect(migrate8_8_0_up).not.toHaveBeenCalled();
|
||||
|
||||
expect(migrate8_8_0_down).toHaveBeenCalledTimes(1);
|
||||
expect(migrate8_8_0_down).toHaveBeenCalledWith(document);
|
||||
|
||||
expect(outputDoc.typeMigrationVersion).toEqual('8.7.0');
|
||||
});
|
||||
|
||||
describe('version schemas', () => {
|
||||
it('apply the correct version schema', () => {
|
||||
const document = createDoc({
|
||||
id: 'foo-1',
|
||||
type: 'foo',
|
||||
typeMigrationVersion: '8.9.0',
|
||||
});
|
||||
|
||||
const schema_8_7_0 = createSchema();
|
||||
const schema_8_8_0 = createSchema();
|
||||
const schema_8_9_0 = createSchema();
|
||||
|
||||
const transforms: TypeTransforms = {
|
||||
transforms: [],
|
||||
latestVersion: latestVersions(),
|
||||
immediateVersion: latestVersions(),
|
||||
versionSchemas: {
|
||||
'8.7.0': schema_8_7_0,
|
||||
'8.8.0': schema_8_8_0,
|
||||
'8.9.0': schema_8_9_0,
|
||||
},
|
||||
};
|
||||
|
||||
const pipeline = new DocumentDowngradePipeline({
|
||||
document,
|
||||
kibanaVersion: '8.8.0',
|
||||
typeTransforms: transforms,
|
||||
targetTypeVersion: '8.7.0',
|
||||
});
|
||||
|
||||
const { document: outputDoc } = pipeline.run();
|
||||
|
||||
expect(outputDoc.typeMigrationVersion).toEqual('8.7.0');
|
||||
expect(schema_8_7_0).toHaveBeenCalledTimes(1);
|
||||
expect(schema_8_8_0).not.toHaveBeenCalled();
|
||||
expect(schema_8_9_0).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not apply the schema if the exact version is missing', () => {
|
||||
const document = createDoc({
|
||||
id: 'foo-1',
|
||||
type: 'foo',
|
||||
typeMigrationVersion: '8.9.0',
|
||||
});
|
||||
|
||||
const schema_8_8_0 = createSchema();
|
||||
const schema_8_9_0 = createSchema();
|
||||
|
||||
const transforms: TypeTransforms = {
|
||||
transforms: [],
|
||||
latestVersion: latestVersions(),
|
||||
immediateVersion: latestVersions(),
|
||||
versionSchemas: {
|
||||
'8.8.0': schema_8_8_0,
|
||||
'8.9.0': schema_8_9_0,
|
||||
},
|
||||
};
|
||||
|
||||
const pipeline = new DocumentDowngradePipeline({
|
||||
document,
|
||||
kibanaVersion: '8.8.0',
|
||||
typeTransforms: transforms,
|
||||
targetTypeVersion: '8.7.0',
|
||||
});
|
||||
|
||||
const { document: outputDoc } = pipeline.run();
|
||||
|
||||
expect(outputDoc.typeMigrationVersion).toEqual('8.7.0');
|
||||
expect(schema_8_8_0).not.toHaveBeenCalled();
|
||||
expect(schema_8_9_0).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -48,9 +48,7 @@ export class DocumentDowngradePipeline implements MigrationPipeline {
|
|||
|
||||
for (const transform of this.getPendingTransforms()) {
|
||||
if (!transform.transformDown) {
|
||||
throw new Error(
|
||||
`Could not apply transformation ${transform.transformType}:${transform.version}: no down conversion registered`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const { transformedDoc } = transform.transformDown(this.document);
|
||||
if (this.document.type !== this.originalDoc.type) {
|
||||
|
|
@ -60,6 +58,7 @@ export class DocumentDowngradePipeline implements MigrationPipeline {
|
|||
}
|
||||
|
||||
this.document = this.ensureVersion(this.document);
|
||||
this.document = this.applyVersionSchema(this.document);
|
||||
|
||||
return {
|
||||
document: this.document,
|
||||
|
|
@ -103,14 +102,7 @@ export class DocumentDowngradePipeline implements MigrationPipeline {
|
|||
* And that the targetTypeVersion is not greater than the document's
|
||||
*/
|
||||
private assertCompatibility() {
|
||||
const { id, typeMigrationVersion: currentVersion } = this.document;
|
||||
const latestVersion = this.typeTransforms.latestVersion.migrate;
|
||||
|
||||
if (currentVersion && Semver.gt(currentVersion, latestVersion)) {
|
||||
throw new Error(
|
||||
`Document "${id}" belongs to a more recent version of Kibana [${currentVersion}] when the last known version is [${latestVersion}].`
|
||||
);
|
||||
}
|
||||
const { typeMigrationVersion: currentVersion } = this.document;
|
||||
|
||||
if (currentVersion && Semver.gt(this.targetTypeVersion, currentVersion)) {
|
||||
throw new Error(
|
||||
|
|
@ -132,4 +124,13 @@ export class DocumentDowngradePipeline implements MigrationPipeline {
|
|||
...(coreMigrationVersion ? { coreMigrationVersion } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
private applyVersionSchema(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc {
|
||||
const targetVersion = this.targetTypeVersion;
|
||||
const versionSchema = this.typeTransforms.versionSchemas[targetVersion];
|
||||
if (versionSchema) {
|
||||
return versionSchema(doc);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ describe('DocumentMigratorPipeline', () => {
|
|||
transforms,
|
||||
immediateVersion: latestVersions(versions),
|
||||
latestVersion: latestVersions(versions),
|
||||
versionSchemas: {},
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ export interface TypeTransforms {
|
|||
latestVersion: Record<TransformType, string>;
|
||||
/** Ordered list of transforms registered for the type **/
|
||||
transforms: Transform[];
|
||||
/** Per-version schemas for the given type */
|
||||
versionSchemas: Record<string, TypeVersionSchema>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -94,3 +96,8 @@ export interface TransformResult {
|
|||
*/
|
||||
additionalDocs: SavedObjectUnsanitizedDoc[];
|
||||
}
|
||||
|
||||
/**
|
||||
* per-version persistence schema for {@link TypeTransforms}
|
||||
*/
|
||||
export type TypeVersionSchema = (doc: SavedObjectUnsanitizedDoc) => SavedObjectUnsanitizedDoc;
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { SavedObjectUnsanitizedDoc } from '@kbn/core-saved-objects-server';
|
||||
import { type Transform, TransformType } from './types';
|
||||
import { transformComparator } from './utils';
|
||||
import { transformComparator, downgradeRequired } from './utils';
|
||||
|
||||
describe('transformComparator', () => {
|
||||
const core1 = { version: '1.0.0', transformType: TransformType.Core } as Transform;
|
||||
|
|
@ -31,3 +32,126 @@ describe('transformComparator', () => {
|
|||
expect(transforms.sort(transformComparator)).toEqual(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('downgradeRequired', () => {
|
||||
const createDoc = (parts: Partial<SavedObjectUnsanitizedDoc>): SavedObjectUnsanitizedDoc => ({
|
||||
type: 'type',
|
||||
id: 'id',
|
||||
attributes: {},
|
||||
...parts,
|
||||
});
|
||||
|
||||
it('returns false when there is an higher convert version than the typeMigrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
typeMigrationVersion: '8.0.0',
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Convert]: '8.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(false);
|
||||
});
|
||||
|
||||
it('returns false when there is an higher convert version than the migrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
migrationVersion: {
|
||||
type: '8.0.0',
|
||||
},
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Convert]: '8.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(false);
|
||||
});
|
||||
|
||||
it('returns false when there is an higher migrate version than the typeMigrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
typeMigrationVersion: '8.0.0',
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Migrate]: '8.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(false);
|
||||
});
|
||||
|
||||
it('returns false when there is an higher migrate version than the migrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
migrationVersion: {
|
||||
type: '8.0.0',
|
||||
},
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Migrate]: '8.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(false);
|
||||
});
|
||||
|
||||
it('returns true when there is no higher convert version than the typeMigrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
typeMigrationVersion: '8.0.0',
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Convert]: '7.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(true);
|
||||
});
|
||||
|
||||
it('returns true when there is no higher convert version than the migrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
migrationVersion: {
|
||||
type: '8.0.0',
|
||||
},
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Convert]: '7.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(true);
|
||||
});
|
||||
|
||||
it('returns true when there is no higher migrate version than the typeMigrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
typeMigrationVersion: '8.0.0',
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Migrate]: '7.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(true);
|
||||
});
|
||||
|
||||
it('returns true when there is no higher migrate version than the migrationVersion', () => {
|
||||
const doc = createDoc({
|
||||
migrationVersion: {
|
||||
type: '8.0.0',
|
||||
},
|
||||
});
|
||||
const latestVersions = {
|
||||
[TransformType.Migrate]: '7.1.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(true);
|
||||
});
|
||||
|
||||
it('returns false when the document has no explicit version', () => {
|
||||
const doc = createDoc({});
|
||||
const latestVersions = {
|
||||
[TransformType.Migrate]: '8.0.0',
|
||||
} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(false);
|
||||
});
|
||||
|
||||
it('returns false when latestVersions no explicit version', () => {
|
||||
const doc = createDoc({
|
||||
typeMigrationVersion: '8.0.0',
|
||||
});
|
||||
const latestVersions = {} as Record<TransformType, string>;
|
||||
|
||||
expect(downgradeRequired(doc, latestVersions)).toEqual(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import type {
|
|||
} from '@kbn/core-saved-objects-server';
|
||||
import { Logger } from '@kbn/logging';
|
||||
import { MigrationLogger } from '../core/migration_logger';
|
||||
import { maxVersion } from './pipelines/utils';
|
||||
import { TransformSavedObjectDocumentError } from '../core/transform_saved_object_document_error';
|
||||
import { type Transform, type TransformFn, TransformType } from './types';
|
||||
|
||||
|
|
@ -85,3 +86,21 @@ export function transformComparator(a: Transform, b: Transform) {
|
|||
|
||||
return Semver.compare(a.version, b.version) || aPriority - bPriority;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given document has an higher version that the last known version, false otherwise
|
||||
*/
|
||||
export function downgradeRequired(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
latestVersions: Record<TransformType, string>
|
||||
): boolean {
|
||||
const docTypeVersion = doc.typeMigrationVersion ?? doc.migrationVersion?.[doc.type];
|
||||
const latestMigrationVersion = maxVersion(
|
||||
latestVersions[TransformType.Migrate],
|
||||
latestVersions[TransformType.Convert]
|
||||
);
|
||||
if (!docTypeVersion || !latestMigrationVersion) {
|
||||
return false;
|
||||
}
|
||||
return Semver.gt(docTypeVersion, latestMigrationVersion);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
type SavedObjectsTypeMappingDefinitions,
|
||||
type SavedObjectsMigrationConfigType,
|
||||
type IKibanaMigrator,
|
||||
type MigrateDocumentOptions,
|
||||
type KibanaMigratorStatus,
|
||||
type MigrationResult,
|
||||
type IndexTypesMap,
|
||||
|
|
@ -266,7 +267,10 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
return this.activeMappings;
|
||||
}
|
||||
|
||||
public migrateDocument(doc: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc {
|
||||
return this.documentMigrator.migrate(doc);
|
||||
public migrateDocument(
|
||||
doc: SavedObjectUnsanitizedDoc,
|
||||
{ allowDowngrade = false }: MigrateDocumentOptions = {}
|
||||
): SavedObjectUnsanitizedDoc {
|
||||
return this.documentMigrator.migrate(doc, { allowDowngrade });
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,5 @@ export const createDocumentMigrator = (): jest.Mocked<VersionedTransformer> => {
|
|||
return {
|
||||
migrate: jest.fn().mockImplementation((doc: unknown) => doc),
|
||||
migrateAndConvert: jest.fn().mockImplementation((doc: unknown) => [doc]),
|
||||
transformDown: jest.fn().mockImplementation((doc: unknown) => doc),
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -135,7 +135,8 @@ export type {
|
|||
SavedObjectModelTransformationResult,
|
||||
SavedObjectModelDataBackfillFn,
|
||||
SavedObjectsModelVersionSchemaDefinitions,
|
||||
SavedObjectModelVersionCreateSchema,
|
||||
SavedObjectModelVersionForwardCompatibilityFn,
|
||||
SavedObjectModelVersionForwardCompatibilitySchema,
|
||||
} from './src/model_version';
|
||||
|
||||
// We re-export the SavedObject types here for convenience.
|
||||
|
|
|
|||
|
|
@ -30,5 +30,6 @@ export type {
|
|||
|
||||
export type {
|
||||
SavedObjectsModelVersionSchemaDefinitions,
|
||||
SavedObjectModelVersionCreateSchema,
|
||||
SavedObjectModelVersionForwardCompatibilitySchema,
|
||||
SavedObjectModelVersionForwardCompatibilityFn,
|
||||
} from './schemas';
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ export interface SavedObjectsModelVersion {
|
|||
*/
|
||||
changes: SavedObjectsModelChange[];
|
||||
/**
|
||||
* The {@link SavedObjectsModelVersionSchemaDefinitions} associated with this version.
|
||||
* The {@link SavedObjectsModelVersionSchemaDefinitions | schemas} associated with this version.
|
||||
*
|
||||
* @remark Currently unimplemented and unused.
|
||||
* Schemas are used to validate / convert the shape and/or content of the documents at various stages of their usages.
|
||||
*/
|
||||
schemas?: SavedObjectsModelVersionSchemaDefinitions;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,18 +8,69 @@
|
|||
|
||||
import type { ObjectType } from '@kbn/config-schema';
|
||||
|
||||
/**
|
||||
* @public
|
||||
* @remark Currently unimplemented and unused.
|
||||
*/
|
||||
export type SavedObjectModelVersionCreateSchema = ObjectType;
|
||||
|
||||
/**
|
||||
* The schemas associated with this model version.
|
||||
*
|
||||
* @public
|
||||
* @remark Currently unimplemented and unused.
|
||||
*/
|
||||
export interface SavedObjectsModelVersionSchemaDefinitions {
|
||||
create?: SavedObjectModelVersionCreateSchema;
|
||||
/**
|
||||
* The schema applied when retrieving documents of a higher version from the cluster.
|
||||
* Used for multi-version compatibility in managed environments.
|
||||
*
|
||||
* When retrieving a savedObject document from an index, if the version of the document
|
||||
* is higher than the latest version known of the Kibana instance, the document will go
|
||||
* through the `forwardCompatibility` schema of the associated model version.
|
||||
*
|
||||
* E.g a Kibana instance with model version `2` for type `foo` types fetches a `foo` document
|
||||
* at model version `3`. The document will then go through the `forwardCompatibility`
|
||||
* of the model version 2 (if present).
|
||||
*
|
||||
* See {@link SavedObjectModelVersionForwardCompatibilitySchema} for more info.
|
||||
*/
|
||||
forwardCompatibility?: SavedObjectModelVersionForwardCompatibilitySchema;
|
||||
}
|
||||
|
||||
/**
|
||||
* Plain javascript function alternative for {@link SavedObjectModelVersionForwardCompatibilitySchema}
|
||||
* @public
|
||||
*/
|
||||
export type SavedObjectModelVersionForwardCompatibilityFn<InAttrs = unknown, OutAttrs = unknown> = (
|
||||
attributes: InAttrs
|
||||
) => OutAttrs;
|
||||
|
||||
/**
|
||||
* Schema used when retrieving a document of a higher version to convert them to the older version.
|
||||
*
|
||||
* These schemas can be defined in multiple ways:
|
||||
* - A `@kbn/config-schema`'s Object schema, that will receive the document's attributes
|
||||
* - An arbitrary function that will receive the document's attributes as parameter and should return the converted attributes
|
||||
*
|
||||
* @remark These conversion mechanism shouldn't assert the data itself, only strip unknown fields to convert
|
||||
* the document to the *shape* of the document at the given version.
|
||||
*
|
||||
* @example using a function:
|
||||
* ```ts
|
||||
* const versionSchema: SavedObjectModelVersionEvictionFn = (attributes) => {
|
||||
* const knownFields = ['someField', 'anotherField'];
|
||||
* return pick(attributes, knownFields);
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* @example using config-schema:
|
||||
* ```ts
|
||||
* const versionSchema = schema.object(
|
||||
* {
|
||||
* someField: schema.maybe(schema.string()),
|
||||
* anotherField: schema.maybe(schema.string()),
|
||||
* },
|
||||
* { unknowns: 'ignore' }
|
||||
* );
|
||||
* ```
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type SavedObjectModelVersionForwardCompatibilitySchema<
|
||||
InAttrs = unknown,
|
||||
OutAttrs = unknown
|
||||
> = ObjectType | SavedObjectModelVersionForwardCompatibilityFn<InAttrs, OutAttrs>;
|
||||
|
|
|
|||
|
|
@ -202,12 +202,18 @@ describe('extractMigrationInfo', () => {
|
|||
changeTypes: ['data_backfill'],
|
||||
hasTransformation: true,
|
||||
newMappings: [],
|
||||
schemas: {
|
||||
forwardCompatibility: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '2',
|
||||
changeTypes: ['mappings_addition'],
|
||||
hasTransformation: false,
|
||||
newMappings: ['foo.type'],
|
||||
schemas: {
|
||||
forwardCompatibility: false,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -245,12 +251,18 @@ describe('extractMigrationInfo', () => {
|
|||
changeTypes: ['data_backfill'],
|
||||
hasTransformation: true,
|
||||
newMappings: [],
|
||||
schemas: {
|
||||
forwardCompatibility: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '2',
|
||||
changeTypes: ['mappings_addition'],
|
||||
hasTransformation: false,
|
||||
newMappings: ['foo.type'],
|
||||
schemas: {
|
||||
forwardCompatibility: false,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -263,6 +275,32 @@ describe('extractMigrationInfo', () => {
|
|||
|
||||
expect(output.modelVersions).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns the correct values for schemas', () => {
|
||||
const type = createType({
|
||||
switchToModelVersionAt: '8.8.0',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: jest.fn(),
|
||||
},
|
||||
},
|
||||
2: {
|
||||
changes: [],
|
||||
schemas: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
const output = extractMigrationInfo(type);
|
||||
|
||||
expect(output.modelVersions[0].schemas).toEqual({
|
||||
forwardCompatibility: true,
|
||||
});
|
||||
expect(output.modelVersions[1].schemas).toEqual({
|
||||
forwardCompatibility: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('migrations and modelVersions', () => {
|
||||
|
|
@ -310,12 +348,18 @@ describe('extractMigrationInfo', () => {
|
|||
changeTypes: ['data_backfill'],
|
||||
hasTransformation: true,
|
||||
newMappings: [],
|
||||
schemas: {
|
||||
forwardCompatibility: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
version: '2',
|
||||
changeTypes: ['mappings_addition'],
|
||||
hasTransformation: false,
|
||||
newMappings: ['foo.type'],
|
||||
schemas: {
|
||||
forwardCompatibility: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ export interface ModelVersionSummary {
|
|||
changeTypes: string[];
|
||||
hasTransformation: boolean;
|
||||
newMappings: string[];
|
||||
schemas: {
|
||||
forwardCompatibility: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -53,13 +56,16 @@ export const extractMigrationInfo = (soType: SavedObjectsType): SavedObjectTypeM
|
|||
? soType.modelVersions()
|
||||
: soType.modelVersions ?? {};
|
||||
const modelVersionIds = Object.keys(modelVersionMap);
|
||||
const modelVersions = modelVersionIds.map((version) => {
|
||||
const modelVersions = modelVersionIds.map<ModelVersionSummary>((version) => {
|
||||
const entry = modelVersionMap[version];
|
||||
return {
|
||||
version,
|
||||
changeTypes: entry.changes.map((change) => change.type),
|
||||
hasTransformation: hasTransformation(entry.changes),
|
||||
newMappings: Object.keys(getFlattenedObject(aggregateMappingAdditions(entry.changes))),
|
||||
schemas: {
|
||||
forwardCompatibility: !!entry.schemas?.forwardCompatibility,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -518,6 +518,29 @@ describe('getMigrationHash', () => {
|
|||
|
||||
expect(getMigrationHash(typeA)).not.toEqual(getMigrationHash(typeB));
|
||||
});
|
||||
|
||||
it('returns different hashes if different schemas are registered', () => {
|
||||
const typeA = createType({
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {
|
||||
forwardCompatibility: jest.fn(),
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const typeB = createType({
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
schemas: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(getMigrationHash(typeA)).not.toEqual(getMigrationHash(typeB));
|
||||
});
|
||||
});
|
||||
|
||||
describe('ignored fields', () => {
|
||||
|
|
|
|||
|
|
@ -35,10 +35,12 @@ export const getMigrationHash = (soType: SavedObjectsType): SavedObjectTypeMigra
|
|||
};
|
||||
|
||||
const serializeModelVersion = (modelVersion: ModelVersionSummary): string => {
|
||||
const schemas = [modelVersion.schemas.forwardCompatibility];
|
||||
return [
|
||||
modelVersion.version,
|
||||
modelVersion.changeTypes.join(','),
|
||||
modelVersion.hasTransformation,
|
||||
schemas.join(','),
|
||||
...modelVersion.newMappings,
|
||||
].join('|');
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue