github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Saved Objects] Update document migrator not to bump core version on every migration (#150443)

Browse source
This commit is contained in:
Michael Dokolin 2023-02-10 17:45:49 +01:00 • committed by GitHub
parent 67f01ed27d
commit caee8cbe8e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 258 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

4
packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/repository.test.ts
View file

@ -72,7 +72,6 @@ import { errors as EsErrors } from '@elastic/elasticsearch';
import type { InternalBulkResolveError } from './internal_bulk_resolve';
import {
KIBANA_VERSION,
CUSTOM_INDEX_TYPE,
NAMESPACE_AGNOSTIC_TYPE,
MULTI_NAMESPACE_TYPE,
@ -935,7 +934,6 @@ describe('SavedObjectsRepository', () => {
...response.items[0].create,
_source: {
...response.items[0].create._source,
coreMigrationVersion: '2.0.0', // the document migrator adds this to all objects before creation
namespaces: response.items[0].create._source.namespaces,
},
_id: expect.stringMatching(/^myspace:config:[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$/),
@ -944,7 +942,6 @@ describe('SavedObjectsRepository', () => {
...response.items[1].create,
_source: {
...response.items[1].create._source,
coreMigrationVersion: '2.0.0', // the document migrator adds this to all objects before creation
namespaces: response.items[1].create._source.namespaces,
},
});
@ -2951,7 +2948,6 @@ describe('SavedObjectsRepository', () => {
references,
namespaces: [namespace ?? 'default'],
migrationVersion: { [MULTI_NAMESPACE_TYPE]: '1.1.1' },
coreMigrationVersion: KIBANA_VERSION,
});
});
});

1
packages/core/saved-objects/core-saved-objects-api-server-internal/src/test_helpers/repository.test.common.ts
View file

@ -596,7 +596,6 @@ export const expectCreateResult = (obj: {
}) => ({
...obj,
migrationVersion: { [obj.type]: '1.1.1' },
coreMigrationVersion: KIBANA_VERSION,
version: mockVersion,
namespaces: obj.namespaces ?? [obj.namespace ?? 'default'],
...mockTimestampFieldsWithCreated,

22
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/build_active_migrations.ts
View file

@ -9,7 +9,7 @@
import _ from 'lodash';
import type { Logger } from '@kbn/logging';
import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server';
import type { Transform, ActiveMigrations } from './types';
import { type ActiveMigrations, type Transform, TransformType } from './types';
import { getReferenceTransforms, getConversionTransforms } from './internal_transforms';
import { validateMigrationsMapObject } from './validate_migrations';
import { transformComparator, wrapWithTry } from './utils';
@ -18,7 +18,7 @@ import { transformComparator, wrapWithTry } from './utils';
* Converts migrations from a format that is convenient for callers to a format that
* is convenient for our internal usage:
* From: { type: { version: fn } }
* To: { type: { latestMigrationVersion?: string; latestCoreMigrationVersion?: string; transforms: [{ version: string, transform: fn }] } }
* To: { type: { latestVersion?: Record<TransformType, string>; transforms: [{ version: string, transform: fn }] } }
*/
export function buildActiveMigrations(
typeRegistry: ISavedObjectTypeRegistry,
@ -36,7 +36,7 @@ export function buildActiveMigrations(
([version, transform]) => ({
version,
transform: wrapWithTry(version, type, transform, log),
transformType: 'migrate',
transformType: TransformType.Migrate,
})
);
const conversionTransforms = getConversionTransforms(type);
@ -50,21 +50,13 @@ export function buildActiveMigrations(
return migrations;
}
const migrationVersionTransforms: Transform[] = [];
const coreMigrationVersionTransforms: Transform[] = [];
transforms.forEach((x) => {
if (x.transformType === 'migrate' || x.transformType === 'convert') {
migrationVersionTransforms.push(x);
} else {
coreMigrationVersionTransforms.push(x);
}
});
return {
...migrations,
[type.name]: {
latestMigrationVersion: _.last(migrationVersionTransforms)?.version,
latestCoreMigrationVersion: _.last(coreMigrationVersionTransforms)?.version,
latestVersion: _.chain(transforms)
.groupBy('transformType')
.mapValues((items) => _.last(items)?.version)
.value() as Record<TransformType, string>,
transforms,
},
};

108
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.test.ts
View file

@ -255,7 +255,6 @@ describe('DocumentMigrator', () => {
type: 'user',
attributes: { name: 'Chris' },
migrationVersion: { user: '1.2.3' },
coreMigrationVersion: kibanaVersion,
});
});
@ -308,7 +307,6 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Tyler' },
migrationVersion: { acl: '2.3.5' },
acl: 'admins-only, sucka!',
coreMigrationVersion: kibanaVersion,
});
});
@ -347,7 +345,6 @@ describe('DocumentMigrator', () => {
id: 'me',
type: 'user',
attributes: { name: 'Tyler' },
coreMigrationVersion: kibanaVersion,
});
});
@ -420,7 +417,6 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Callie', b: 'B', c: 'C' },
migrationVersion: { dog: '2.0.1' },
coreMigrationVersion: kibanaVersion,
});
});
@ -525,7 +521,6 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Callie', a: 1, b: 2, c: 3 },
migrationVersion: { dog: '10.0.1' },
coreMigrationVersion: kibanaVersion,
});
});
@ -560,7 +555,6 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Callie' },
animal: 'Animal: Doggie',
migrationVersion: { animal: '1.0.0', dog: '2.2.4' },
coreMigrationVersion: kibanaVersion,
});
});
@ -588,7 +582,6 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { title: 'Title: Name: Callie' },
migrationVersion: { dog: '1.0.2' },
coreMigrationVersion: kibanaVersion,
});
});
@ -622,7 +615,6 @@ describe('DocumentMigrator', () => {
type: 'cat',
attributes: { name: 'Kitty Callie' },
migrationVersion: { dog: '2.2.4', cat: '1.0.0' },
coreMigrationVersion: kibanaVersion,
});
});
@ -694,7 +686,6 @@ describe('DocumentMigrator', () => {
type: 'cat',
attributes: { name: 'Boo' },
migrationVersion: { cat: '1.0.0', foo: '5.6.7' },
coreMigrationVersion: kibanaVersion,
});
});
@ -824,6 +815,35 @@ describe('DocumentMigrator', () => {
]);
});
it('does not fail when encountering documents with coreMigrationVersion higher than the latest known', () => {
const migrator = new DocumentMigrator({
...testOpts(),
typeRegistry: createRegistry(
{ name: 'dog', namespaceType: 'multiple', convertToMultiNamespaceTypeVersion: '1.0.0' }
// no migration transforms are defined, the migrationVersion will be derived from 'convertToMultiNamespaceTypeVersion'
),
});
migrator.prepareMigrations();
const obj = {
id: 'mischievous',
type: 'dog',
attributes: { name: 'Ann' },
migrationVersion: { dog: '0.1.0' },
coreMigrationVersion: '2.0.0',
} as SavedObjectUnsanitizedDoc;
const actual = migrator.migrateAndConvert(obj);
expect(actual).toEqual([
{
id: 'mischievous',
type: 'dog',
attributes: { name: 'Ann' },
migrationVersion: { dog: '1.0.0' },
coreMigrationVersion: '2.0.0',
namespaces: ['default'],
},
]);
});
it('skips reference transforms and conversion transforms when using `migrate`', () => {
const migrator = new DocumentMigrator({
...testOpts(),
@ -847,9 +867,43 @@ describe('DocumentMigrator', () => {
id: 'cowardly',
type: 'dog',
attributes: { name: 'Leslie' },
migrationVersion: { dog: '1.0.0' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }],
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespace: 'foo-namespace',
// there is no 'namespaces' field because no conversion transform was applied; this scenario is contrived for a clean test case but is not indicative of a real-world scenario
});
});
it('should keep the same `migrationVersion` when the conversion transforms are skipped', () => {
const migrator = new DocumentMigrator({
...testOpts(),
typeRegistry: createRegistry({
name: 'dog',
namespaceType: 'multiple',
convertToMultiNamespaceTypeVersion: '3.0.0',
migrations: {
'1.0.0': (doc: SavedObjectUnsanitizedDoc) => doc,
},
}),
});
migrator.prepareMigrations();
const obj = {
id: 'cowardly',
type: 'dog',
attributes: { name: 'Leslie' },
migrationVersion: { dog: '2.0.0' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }],
namespace: 'foo-namespace',
};
const actual = migrator.migrate(obj);
expect(mockGetConvertedObjectId).not.toHaveBeenCalled();
expect(actual).toEqual({
id: 'cowardly',
type: 'dog',
attributes: { name: 'Leslie' },
migrationVersion: { dog: '2.0.0' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }],
coreMigrationVersion: '3.0.0',
namespace: 'foo-namespace',
// there is no 'namespaces' field because no conversion transform was applied; this scenario is contrived for a clean test case but is not indicative of a real-world scenario
});
@ -881,7 +935,7 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Sweet Peach' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
},
]);
});
@ -896,7 +950,7 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Sweet Peach' },
references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespace: 'foo-namespace',
},
]);
@ -929,7 +983,7 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Wally' },
migrationVersion: { dog: '1.0.0' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['default'],
},
]);
@ -945,7 +999,7 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Wally' },
migrationVersion: { dog: '1.0.0' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['foo-namespace'],
originId: 'loud',
},
@ -960,7 +1014,7 @@ describe('DocumentMigrator', () => {
purpose: 'savedObjectConversion',
},
migrationVersion: { [LEGACY_URL_ALIAS_TYPE]: '0.1.2' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
},
]);
});
@ -993,7 +1047,7 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Too' },
migrationVersion: { dog: '1.0.0' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['default'],
},
]);
@ -1021,7 +1075,7 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Too' },
migrationVersion: { dog: '1.0.0' },
references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['foo-namespace'],
originId: 'cute',
},
@ -1036,7 +1090,7 @@ describe('DocumentMigrator', () => {
purpose: 'savedObjectConversion',
},
migrationVersion: { [LEGACY_URL_ALIAS_TYPE]: '0.1.2' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
},
]);
});
@ -1078,7 +1132,7 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Patches', age: '11', color: 'tri-color' },
migrationVersion: { dog: '2.0.0' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
},
]);
});
@ -1094,7 +1148,7 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Patches', age: '11', color: 'tri-color' },
migrationVersion: { dog: '2.0.0' },
references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespace: 'foo-namespace',
},
]);
@ -1131,7 +1185,7 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Remy' },
migrationVersion: { dog: '2.0.0' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['default'],
},
]);
@ -1147,7 +1201,7 @@ describe('DocumentMigrator', () => {
type: 'dog',
attributes: { name: 'Remy' },
migrationVersion: { dog: '2.0.0' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['foo-namespace'],
originId: 'hungry',
},
@ -1162,7 +1216,7 @@ describe('DocumentMigrator', () => {
purpose: 'savedObjectConversion',
},
migrationVersion: { [LEGACY_URL_ALIAS_TYPE]: '0.1.2' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
},
]);
});
@ -1203,7 +1257,7 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Sasha' },
migrationVersion: { dog: '2.0.0' },
references: [{ id: 'favorite', type: 'toy', name: 'BALL!' }], // no change
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['default'],
},
]);
@ -1231,7 +1285,7 @@ describe('DocumentMigrator', () => {
attributes: { name: 'Sasha' },
migrationVersion: { dog: '2.0.0' },
references: [{ id: 'uuidv5', type: 'toy', name: 'BALL!' }], // changed
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
namespaces: ['foo-namespace'],
originId: 'pretty',
},
@ -1246,7 +1300,7 @@ describe('DocumentMigrator', () => {
purpose: 'savedObjectConversion',
},
migrationVersion: { [LEGACY_URL_ALIAS_TYPE]: '0.1.2' },
coreMigrationVersion: kibanaVersion,
coreMigrationVersion: '1.0.0',
},
]);
});

177
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/document_migrator.ts
View file

@ -52,6 +52,7 @@ import type {
ISavedObjectTypeRegistry,
} from '@kbn/core-saved-objects-server';
import type { ActiveMigrations, TransformResult } from './types';
import { maxVersion } from './utils';
import { buildActiveMigrations } from './build_active_migrations';
import { validateMigrationDefinition } from './validate_migrations';
@ -119,8 +120,9 @@ export class DocumentMigrator implements VersionedTransformer {
throw new Error('Migrations are not ready. Make sure prepareMigrations is called first.');
}
return Object.entries(this.migrations).reduce((acc, [prop, { latestMigrationVersion }]) => {
// some migration objects won't have a latestMigrationVersion (they only contain reference transforms that are applied from other types)
return Object.entries(this.migrations).reduce((acc, [prop, { latestVersion }]) => {
// some migration objects won't have a latest migration version (they only contain reference transforms that are applied from other types)
const latestMigrationVersion = maxVersion(latestVersion.migrate, latestVersion.convert);
if (latestMigrationVersion) {
return { ...acc, [prop]: latestMigrationVersion };
}
@ -208,13 +210,13 @@ function buildDocumentTransform({
let transformedDoc: SavedObjectUnsanitizedDoc;
let additionalDocs: SavedObjectUnsanitizedDoc[] = [];
if (doc.migrationVersion) {
const result = applyMigrations(doc, migrations, kibanaVersion, convertNamespaceTypes);
const result = applyMigrations(doc, migrations, convertNamespaceTypes);
transformedDoc = result.transformedDoc;
additionalDocs = additionalDocs.concat(
result.additionalDocs.map((x) => markAsUpToDate(x, migrations, kibanaVersion))
result.additionalDocs.map((x) => markAsUpToDate(x, migrations))
);
} else {
transformedDoc = markAsUpToDate(doc, migrations, kibanaVersion);
transformedDoc = markAsUpToDate(doc, migrations);
}
// In order to keep tests a bit more stable, we won't
@ -244,7 +246,7 @@ function validateCoreMigrationVersion(doc: SavedObjectUnsanitizedDoc, kibanaVers
);
}
if (doc.coreMigrationVersion && Semver.gt(docVersion, kibanaVersion)) {
if (docVersion && Semver.gt(docVersion, kibanaVersion)) {
throw Boom.badData(
`Document "${id}" has a "coreMigrationVersion" which belongs to a more recent version` +
` of Kibana [${docVersion}]. The current version is [${kibanaVersion}].`,
@ -256,17 +258,22 @@ function validateCoreMigrationVersion(doc: SavedObjectUnsanitizedDoc, kibanaVers
function applyMigrations(
doc: SavedObjectUnsanitizedDoc,
migrations: ActiveMigrations,
kibanaVersion: string,
convertNamespaceTypes: boolean
) {
let additionalDocs: SavedObjectUnsanitizedDoc[] = [];
while (true) {
const prop = nextUnmigratedProp(doc, migrations);
const prop = nextUnmigratedProp(doc, migrations, convertNamespaceTypes);
if (!prop) {
// regardless of whether or not any reference transform was applied, update the coreMigrationVersion
// this is needed to ensure that newly created documents have an up-to-date coreMigrationVersion field
// Ensure that newly created documents have an up-to-date coreMigrationVersion field
const { coreMigrationVersion = getLatestCoreVersion(doc, migrations), ...transformedDoc } =
doc;
return {
transformedDoc: { ...doc, coreMigrationVersion: kibanaVersion },
transformedDoc: {
...transformedDoc,
...(coreMigrationVersion ? { coreMigrationVersion } : {}),
},
additionalDocs,
};
}
@ -286,28 +293,27 @@ function props(doc: SavedObjectUnsanitizedDoc) {
/**
* Looks up the prop version in a saved object document or in our latest migrations.
*/
function propVersion(doc: SavedObjectUnsanitizedDoc | ActiveMigrations, prop: string) {
return (
((doc as any)[prop] && (doc as any)[prop].latestMigrationVersion) ||
(doc.migrationVersion && (doc as any).migrationVersion[prop])
);
function propVersion(doc: SavedObjectUnsanitizedDoc, prop: string) {
return doc.migrationVersion && (doc as any).migrationVersion[prop];
}
/**
* Sets the doc's migrationVersion to be the most recent version
*/
function markAsUpToDate(
doc: SavedObjectUnsanitizedDoc,
migrations: ActiveMigrations,
kibanaVersion: string
) {
function markAsUpToDate(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMigrations) {
const { coreMigrationVersion = getLatestCoreVersion(doc, migrations), ...rest } = doc;
return {
...doc,
...rest,
migrationVersion: props(doc).reduce((acc, prop) => {
const version = propVersion(migrations, prop);
const version = maxVersion(
migrations[prop]?.latestVersion.migrate,
migrations[prop]?.latestVersion.convert
);
return version ? set(acc, prop, version) : acc;
}, {}),
coreMigrationVersion: kibanaVersion,
...(coreMigrationVersion ? { coreMigrationVersion } : {}),
};
}
@ -315,7 +321,7 @@ function markAsUpToDate(
* Determines whether or not a document has any pending transforms that should be applied based on its coreMigrationVersion field.
* Currently, only reference transforms qualify.
*/
function getHasPendingCoreMigrationVersionTransform(
function hasPendingCoreTransform(
doc: SavedObjectUnsanitizedDoc,
migrations: ActiveMigrations,
prop: string
@ -324,7 +330,7 @@ function getHasPendingCoreMigrationVersionTransform(
return false;
}
const { latestCoreMigrationVersion } = migrations[prop];
const latestCoreMigrationVersion = migrations[prop].latestVersion.reference;
const { coreMigrationVersion } = doc;
return (
latestCoreMigrationVersion &&
@ -332,12 +338,67 @@ function getHasPendingCoreMigrationVersionTransform(
);
}
/**
* Determines whether or not a document has any pending conversion transforms that should be applied.
* Currently, only reference transforms qualify.
*/
function hasPendingConversionTransform(
doc: SavedObjectUnsanitizedDoc,
migrations: ActiveMigrations,
prop: string
) {
if (!migrations.hasOwnProperty(prop)) {
return false;
}
const latestVersion = migrations[prop].latestVersion.convert;
const migrationVersion = doc.migrationVersion?.[prop];
return latestVersion && (!migrationVersion || Semver.gt(latestVersion, migrationVersion));
}
/**
* Determines whether or not a document has any pending transforms that should be applied based on its coreMigrationVersion field.
* Currently, only reference transforms qualify.
*/
function hasPendingMigrationTransform(
doc: SavedObjectUnsanitizedDoc,
migrations: ActiveMigrations,
prop: string
) {
if (!migrations.hasOwnProperty(prop)) {
return false;
}
const latestVersion = migrations[prop].latestVersion.migrate;
const migrationVersion = doc.migrationVersion?.[prop];
return latestVersion && (!migrationVersion || Semver.gt(latestVersion, migrationVersion));
}
function getLatestCoreVersion(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMigrations) {
let latestVersion: string | undefined;
for (const prop of props(doc)) {
latestVersion = maxVersion(latestVersion, migrations[prop]?.latestVersion.reference);
}
return latestVersion;
}
/**
* Finds the first unmigrated property in the specified document.
*/
function nextUnmigratedProp(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMigrations) {
function nextUnmigratedProp(
doc: SavedObjectUnsanitizedDoc,
migrations: ActiveMigrations,
convertNamespaceTypes: boolean
) {
return props(doc).find((p) => {
const latestMigrationVersion = propVersion(migrations, p);
const latestMigrationVersion = maxVersion(
migrations[p]?.latestVersion.migrate,
migrations[p]?.latestVersion.convert
);
const docVersion = propVersion(doc, p);
// We verify that the version is not greater than the version supported by Kibana.
@ -354,8 +415,10 @@ function nextUnmigratedProp(doc: SavedObjectUnsanitizedDoc, migrations: ActiveMi
}
return (
(latestMigrationVersion && latestMigrationVersion !== docVersion) ||
getHasPendingCoreMigrationVersionTransform(doc, migrations, p) // If the object itself is up-to-date, check if its references are up-to-date too
hasPendingMigrationTransform(doc, migrations, p) ||
(convertNamespaceTypes && // If the object itself is up-to-date, check if its references are up-to-date too
(hasPendingCoreTransform(doc, migrations, p) ||
hasPendingConversionTransform(doc, migrations, p)))
);
});
}
@ -373,13 +436,16 @@ function migrateProp(
let migrationVersion = _.clone(doc.migrationVersion) || {};
let additionalDocs: SavedObjectUnsanitizedDoc[] = [];
for (const { version, transform, transformType } of applicableTransforms(migrations, doc, prop)) {
if (convertNamespaceTypes || (transformType !== 'convert' && transformType !== 'reference')) {
// migrate transforms are always applied, but conversion transforms and reference transforms are only applied during index migrations
const result = transform(doc);
doc = result.transformedDoc;
additionalDocs = [...additionalDocs, ...result.additionalDocs];
}
for (const { version, transform, transformType } of applicableTransforms(
doc,
prop,
migrations,
convertNamespaceTypes
)) {
const result = transform(doc);
doc = result.transformedDoc;
additionalDocs = [...additionalDocs, ...result.additionalDocs];
if (transformType === 'reference') {
// regardless of whether or not the reference transform was applied, update the object's coreMigrationVersion
// this is needed to ensure that we don't have an endless migration loop
@ -402,20 +468,28 @@ function migrateProp(
* Retrieves any prop transforms that have not been applied to doc.
*/
function applicableTransforms(
migrations: ActiveMigrations,
doc: SavedObjectUnsanitizedDoc,
prop: string
prop: string,
migrations: ActiveMigrations,
convertNamespaceTypes: boolean
) {
const minVersion = propVersion(doc, prop);
const minReferenceVersion = doc.coreMigrationVersion || '0.0.0';
const minMigrationVersion = propVersion(doc, prop);
const minCoreMigrationVersion = doc.coreMigrationVersion || '0.0.0';
const { transforms } = migrations[prop];
return minVersion
? transforms.filter(({ version, transformType }) =>
transformType === 'reference'
? Semver.gt(version, minReferenceVersion)
: Semver.gt(version, minVersion)
)
: transforms;
return transforms
.filter(
({ transformType }) =>
convertNamespaceTypes || !['convert', 'reference'].includes(transformType)
)
.filter(
({ transformType, version }) =>
!minMigrationVersion ||
Semver.gt(
version,
transformType === 'reference' ? minCoreMigrationVersion : minMigrationVersion
)
);
}
/**
@ -429,9 +503,10 @@ function updateMigrationVersion(
version: string
) {
assertNoDowngrades(doc, migrationVersion, prop, version);
const docVersion = propVersion(doc, prop) || '0.0.0';
const maxVersion = Semver.gt(docVersion, version) ? docVersion : version;
return { ...(doc.migrationVersion || migrationVersion), [prop]: maxVersion };
return {
...(doc.migrationVersion || migrationVersion),
[prop]: maxVersion(propVersion(doc, prop), version) ?? '0.0.0',
};
}
/**

6
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/internal_transforms.ts
View file

@ -16,7 +16,7 @@ import {
LEGACY_URL_ALIAS_TYPE,
LegacyUrlAlias,
} from '@kbn/core-saved-objects-base-server-internal';
import type { Transform } from './types';
import { type Transform, TransformType } from './types';
/**
* Returns all applicable conversion transforms for a given object type.
@ -30,7 +30,7 @@ export function getConversionTransforms(type: SavedObjectsType): Transform[] {
{
version: convertToMultiNamespaceTypeVersion,
transform: convertNamespaceType,
transformType: 'convert',
transformType: TransformType.Convert,
},
];
}
@ -68,7 +68,7 @@ export function getReferenceTransforms(typeRegistry: ISavedObjectTypeRegistry):
}
return { transformedDoc: doc, additionalDocs: [] };
},
transformType: 'reference',
transformType: TransformType.Reference,
}));
}

12
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/types.ts
View file

@ -19,10 +19,8 @@ export interface ActiveMigrations {
* Structure containing all the required info to perform a type's conversion
*/
export interface TypeConversion {
/** Derived from `migrate` transforms and `convert` transforms */
latestMigrationVersion?: string;
/** Derived from `reference` transforms */
latestCoreMigrationVersion?: string;
/** Derived from the related transforms */
latestVersion: Record<TransformType, string>;
/** List of transforms registered for the type **/
transforms: Transform[];
}
@ -50,7 +48,11 @@ export interface Transform {
* * `reference` - These transforms are defined by core and added by consumers using the type registry; they are applied to all object
* types based on their `coreMigrationVersion` field. These are applied during index migrations, NOT document migrations.
*/
export type TransformType = 'migrate' | 'convert' | 'reference';
export enum TransformType {
Migrate = 'migrate',
Convert = 'convert',
Reference = 'reference',
}
/**
* Transformation function for a {@link Transform}

21
packages/core/saved-objects/core-saved-objects-migration-server-internal/src/document_migrator/utils.ts
View file

@ -15,7 +15,7 @@ import {
import { Logger } from '@kbn/logging';
import { MigrationLogger } from '../core/migration_logger';
import { TransformSavedObjectDocumentError } from '../core/transform_saved_object_document_error';
import type { Transform, TransformFn } from './types';
import { type Transform, type TransformFn, TransformType } from './types';
/**
* If a specific transform function fails, this tacks on a bit of information
@ -64,15 +64,26 @@ export function transformComparator(a: Transform, b: Transform) {
if (semver !== 0) {
return semver;
} else if (a.transformType !== b.transformType) {
if (a.transformType === 'migrate') {
if (a.transformType === TransformType.Migrate) {
return 1;
} else if (b.transformType === 'migrate') {
} else if (b.transformType === TransformType.Migrate) {
return -1;
} else if (a.transformType === 'convert') {
} else if (a.transformType === TransformType.Convert) {
return 1;
} else if (b.transformType === 'convert') {
} else if (b.transformType === TransformType.Convert) {
return -1;
}
}
return 0;
}
export function maxVersion(a?: string, b?: string) {
if (!a) {
return b;
}
if (!b) {
return a;
}
return Semver.gt(a, b) ? a : b;
}

2
src/core/server/integration_tests/saved_objects/migrations/group2/outdated_docs.test.ts
View file

@ -95,7 +95,7 @@ describe('migration v2', () => {
expect(migratedDocs.length).toBe(1);
const [doc] = migratedDocs;
expect(doc._source.migrationVersion.foo).toBe('7.14.0');
expect(doc._source.coreMigrationVersion).toBe(pkg.version);
expect(doc._source.coreMigrationVersion).toBe('8.0.0');
});
});

12
src/core/server/integration_tests/saved_objects/migrations/group3/rewriting_id.test.ts
View file

@ -188,7 +188,7 @@ describe('migration v2', () => {
references: [],
namespaces: ['default'],
migrationVersion: { foo: '8.0.0' },
coreMigrationVersion: pkg.version,
coreMigrationVersion: '8.0.0',
},
{
id: `foo:${newFooId}`,
@ -198,7 +198,7 @@ describe('migration v2', () => {
namespaces: ['spacex'],
originId: '1',
migrationVersion: { foo: '8.0.0' },
coreMigrationVersion: pkg.version,
coreMigrationVersion: '8.0.0',
},
{
// new object for spacex:foo:1
@ -213,7 +213,7 @@ describe('migration v2', () => {
},
migrationVersion: { 'legacy-url-alias': '8.2.0' },
references: [],
coreMigrationVersion: pkg.version,
coreMigrationVersion: '8.0.0',
},
{
id: 'bar:1',
@ -222,7 +222,7 @@ describe('migration v2', () => {
references: [{ type: 'foo', id: '1', name: 'Foo 1 default' }],
namespaces: ['default'],
migrationVersion: { bar: '8.0.0' },
coreMigrationVersion: pkg.version,
coreMigrationVersion: '8.0.0',
},
{
id: `bar:${newBarId}`,
@ -232,7 +232,7 @@ describe('migration v2', () => {
namespaces: ['spacex'],
originId: '1',
migrationVersion: { bar: '8.0.0' },
coreMigrationVersion: pkg.version,
coreMigrationVersion: '8.0.0',
},
{
// new object for spacex:bar:1
@ -247,7 +247,7 @@ describe('migration v2', () => {
},
migrationVersion: { 'legacy-url-alias': '8.2.0' },
references: [],
coreMigrationVersion: pkg.version,
coreMigrationVersion: '8.0.0',
},
].sort(sortByTypeAndId)
);

6
test/api_integration/apis/saved_objects/bulk_create.ts
View file

@ -8,7 +8,6 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { getKibanaVersion } from './lib/saved_objects_test_utils';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
@ -34,10 +33,7 @@ export default function ({ getService }: FtrProviderContext) {
];
describe('_bulk_create', () => {
let KIBANA_VERSION: string;
before(async () => {
KIBANA_VERSION = await getKibanaVersion(getService);
await kibanaServer.spaces.create({ id: SPACE_ID, name: SPACE_ID });
await kibanaServer.importExport.load(
'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json',
@ -77,7 +73,7 @@ export default function ({ getService }: FtrProviderContext) {
migrationVersion: {
dashboard: resp.body.saved_objects[1].migrationVersion.dashboard,
},
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '8.0.0',
references: [],
namespaces: [SPACE_ID],
},

8
test/api_integration/apis/saved_objects/bulk_get.ts
View file

@ -8,7 +8,6 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { getKibanaVersion } from './lib/saved_objects_test_utils';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
@ -30,10 +29,7 @@ export default function ({ getService }: FtrProviderContext) {
];
describe('_bulk_get', () => {
let KIBANA_VERSION: string;
before(async () => {
KIBANA_VERSION = await getKibanaVersion(getService);
await kibanaServer.importExport.load(
'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json'
);
@ -76,7 +72,7 @@ export default function ({ getService }: FtrProviderContext) {
resp.body.saved_objects[0].attributes.kibanaSavedObjectMeta,
},
migrationVersion: resp.body.saved_objects[0].migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
namespaces: ['default'],
references: [
{
@ -107,7 +103,7 @@ export default function ({ getService }: FtrProviderContext) {
},
namespaces: ['default'],
migrationVersion: resp.body.saved_objects[2].migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
references: [],
},
],

10
test/api_integration/apis/saved_objects/create.ts
View file

@ -8,17 +8,13 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { getKibanaVersion } from './lib/saved_objects_test_utils';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
describe('create', () => {
let KIBANA_VERSION: string;
before(async () => {
KIBANA_VERSION = await getKibanaVersion(getService);
await kibanaServer.importExport.load(
'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json'
);
@ -54,7 +50,7 @@ export default function ({ getService }: FtrProviderContext) {
id: resp.body.id,
type: 'visualization',
migrationVersion: resp.body.migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '8.0.0',
updated_at: resp.body.updated_at,
created_at: resp.body.created_at,
version: resp.body.version,
@ -68,7 +64,7 @@ export default function ({ getService }: FtrProviderContext) {
});
});
it('result should be updated to the latest coreMigrationVersion', async () => {
it('result should not be updated to the latest Kibana version if there are no migrations', async () => {
await supertest
.post(`/api/saved_objects/visualization`)
.send({
@ -79,7 +75,7 @@ export default function ({ getService }: FtrProviderContext) {
})
.expect(200)
.then((resp) => {
expect(resp.body.coreMigrationVersion).to.eql(KIBANA_VERSION);
expect(resp.body.coreMigrationVersion).to.eql('1.2.3');
});
});
});

10
test/api_integration/apis/saved_objects/export.ts
View file

@ -8,7 +8,6 @@
import expect from '@kbn/expect';
import type { FtrProviderContext } from '../../ftr_provider_context';
import { getKibanaVersion } from './lib/saved_objects_test_utils';
function ndjsonToObject(input: string) {
return input.split('\n').map((str) => JSON.parse(str));
@ -19,10 +18,7 @@ export default function ({ getService }: FtrProviderContext) {
const SPACE_ID = 'ftr-so-export';
describe('export', () => {
let KIBANA_VERSION: string;
before(async () => {
KIBANA_VERSION = await getKibanaVersion(getService);
await kibanaServer.spaces.create({ id: SPACE_ID, name: SPACE_ID });
await kibanaServer.importExport.load(
'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json',
@ -353,7 +349,7 @@ export default function ({ getService }: FtrProviderContext) {
},
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
migrationVersion: objects[0].migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
references: [
{
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
@ -414,7 +410,7 @@ export default function ({ getService }: FtrProviderContext) {
},
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
migrationVersion: objects[0].migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
references: [
{
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',
@ -480,7 +476,7 @@ export default function ({ getService }: FtrProviderContext) {
},
id: 'be3733a0-9efe-11e7-acb3-3dab96693fab',
migrationVersion: objects[0].migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
references: [
{
id: 'dd7caf20-9efd-11e7-acb3-3dab96693fab',

6
test/api_integration/apis/saved_objects/get.ts
View file

@ -8,17 +8,13 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { getKibanaVersion } from './lib/saved_objects_test_utils';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
describe('get', () => {
let KIBANA_VERSION: string;
before(async () => {
KIBANA_VERSION = await getKibanaVersion(getService);
await kibanaServer.importExport.load(
'test/api_integration/fixtures/kbn_archiver/saved_objects/basic.json'
);
@ -41,7 +37,7 @@ export default function ({ getService }: FtrProviderContext) {
created_at: resp.body.created_at,
version: resp.body.version,
migrationVersion: resp.body.migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
attributes: {
title: 'Count of requests',
description: '',

9
test/api_integration/apis/saved_objects/resolve.ts
View file

@ -8,19 +8,12 @@
import expect from '@kbn/expect';
import type { FtrProviderContext } from '../../ftr_provider_context';
import { getKibanaVersion } from './lib/saved_objects_test_utils';
export default function ({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const kibanaServer = getService('kibanaServer');
describe('resolve', () => {
let KIBANA_VERSION: string;
before(async () => {
KIBANA_VERSION = await getKibanaVersion(getService);
});
describe('with kibana index', () => {
before(async () => {
await kibanaServer.importExport.load(
@ -49,7 +42,7 @@ export default function ({ getService }: FtrProviderContext) {
created_at: '2015-01-01T00:00:00.000Z',
version: resp.body.saved_object.version,
migrationVersion: resp.body.saved_object.migrationVersion,
coreMigrationVersion: KIBANA_VERSION,
coreMigrationVersion: '7.14.0',
attributes: {
title: 'Count of requests',
description: '',