mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 17:59:23 -04:00
Use modelVersions instead of hashes to track changes in SO types (#176803)
## Summary Address #169734 . We're currently storing information about _Saved Object_ types in the `<index>.mapping._meta`. More specifically, we're storing hashes of the mappings of each type, under the `migrationMappingPropertyHashes` property. This allows us to detect which types' mappings have changed, in order to _update and pick up_ changes only for those types. **The problem is that `md5` cannot be used if we want to be FIPS compliant.** Thus, the idea is to stop using hashes to track changes in the SO mappings. Instead, we're going to use the same `modelVersions` introduced by ZDT: Whenever mappings change for a given SO type, the corresponding `modelVersion` will change too, so `modelVersions` can be used to determine if mappings have changed. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Gerard Soldevila
GitHub
parent
d62490699b
commit
2f5396ca10
56 changed files with 2039 additions and 825 deletions
|
|
@ -181,7 +181,7 @@ export interface SavedObjectsClientContract {
|
|||
* @param {object} attributes - the attributes to update
|
||||
* @param {object} options {@link SavedObjectsUpdateOptions}
|
||||
* @prop {integer} options.version - ensures version matches that of persisted object
|
||||
* @returns the udpated simple saved object
|
||||
* @returns the updated simple saved object
|
||||
* @deprecated See https://github.com/elastic/kibana/issues/149098
|
||||
*/
|
||||
update<T = unknown>(
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
export { DEFAULT_INDEX_TYPES_MAP } from './src/constants';
|
||||
export { DEFAULT_INDEX_TYPES_MAP, HASH_TO_VERSION_MAP } from './src/constants';
|
||||
export { LEGACY_URL_ALIAS_TYPE, type LegacyUrlAlias } from './src/legacy_alias';
|
||||
export {
|
||||
getProperty,
|
||||
|
|
@ -65,6 +65,7 @@ export {
|
|||
getCurrentVirtualVersion,
|
||||
getLatestMigrationVersion,
|
||||
getVirtualVersionMap,
|
||||
getLatestMappingsVirtualVersionMap,
|
||||
type ModelVersionMap,
|
||||
type VirtualVersionMap,
|
||||
compareVirtualVersions,
|
||||
|
|
|
|||
|
|
@ -114,3 +114,126 @@ export const DEFAULT_INDEX_TYPES_MAP: IndexTypesMap = {
|
|||
'workplace_search_telemetry',
|
||||
],
|
||||
};
|
||||
|
||||
/**
|
||||
* In order to be FIPS compliant, the migration logic has switched
|
||||
* from using hashes (stored in _meta.migrationMappingPropertyHashes)
|
||||
* to using model versions (stored in _meta.mappingVersions).
|
||||
*
|
||||
* This map holds a breakdown of md5 hashes to model versions.
|
||||
* This allows keeping track of changes in mappings for the different SO types:
|
||||
* When upgrading from a Kibana version prior to the introduction of model versions for V2,
|
||||
* the V2 logic will map stored hashes to their corresponding model versions.
|
||||
* These model versions will then be compared against the ones defined in the typeRegistry,
|
||||
* in order to determine which types' mappings have changed.
|
||||
*/
|
||||
export const HASH_TO_VERSION_MAP = {
|
||||
'action_task_params|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'action|0be88ebcc8560a075b6898236a202eb1': '10.0.0',
|
||||
'alert|96a5a144778243a9f4fece0e71c2197f': '10.0.0',
|
||||
'api_key_pending_invalidation|16f515278a295f6245149ad7c5ddedb7': '10.0.0',
|
||||
'apm-custom-dashboards|561810b957ac3c09fcfc08f32f168e97': '10.0.0',
|
||||
'apm-indices|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'apm-server-schema|b1d71908f324c17bf744ac72af5038fb': '10.0.0',
|
||||
'apm-service-group|2af509c6506f29a858e5a0950577d9fa': '10.0.0',
|
||||
'apm-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'app_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'application_usage_daily|43b8830d5d0df85a6823d290885fc9fd': '10.0.0',
|
||||
'application_usage_totals|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'canvas-element|7390014e1091044523666d97247392fc': '10.0.0',
|
||||
'canvas-workpad-template|ae2673f678281e2c055d764b153e9715': '10.0.0',
|
||||
'canvas-workpad|b0a1706d356228dbdcb4a17e6b9eb231': '10.0.0',
|
||||
'cases-comments|93535d41ca0279a4a2e5d08acd3f28e3': '10.0.0',
|
||||
'cases-configure|c124bd0be4c139d0f0f91fb9eeca8e37': '10.0.0',
|
||||
'cases-connector-mappings|a98c33813f364f0b068e8c592ac6ef6d': '10.0.0',
|
||||
'cases-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'cases-user-actions|07a6651cf37853dd5d64bfb2c796e102': '10.0.0',
|
||||
'cases|8f7dc53b17c272ea19f831537daa082d': '10.1.0',
|
||||
'cloud-security-posture-settings|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'config-global|c63748b75f39d0c54de12d12c1ccbc20': '10.0.0',
|
||||
'config|c63748b75f39d0c54de12d12c1ccbc20': '10.0.0',
|
||||
'connector_token|740b3fd18387d4097dca8d177e6a35c6': '10.0.0',
|
||||
'core-usage-stats|3d1b76c39bfb2cc8296b024d73854724': '7.14.1',
|
||||
'csp-rule-template|6ee70dc06c0ca3ddffc18222f202ab25': '10.0.0',
|
||||
'dashboard|b8aa800aa5e0d975c5e8dc57f03d41f8': '10.2.0',
|
||||
'endpoint:user-artifact-manifest|7502b5c5bc923abe8aa5ccfd636e8c3d': '10.0.0',
|
||||
'enterprise_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'epm-packages-assets|44621b2f6052ef966da47b7c3a00f33b': '10.0.0',
|
||||
'epm-packages|c1e2020399dbebba2448096ca007c668': '10.1.0',
|
||||
'event_loop_delays_daily|5df7e292ddd5028e07c1482e130e6654': '10.0.0',
|
||||
'event-annotation-group|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0',
|
||||
'exception-list-agnostic|8a1defe5981db16792cb9a772e84bb9a': '10.0.0',
|
||||
'exception-list|8a1defe5981db16792cb9a772e84bb9a': '10.0.0',
|
||||
'file-upload-usage-collection-telemetry|a34fbb8e3263d105044869264860c697': '10.0.0',
|
||||
'file|8e9dd7f8a22efdb8fb1c15ed38fde9f6': '10.0.0',
|
||||
'fileShare|aa8f7ac2ddf8ab1a91bd34e347046caa': '10.0.0',
|
||||
'fleet-fleet-server-host|c28ce72481d1696a9aac8b2cdebcecfa': '10.1.0',
|
||||
'fleet-message-signing-keys|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'fleet-preconfiguration-deletion-record|4c36f199189a367e43541f236141204c': '10.0.0',
|
||||
'fleet-proxy|05b7a22977de25ce67a77e44dd8e6c33': '10.0.0',
|
||||
'fleet-uninstall-tokens|cdb2b655f6b468ecb57d132972425f2e': '10.0.0',
|
||||
'graph-workspace|27a94b2edcb0610c6aea54a7c56d7752': '10.0.0',
|
||||
'guided-onboarding-guide-state|a3db59c45a3fd2730816d4f53c35c7d9': '10.0.0',
|
||||
'guided-onboarding-plugin-state|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'index-pattern|83c02d842fe2a94d14dfa13f7dcd6e87': '10.0.0',
|
||||
'infra-custom-dashboards|6eed22cbe14594bad8c076fa864930de': '10.0.0',
|
||||
'infrastructure-monitoring-log-view|c50526fc6040c5355ed027d34d05b35c': '10.0.0',
|
||||
'infrastructure-ui-source|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'ingest_manager_settings|b91ffb075799c78ffd7dbd51a279c8c9': '10.1.0',
|
||||
'ingest-agent-policies|20768dc7ce5eced3eb309e50d8a6cf76': '10.0.0',
|
||||
'ingest-download-sources|0b0f6828e59805bd07a650d80817c342': '10.0.0',
|
||||
'ingest-outputs|b1237f7fdc0967709e75d65d208ace05': '10.6.0',
|
||||
'ingest-package-policies|a1a074bad36e68d54f98d2158d60f879': '10.0.0',
|
||||
'inventory-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'kql-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'legacy-url-alias|0750774cf16475f88f2361e99cc5c8f0': '8.2.0',
|
||||
'lens-ui-telemetry|509bfa5978586998e05f9e303c07a327': '10.0.0',
|
||||
'lens|b0da10d5ab9ebd81d61700737ddc76c9': '10.0.0',
|
||||
'links|3378bb9b651572865d9f61f5b448e415': '10.0.0',
|
||||
'maintenance-window|a58ac2ef53ff5103710093e669dcc1d8': '10.0.0',
|
||||
'map|9134b47593116d7953f6adba096fc463': '10.0.0',
|
||||
'metrics-data-source|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'metrics-explorer-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'ml-job|3bb64c31915acf93fc724af137a0891b': '10.0.0',
|
||||
'ml-module|f6c6b7b7ebdca4154246923f24d6340d': '10.0.0',
|
||||
'ml-trained-model|d2f03c1a5dd038fa58af14a56944312b': '10.0.0',
|
||||
'monitoring-telemetry|2669d5ec15e82391cf58df4294ee9c68': '10.0.0',
|
||||
'observability-onboarding-state|a4e5c9d018037114140bdb1647c2d568': '10.0.0',
|
||||
'osquery-manager-usage-metric|4dc4f647d27247c002f56f22742175fe': '10.0.0',
|
||||
'osquery-pack-asset|fe0dfa13c4c24ac37ce1aec04c560a81': '10.1.0',
|
||||
'osquery-pack|6bc20973adab06f00156cbc4578a19ac': '10.1.0',
|
||||
'osquery-saved-query|a05ec7031231a4b71bfb4493a07b2dc5': '10.1.0',
|
||||
'policy-settings-protection-updates-note|37d4035a1dc3c5e58f1b519f99093f21': '10.0.0',
|
||||
'query|aa811b49f48906074f59110bfa83984c': '10.2.0',
|
||||
'risk-engine-configuration|431232781a82926aad5b1fd849715c0f': '10.1.0',
|
||||
'rules-settings|001f60645e96c71520214b57f3ea7590': '10.0.0',
|
||||
'sample-data-telemetry|7d3cfeb915303c9641c59681967ffeb4': '10.0.0',
|
||||
'search-session|fea3612a90b81672991617646f229a61': '10.0.0',
|
||||
'search-telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'search|df07b1a361c32daf4e6842c1d5521dbe': '10.0.0',
|
||||
'security-rule|9d9d11b97e3aaa87fbaefbace2b5c25f': '10.0.0',
|
||||
'security-solution-signals-migration|4060b5a63dddfd54d2cd56450882cc0e': '10.0.0',
|
||||
'siem-detection-engine-rule-actions|f5c218f837bac10ab2c3980555176cf9': '10.0.0',
|
||||
'siem-ui-timeline-note|28393dfdeb4e4413393eb5f7ec8c5436': '10.0.0',
|
||||
'siem-ui-timeline-pinned-event|293fce142548281599060e07ad2c9ddb': '10.0.0',
|
||||
'siem-ui-timeline|f6739fd4b17646a6c86321a746c247ef': '10.1.0',
|
||||
'slo|dc7f35c0cf07d71bb36f154996fe10c6': '10.1.0',
|
||||
'space|c3aec2a5d4afcb75554fed96411170e1': '10.0.0',
|
||||
'spaces-usage-stats|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'synthetics-monitor|50b48ccda9f2f7d73d31fd50c41bf305': '10.0.0',
|
||||
'synthetics-param|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'synthetics-privates-locations|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'tag|83d55da58f6530f7055415717ec06474': '10.0.0',
|
||||
'task|b4a368fd68cd32ef6990877634639db6': '10.0.0',
|
||||
'telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'threshold-explorer-view|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'ui-metric|0d409297dc5ebe1e3a1da691c6ee32e3': '10.0.0',
|
||||
'upgrade-assistant-ml-upgrade-operation|3caf305ad2da94d80d49453b0970156d': '10.0.0',
|
||||
'upgrade-assistant-reindex-operation|6d1e2aca91767634e1829c30f20f6b16': '10.0.0',
|
||||
'uptime-dynamic-settings|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
'uptime-synthetics-api-key|c3178f0fde61e18d3530ba9a70bc278a': '10.0.0',
|
||||
'url|a37dbae7645ad5811045f4dd3dc1c0a8': '10.0.0',
|
||||
'usage-counters|8cc260bdceffec4ffc3ad165c97dc1b4': '10.0.0',
|
||||
'visualization|4891c012863513388881fc109fec4809': '10.0.0',
|
||||
'workplace_search_telemetry|3d1b76c39bfb2cc8296b024d73854724': '10.0.0',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ export interface V2AlgoIndexMappingMeta {
|
|||
* the md5 hash of that mapping's value when the index was created.
|
||||
*
|
||||
* @remark: Only defined for indices using the v2 migration algorithm.
|
||||
* @deprecated Replaced by mappingVersions (FIPS-compliant initiative)
|
||||
*/
|
||||
migrationMappingPropertyHashes?: { [k: string]: string };
|
||||
/**
|
||||
|
|
@ -74,20 +75,20 @@ export interface V2AlgoIndexMappingMeta {
|
|||
* @remark: Only defined for indices using the v2 migration algorithm.
|
||||
*/
|
||||
indexTypesMap?: IndexTypesMap;
|
||||
/**
|
||||
* The current virtual version of the mapping of the index.
|
||||
*/
|
||||
mappingVersions?: { [k: string]: string };
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface ZdtAlgoIndexMappingMeta {
|
||||
/**
|
||||
* The current virtual version of the mapping of the index.
|
||||
*
|
||||
* @remark: Only defined for indices using the zdt migration algorithm.
|
||||
*/
|
||||
mappingVersions?: { [k: string]: string };
|
||||
/**
|
||||
* The current virtual versions of the documents of the index.
|
||||
*
|
||||
* @remark: Only defined for indices using the zdt migration algorithm.
|
||||
*/
|
||||
docVersions?: { [k: string]: string };
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export {
|
|||
getCurrentVirtualVersion,
|
||||
getVirtualVersionMap,
|
||||
getLatestMigrationVersion,
|
||||
getLatestMappingsVirtualVersionMap,
|
||||
type ModelVersionMap,
|
||||
type VirtualVersionMap,
|
||||
} from './version_map';
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ import {
|
|||
getLatestMigrationVersion,
|
||||
getCurrentVirtualVersion,
|
||||
getVirtualVersionMap,
|
||||
getLatestMappingsVersionNumber,
|
||||
getLatestMappingsModelVersion,
|
||||
getLatestMappingsVirtualVersionMap,
|
||||
} from './version_map';
|
||||
|
||||
describe('ModelVersion map utilities', () => {
|
||||
|
|
@ -28,6 +31,24 @@ describe('ModelVersion map utilities', () => {
|
|||
changes: [],
|
||||
});
|
||||
|
||||
const dummyModelVersionWithMappingsChanges = (): SavedObjectsModelVersion => ({
|
||||
changes: [
|
||||
{
|
||||
type: 'mappings_addition',
|
||||
addedMappings: {},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const dummyModelVersionWithDataRemoval = (): SavedObjectsModelVersion => ({
|
||||
changes: [
|
||||
{
|
||||
type: 'data_removal',
|
||||
removedAttributePaths: ['some.attribute'],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const dummyMigration = jest.fn();
|
||||
|
||||
describe('getLatestModelVersion', () => {
|
||||
|
|
@ -259,4 +280,146 @@ describe('ModelVersion map utilities', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getLatestMappingsVersionNumber', () => {
|
||||
it('returns 0 when no model versions are registered', () => {
|
||||
expect(getLatestMappingsVersionNumber(buildType({ modelVersions: {} }))).toEqual(0);
|
||||
expect(getLatestMappingsVersionNumber(buildType({ modelVersions: undefined }))).toEqual(0);
|
||||
});
|
||||
|
||||
it('throws if an invalid version is provided', () => {
|
||||
expect(() =>
|
||||
getLatestMappingsVersionNumber(
|
||||
buildType({
|
||||
modelVersions: {
|
||||
foo: dummyModelVersionWithMappingsChanges(),
|
||||
},
|
||||
})
|
||||
)
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it('returns the latest version that brings mappings changes', () => {
|
||||
expect(
|
||||
getLatestMappingsVersionNumber(
|
||||
buildType({
|
||||
modelVersions: {
|
||||
'1': dummyModelVersion(),
|
||||
'2': dummyModelVersionWithMappingsChanges(),
|
||||
'3': dummyModelVersionWithDataRemoval(),
|
||||
},
|
||||
})
|
||||
)
|
||||
).toEqual(2);
|
||||
});
|
||||
|
||||
it('accepts provider functions', () => {
|
||||
expect(
|
||||
getLatestMappingsVersionNumber(
|
||||
buildType({
|
||||
modelVersions: () => ({
|
||||
'1': dummyModelVersion(),
|
||||
'2': dummyModelVersionWithMappingsChanges(),
|
||||
'3': dummyModelVersionWithDataRemoval(),
|
||||
}),
|
||||
})
|
||||
)
|
||||
).toEqual(2);
|
||||
});
|
||||
|
||||
it('supports unordered maps', () => {
|
||||
expect(
|
||||
getLatestMappingsVersionNumber(
|
||||
buildType({
|
||||
modelVersions: {
|
||||
'3': dummyModelVersionWithDataRemoval(),
|
||||
'1': dummyModelVersion(),
|
||||
'2': dummyModelVersionWithMappingsChanges(),
|
||||
},
|
||||
})
|
||||
)
|
||||
).toEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getLatestMappingsModelVersion', () => {
|
||||
it('returns the latest registered migration if switchToModelVersionAt is unset', () => {
|
||||
expect(
|
||||
getLatestMappingsModelVersion(
|
||||
buildType({
|
||||
migrations: {
|
||||
'7.17.2': dummyMigration,
|
||||
'8.6.0': dummyMigration,
|
||||
},
|
||||
modelVersions: {
|
||||
1: dummyModelVersionWithMappingsChanges(),
|
||||
2: dummyModelVersion(),
|
||||
},
|
||||
})
|
||||
)
|
||||
).toEqual('8.6.0');
|
||||
});
|
||||
|
||||
it('returns the virtual version of the latest model version if switchToModelVersionAt is set', () => {
|
||||
expect(
|
||||
getLatestMappingsModelVersion(
|
||||
buildType({
|
||||
switchToModelVersionAt: '8.7.0',
|
||||
migrations: {
|
||||
'7.17.2': dummyMigration,
|
||||
'8.6.0': dummyMigration,
|
||||
},
|
||||
modelVersions: {
|
||||
1: dummyModelVersionWithMappingsChanges(),
|
||||
2: dummyModelVersion(),
|
||||
},
|
||||
})
|
||||
)
|
||||
).toEqual('10.1.0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getLatestMappingsVirtualVersionMap', () => {
|
||||
it('returns the virtual version for each of the provided types', () => {
|
||||
expect(
|
||||
getLatestMappingsVirtualVersionMap([
|
||||
buildType({
|
||||
name: 'foo',
|
||||
switchToModelVersionAt: '8.7.0',
|
||||
migrations: {
|
||||
'7.17.2': dummyMigration,
|
||||
'8.6.0': dummyMigration,
|
||||
},
|
||||
modelVersions: {
|
||||
1: dummyModelVersionWithMappingsChanges(),
|
||||
2: dummyModelVersion(),
|
||||
},
|
||||
}),
|
||||
buildType({
|
||||
name: 'bar',
|
||||
migrations: {
|
||||
'7.17.2': dummyMigration,
|
||||
'8.6.0': dummyMigration,
|
||||
},
|
||||
modelVersions: {
|
||||
1: dummyModelVersionWithMappingsChanges(),
|
||||
2: dummyModelVersion(),
|
||||
},
|
||||
}),
|
||||
buildType({
|
||||
name: 'dolly',
|
||||
switchToModelVersionAt: '8.7.0',
|
||||
migrations: {
|
||||
'7.17.2': dummyMigration,
|
||||
'8.6.0': dummyMigration,
|
||||
},
|
||||
}),
|
||||
])
|
||||
).toEqual({
|
||||
foo: '10.1.0',
|
||||
bar: '8.6.0',
|
||||
dolly: '10.0.0',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -86,3 +86,47 @@ export const getVirtualVersionMap = (types: SavedObjectsType[]): VirtualVersionM
|
|||
return versionMap;
|
||||
}, {});
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the latest version number that includes changes in the mappings, for the given type.
|
||||
* If none of the versions are updating the mappings, it will return 0
|
||||
*/
|
||||
export const getLatestMappingsVersionNumber = (type: SavedObjectsType): number => {
|
||||
const versionMap =
|
||||
typeof type.modelVersions === 'function' ? type.modelVersions() : type.modelVersions ?? {};
|
||||
return Object.entries(versionMap)
|
||||
.filter(([version, info]) =>
|
||||
info.changes?.some((change) => change.type === 'mappings_addition')
|
||||
)
|
||||
.reduce<number>((memo, [current]) => {
|
||||
return Math.max(memo, assertValidModelVersion(current));
|
||||
}, 0);
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the latest model version that includes changes in the mappings, for the given type.
|
||||
* It will either be a model version if the type
|
||||
* already switched to using them (switchToModelVersionAt is set),
|
||||
* or the latest migration version for the type otherwise.
|
||||
*/
|
||||
export const getLatestMappingsModelVersion = (type: SavedObjectsType): string => {
|
||||
if (type.switchToModelVersionAt) {
|
||||
const modelVersion = getLatestMappingsVersionNumber(type);
|
||||
return modelVersionToVirtualVersion(modelVersion);
|
||||
} else {
|
||||
return getLatestMigrationVersion(type);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns a map of virtual model version for the given types.
|
||||
* See {@link getLatestMappingsModelVersion}
|
||||
*/
|
||||
export const getLatestMappingsVirtualVersionMap = (
|
||||
types: SavedObjectsType[]
|
||||
): VirtualVersionMap => {
|
||||
return types.reduce<VirtualVersionMap>((versionMap, type) => {
|
||||
versionMap[type.name] = getLatestMappingsModelVersion(type);
|
||||
return versionMap;
|
||||
}, {});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,22 +2,6 @@
|
|||
|
||||
exports[`KibanaMigrator getActiveMappings returns full index mappings w/ core properties 1`] = `
|
||||
Object {
|
||||
"_meta": Object {
|
||||
"migrationMappingPropertyHashes": Object {
|
||||
"amap": "510f1f0adb69830cf8a1c5ce2923ed82",
|
||||
"bmap": "510f1f0adb69830cf8a1c5ce2923ed82",
|
||||
"coreMigrationVersion": "2f4316de49999235636386fe51dc06c1",
|
||||
"created_at": "00da57df13e94e9d98437d13ace4bfe0",
|
||||
"managed": "88cf246b441a6362458cb6a56ca3f7d7",
|
||||
"namespace": "2f4316de49999235636386fe51dc06c1",
|
||||
"namespaces": "2f4316de49999235636386fe51dc06c1",
|
||||
"originId": "2f4316de49999235636386fe51dc06c1",
|
||||
"references": "7997cf5a56cc02bdc9c93361bde732b0",
|
||||
"type": "2f4316de49999235636386fe51dc06c1",
|
||||
"typeMigrationVersion": "539e3ecebb3abc1133618094cc3b7ae7",
|
||||
"updated_at": "00da57df13e94e9d98437d13ace4bfe0",
|
||||
},
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": Object {
|
||||
"amap": Object {
|
||||
|
|
|
|||
|
|
@ -141,7 +141,20 @@ Object {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"task|someHash": "10.1.0",
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
"typeD|someHash": "10.1.0",
|
||||
"typeE|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -158,6 +171,7 @@ Object {
|
|||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"latestMappingsVersions": Object {},
|
||||
"legacyIndex": ".my-so-index",
|
||||
"logs": Array [
|
||||
Object {
|
||||
|
|
@ -188,22 +202,6 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -365,7 +363,20 @@ Object {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"task|someHash": "10.1.0",
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
"typeD|someHash": "10.1.0",
|
||||
"typeE|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -382,6 +393,7 @@ Object {
|
|||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"latestMappingsVersions": Object {},
|
||||
"legacyIndex": ".my-so-index",
|
||||
"logs": Array [
|
||||
Object {
|
||||
|
|
@ -416,22 +428,6 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -593,7 +589,20 @@ Object {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"task|someHash": "10.1.0",
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
"typeD|someHash": "10.1.0",
|
||||
"typeE|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -610,6 +619,7 @@ Object {
|
|||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"latestMappingsVersions": Object {},
|
||||
"legacyIndex": ".my-so-index",
|
||||
"logs": Array [
|
||||
Object {
|
||||
|
|
@ -648,22 +658,6 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -825,7 +819,20 @@ Object {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"task|someHash": "10.1.0",
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
"typeD|someHash": "10.1.0",
|
||||
"typeE|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -842,6 +849,7 @@ Object {
|
|||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"latestMappingsVersions": Object {},
|
||||
"legacyIndex": ".my-so-index",
|
||||
"logs": Array [
|
||||
Object {
|
||||
|
|
@ -884,22 +892,6 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -1093,7 +1085,20 @@ Object {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"task|someHash": "10.1.0",
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
"typeD|someHash": "10.1.0",
|
||||
"typeE|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -1110,6 +1115,7 @@ Object {
|
|||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"latestMappingsVersions": Object {},
|
||||
"legacyIndex": ".my-so-index",
|
||||
"logs": Array [
|
||||
Object {
|
||||
|
|
@ -1145,22 +1151,6 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
@ -1328,7 +1318,20 @@ Object {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"task|someHash": "10.1.0",
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
"typeD|someHash": "10.1.0",
|
||||
"typeE|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".my-so-index",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -1345,6 +1348,7 @@ Object {
|
|||
},
|
||||
"kibanaVersion": "7.11.0",
|
||||
"knownTypes": Array [],
|
||||
"latestMappingsVersions": Object {},
|
||||
"legacyIndex": ".my-so-index",
|
||||
"logs": Array [
|
||||
Object {
|
||||
|
|
@ -1384,22 +1388,6 @@ Object {
|
|||
"retryCount": 0,
|
||||
"retryDelay": 0,
|
||||
"targetIndexMappings": Object {
|
||||
"_meta": Object {
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
".kibana_cases": Array [
|
||||
"typeD",
|
||||
"typeE",
|
||||
],
|
||||
".kibana_task_manager": Array [
|
||||
"task",
|
||||
],
|
||||
},
|
||||
},
|
||||
"properties": Object {},
|
||||
},
|
||||
"tempIndex": ".my-so-index_7.11.0_reindex_temp",
|
||||
|
|
|
|||
|
|
@ -9,28 +9,23 @@
|
|||
import * as Either from 'fp-ts/lib/Either';
|
||||
import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
checkTargetMappings,
|
||||
type ComparedMappingsChanged,
|
||||
type ComparedMappingsMatch,
|
||||
} from './check_target_mappings';
|
||||
import { getUpdatedHashes } from '../core/build_active_mappings';
|
||||
import { checkTargetMappings } from './check_target_mappings';
|
||||
import { getBaseMappings } from '../core';
|
||||
|
||||
jest.mock('../core/build_active_mappings');
|
||||
|
||||
const getUpdatedHashesMock = getUpdatedHashes as jest.MockedFn<typeof getUpdatedHashes>;
|
||||
const indexTypes = ['type1', 'type2', 'type3'];
|
||||
|
||||
const properties: SavedObjectsMappingProperties = {
|
||||
...getBaseMappings().properties,
|
||||
type1: { type: 'long' },
|
||||
type2: { type: 'long' },
|
||||
};
|
||||
|
||||
const migrationMappingPropertyHashes = {
|
||||
type1: 'type1Hash',
|
||||
type2: 'type2Hash',
|
||||
type1: 'someHash',
|
||||
type2: 'anotherHash',
|
||||
};
|
||||
|
||||
const expectedMappings: IndexMapping = {
|
||||
const legacyMappings: IndexMapping = {
|
||||
properties,
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
|
|
@ -38,96 +33,231 @@ const expectedMappings: IndexMapping = {
|
|||
},
|
||||
};
|
||||
|
||||
const outdatedModelVersions = {
|
||||
type1: '10.1.0',
|
||||
type2: '10.2.0',
|
||||
type3: '10.4.0',
|
||||
};
|
||||
|
||||
const modelVersions = {
|
||||
type1: '10.1.0',
|
||||
type2: '10.3.0',
|
||||
type3: '10.5.0',
|
||||
};
|
||||
|
||||
const latestMappingsVersions = {
|
||||
type1: '10.1.0',
|
||||
type2: '10.2.0', // type is on '10.3.0' but its mappings were last updated on 10.2.0
|
||||
type3: '10.5.0',
|
||||
};
|
||||
|
||||
const appMappings: IndexMapping = {
|
||||
properties,
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes, // deprecated, but preserved to facilitate rollback
|
||||
mappingVersions: modelVersions,
|
||||
},
|
||||
};
|
||||
|
||||
describe('checkTargetMappings', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('when actual mappings are incomplete', () => {
|
||||
it("returns 'actual_mappings_incomplete' if actual mappings are not defined", async () => {
|
||||
describe('when index mappings are missing required properties', () => {
|
||||
it("returns 'index_mappings_incomplete' if index mappings are not defined", async () => {
|
||||
const task = checkTargetMappings({
|
||||
expectedMappings,
|
||||
indexTypes,
|
||||
appMappings,
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const }));
|
||||
expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const }));
|
||||
});
|
||||
|
||||
it("returns 'actual_mappings_incomplete' if actual mappings do not define _meta", async () => {
|
||||
it("returns 'index_mappings_incomplete' if index mappings do not define _meta", async () => {
|
||||
const task = checkTargetMappings({
|
||||
expectedMappings,
|
||||
actualMappings: {
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: {
|
||||
properties,
|
||||
dynamic: 'strict',
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const }));
|
||||
expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const }));
|
||||
});
|
||||
|
||||
it("returns 'actual_mappings_incomplete' if actual mappings do not define migrationMappingPropertyHashes", async () => {
|
||||
it("returns 'index_mappings_incomplete' if index mappings do not define migrationMappingPropertyHashes nor mappingVersions", async () => {
|
||||
const task = checkTargetMappings({
|
||||
expectedMappings,
|
||||
actualMappings: {
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: {
|
||||
properties,
|
||||
dynamic: 'strict',
|
||||
_meta: {},
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const }));
|
||||
expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const }));
|
||||
});
|
||||
|
||||
it("returns 'actual_mappings_incomplete' if actual mappings define a different value for 'dynamic' property", async () => {
|
||||
it("returns 'index_mappings_incomplete' if index mappings define a different value for 'dynamic' property", async () => {
|
||||
const task = checkTargetMappings({
|
||||
expectedMappings,
|
||||
actualMappings: {
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: {
|
||||
properties,
|
||||
dynamic: false,
|
||||
_meta: { migrationMappingPropertyHashes },
|
||||
_meta: appMappings._meta,
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(Either.left({ type: 'actual_mappings_incomplete' as const }));
|
||||
expect(result).toEqual(Either.left({ type: 'index_mappings_incomplete' as const }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('when actual mappings are complete', () => {
|
||||
describe('and mappings do not match', () => {
|
||||
it('returns the lists of changed root fields and types', async () => {
|
||||
describe('when index mappings have all required properties', () => {
|
||||
describe('when some core properties (aka root fields) have changed', () => {
|
||||
it('returns the list of fields that have changed', async () => {
|
||||
const task = checkTargetMappings({
|
||||
expectedMappings,
|
||||
actualMappings: expectedMappings,
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: {
|
||||
...legacyMappings,
|
||||
properties: {
|
||||
...legacyMappings.properties,
|
||||
references: {
|
||||
properties: {
|
||||
...legacyMappings.properties.references.properties,
|
||||
description: { type: 'text' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
getUpdatedHashesMock.mockReturnValueOnce(['type1', 'type2', 'someRootField']);
|
||||
|
||||
const result = await task();
|
||||
const expected: ComparedMappingsChanged = {
|
||||
type: 'compared_mappings_changed' as const,
|
||||
updatedHashes: ['type1', 'type2', 'someRootField'],
|
||||
};
|
||||
expect(result).toEqual(Either.left(expected));
|
||||
expect(result).toEqual(
|
||||
Either.left({
|
||||
type: 'root_fields_changed' as const,
|
||||
updatedFields: ['references'],
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('and mappings match', () => {
|
||||
it('returns a compared_mappings_match response', async () => {
|
||||
const task = checkTargetMappings({
|
||||
expectedMappings,
|
||||
actualMappings: expectedMappings,
|
||||
describe('when core properties have NOT changed', () => {
|
||||
describe('when index mappings ONLY contain the legacy hashes', () => {
|
||||
describe('and legacy hashes match the current model versions', () => {
|
||||
it('returns a compared_mappings_match response', async () => {
|
||||
const task = checkTargetMappings({
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: legacyMappings,
|
||||
hashToVersionMap: {
|
||||
'type1|someHash': '10.1.0',
|
||||
'type2|anotherHash': '10.2.0',
|
||||
// type 3 is a new type
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(
|
||||
Either.right({
|
||||
type: 'compared_mappings_match' as const,
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
getUpdatedHashesMock.mockReturnValueOnce([]);
|
||||
describe('and legacy hashes do NOT match the current model versions', () => {
|
||||
it('returns the list of updated SO types', async () => {
|
||||
const task = checkTargetMappings({
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: legacyMappings,
|
||||
hashToVersionMap: {
|
||||
'type1|someHash': '10.1.0',
|
||||
'type2|anotherHash': '10.1.0', // type2's mappings were updated on 10.2.0
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
const expected: ComparedMappingsMatch = {
|
||||
type: 'compared_mappings_match' as const,
|
||||
};
|
||||
expect(result).toEqual(Either.right(expected));
|
||||
const result = await task();
|
||||
expect(result).toEqual(
|
||||
Either.left({
|
||||
type: 'types_changed' as const,
|
||||
updatedTypes: ['type2'],
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when index mappings contain the mappingVersions', () => {
|
||||
describe('and mappingVersions match', () => {
|
||||
it('returns a compared_mappings_match response', async () => {
|
||||
const task = checkTargetMappings({
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: {
|
||||
...appMappings,
|
||||
_meta: {
|
||||
...appMappings._meta,
|
||||
mappingVersions: {
|
||||
type1: '10.1.0',
|
||||
type2: '10.2.0', // type 2 was still on 10.2.0, but 10.3.0 does not bring mappings changes
|
||||
type3: '10.5.0',
|
||||
},
|
||||
},
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(
|
||||
Either.right({
|
||||
type: 'compared_mappings_match' as const,
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('and mappingVersions do NOT match', () => {
|
||||
it('returns the list of updated SO types', async () => {
|
||||
const task = checkTargetMappings({
|
||||
indexTypes,
|
||||
appMappings,
|
||||
indexMappings: {
|
||||
properties,
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
mappingVersions: outdatedModelVersions,
|
||||
},
|
||||
},
|
||||
latestMappingsVersions,
|
||||
});
|
||||
|
||||
const result = await task();
|
||||
expect(result).toEqual(
|
||||
Either.left({
|
||||
type: 'types_changed' as const,
|
||||
updatedTypes: ['type3'],
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,13 +8,16 @@
|
|||
import * as Either from 'fp-ts/lib/Either';
|
||||
import * as TaskEither from 'fp-ts/lib/TaskEither';
|
||||
|
||||
import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getUpdatedHashes } from '../core/build_active_mappings';
|
||||
import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getUpdatedRootFields, getUpdatedTypes } from '../core/compare_mappings';
|
||||
|
||||
/** @internal */
|
||||
export interface CheckTargetMappingsParams {
|
||||
actualMappings?: IndexMapping;
|
||||
expectedMappings: IndexMapping;
|
||||
indexTypes: string[];
|
||||
indexMappings?: IndexMapping;
|
||||
appMappings: IndexMapping;
|
||||
latestMappingsVersions: VirtualVersionMap;
|
||||
hashToVersionMap?: Record<string, string>;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
|
|
@ -22,40 +25,60 @@ export interface ComparedMappingsMatch {
|
|||
type: 'compared_mappings_match';
|
||||
}
|
||||
|
||||
export interface ActualMappingsIncomplete {
|
||||
type: 'actual_mappings_incomplete';
|
||||
export interface IndexMappingsIncomplete {
|
||||
type: 'index_mappings_incomplete';
|
||||
}
|
||||
|
||||
export interface ComparedMappingsChanged {
|
||||
type: 'compared_mappings_changed';
|
||||
updatedHashes: string[];
|
||||
export interface RootFieldsChanged {
|
||||
type: 'root_fields_changed';
|
||||
updatedFields: string[];
|
||||
}
|
||||
|
||||
export interface TypesChanged {
|
||||
type: 'types_changed';
|
||||
updatedTypes: string[];
|
||||
}
|
||||
|
||||
export const checkTargetMappings =
|
||||
({
|
||||
actualMappings,
|
||||
expectedMappings,
|
||||
indexTypes,
|
||||
indexMappings,
|
||||
appMappings,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap = {},
|
||||
}: CheckTargetMappingsParams): TaskEither.TaskEither<
|
||||
ActualMappingsIncomplete | ComparedMappingsChanged,
|
||||
IndexMappingsIncomplete | RootFieldsChanged | TypesChanged,
|
||||
ComparedMappingsMatch
|
||||
> =>
|
||||
async () => {
|
||||
if (
|
||||
!actualMappings?._meta?.migrationMappingPropertyHashes ||
|
||||
actualMappings.dynamic !== expectedMappings.dynamic
|
||||
(!indexMappings?._meta?.migrationMappingPropertyHashes &&
|
||||
!indexMappings?._meta?.mappingVersions) ||
|
||||
indexMappings.dynamic !== appMappings.dynamic
|
||||
) {
|
||||
return Either.left({ type: 'actual_mappings_incomplete' as const });
|
||||
return Either.left({ type: 'index_mappings_incomplete' as const });
|
||||
}
|
||||
|
||||
const updatedHashes = getUpdatedHashes({
|
||||
actual: actualMappings,
|
||||
expected: expectedMappings,
|
||||
const updatedFields = getUpdatedRootFields(indexMappings);
|
||||
|
||||
if (updatedFields.length) {
|
||||
return Either.left({
|
||||
type: 'root_fields_changed',
|
||||
updatedFields,
|
||||
});
|
||||
}
|
||||
|
||||
const updatedTypes = getUpdatedTypes({
|
||||
indexTypes,
|
||||
indexMeta: indexMappings?._meta,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap,
|
||||
});
|
||||
|
||||
if (updatedHashes.length) {
|
||||
if (updatedTypes.length) {
|
||||
return Either.left({
|
||||
type: 'compared_mappings_changed' as const,
|
||||
updatedHashes,
|
||||
type: 'types_changed' as const,
|
||||
updatedTypes,
|
||||
});
|
||||
} else {
|
||||
return Either.right({ type: 'compared_mappings_match' as const });
|
||||
|
|
|
|||
|
|
@ -108,7 +108,11 @@ import type { UnknownDocsFound } from './check_for_unknown_docs';
|
|||
import type { IncompatibleClusterRoutingAllocation } from './initialize_action';
|
||||
import type { ClusterShardLimitExceeded } from './create_index';
|
||||
import type { SynchronizationFailed } from './synchronize_migrators';
|
||||
import type { ActualMappingsIncomplete, ComparedMappingsChanged } from './check_target_mappings';
|
||||
import type {
|
||||
IndexMappingsIncomplete,
|
||||
RootFieldsChanged,
|
||||
TypesChanged,
|
||||
} from './check_target_mappings';
|
||||
|
||||
export type {
|
||||
CheckForUnknownDocsParams,
|
||||
|
|
@ -182,8 +186,9 @@ export interface ActionErrorTypeMap {
|
|||
cluster_shard_limit_exceeded: ClusterShardLimitExceeded;
|
||||
es_response_too_large: EsResponseTooLargeError;
|
||||
synchronization_failed: SynchronizationFailed;
|
||||
actual_mappings_incomplete: ActualMappingsIncomplete;
|
||||
compared_mappings_changed: ComparedMappingsChanged;
|
||||
index_mappings_incomplete: IndexMappingsIncomplete;
|
||||
root_fields_changed: RootFieldsChanged;
|
||||
types_changed: TypesChanged;
|
||||
operation_not_supported: OperationNotSupported;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,8 +27,11 @@ export interface IncompatibleMappingException {
|
|||
}
|
||||
|
||||
/**
|
||||
* Updates an index's mappings and runs an pickupUpdatedMappings task so that the mapping
|
||||
* changes are "picked up". Returns a taskId to track progress.
|
||||
* Attempts to update the SO index mappings.
|
||||
* Includes an automatic retry mechanism for retriable errors.
|
||||
* Returns an 'update_mappings_succeeded' upon success.
|
||||
* If changes in the mappings are NOT compatible and the update fails on ES side,
|
||||
* this method will return an 'incompatible_mapping_exception'.
|
||||
*/
|
||||
export const updateMappings = ({
|
||||
client,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
updateSourceMappingsProperties,
|
||||
type UpdateSourceMappingsPropertiesParams,
|
||||
} from './update_source_mappings_properties';
|
||||
import { getBaseMappings } from '../core';
|
||||
|
||||
describe('updateSourceMappingsProperties', () => {
|
||||
let client: ReturnType<typeof elasticsearchClientMock.createInternalClient>;
|
||||
|
|
@ -22,11 +23,18 @@ describe('updateSourceMappingsProperties', () => {
|
|||
client = elasticsearchClientMock.createInternalClient();
|
||||
params = {
|
||||
client,
|
||||
indexTypes: ['a', 'b', 'c'],
|
||||
latestMappingsVersions: {
|
||||
a: '10.1.0',
|
||||
b: '10.2.0',
|
||||
c: '10.5.0',
|
||||
},
|
||||
sourceIndex: '.kibana_8.7.0_001',
|
||||
sourceMappings: {
|
||||
indexMappings: {
|
||||
properties: {
|
||||
a: { type: 'keyword' },
|
||||
b: { type: 'long' },
|
||||
...getBaseMappings().properties,
|
||||
},
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
|
|
@ -35,23 +43,33 @@ describe('updateSourceMappingsProperties', () => {
|
|||
},
|
||||
},
|
||||
},
|
||||
targetMappings: {
|
||||
appMappings: {
|
||||
properties: {
|
||||
a: { type: 'keyword' },
|
||||
c: { type: 'long' },
|
||||
...getBaseMappings().properties,
|
||||
},
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
a: '000',
|
||||
c: '222',
|
||||
mappingVersions: {
|
||||
a: '10.1.0',
|
||||
b: '10.3.0',
|
||||
c: '10.5.0',
|
||||
},
|
||||
},
|
||||
},
|
||||
hashToVersionMap: {
|
||||
'a|000': '10.1.0',
|
||||
'b|111': '10.1.0',
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
it('should not update mappings when there are no changes', async () => {
|
||||
const sameMappingsParams = chain(params).set('targetMappings', params.sourceMappings).value();
|
||||
// we overwrite the app mappings to have the "unchanged" values with respect to the index mappings
|
||||
const sameMappingsParams = chain(params)
|
||||
// even if the app versions are more recent, we emulate a scenario where mappings haven NOT changed
|
||||
.set('latestMappingsVersions', { a: '10.1.0', b: '10.1.0', c: '10.1.0' })
|
||||
.value();
|
||||
const result = await updateSourceMappingsProperties(sameMappingsParams)();
|
||||
|
||||
expect(client.indices.putMapping).not.toHaveBeenCalled();
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ import { omit } from 'lodash';
|
|||
import * as TaskEither from 'fp-ts/lib/TaskEither';
|
||||
import { pipe } from 'fp-ts/lib/pipeable';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { diffMappings } from '../core/build_active_mappings';
|
||||
import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { diffMappings } from '../core/diff_mappings';
|
||||
import type { RetryableEsClientError } from './catch_retryable_es_client_errors';
|
||||
import { updateMappings } from './update_mappings';
|
||||
import type { IncompatibleMappingException } from './update_mappings';
|
||||
|
|
@ -20,8 +20,11 @@ import type { IncompatibleMappingException } from './update_mappings';
|
|||
export interface UpdateSourceMappingsPropertiesParams {
|
||||
client: ElasticsearchClient;
|
||||
sourceIndex: string;
|
||||
sourceMappings: IndexMapping;
|
||||
targetMappings: IndexMapping;
|
||||
indexMappings: IndexMapping;
|
||||
appMappings: IndexMapping;
|
||||
indexTypes: string[];
|
||||
latestMappingsVersions: VirtualVersionMap;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -31,14 +34,23 @@ export interface UpdateSourceMappingsPropertiesParams {
|
|||
export const updateSourceMappingsProperties = ({
|
||||
client,
|
||||
sourceIndex,
|
||||
sourceMappings,
|
||||
targetMappings,
|
||||
indexMappings,
|
||||
appMappings,
|
||||
indexTypes,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap,
|
||||
}: UpdateSourceMappingsPropertiesParams): TaskEither.TaskEither<
|
||||
RetryableEsClientError | IncompatibleMappingException,
|
||||
'update_mappings_succeeded'
|
||||
> => {
|
||||
return pipe(
|
||||
diffMappings(sourceMappings, targetMappings),
|
||||
diffMappings({
|
||||
indexMappings,
|
||||
appMappings,
|
||||
indexTypes,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap,
|
||||
}),
|
||||
TaskEither.fromPredicate(
|
||||
(changes) => !!changes,
|
||||
() => 'update_mappings_succeeded' as const
|
||||
|
|
@ -48,7 +60,7 @@ export const updateSourceMappingsProperties = ({
|
|||
updateMappings({
|
||||
client,
|
||||
index: sourceIndex,
|
||||
mappings: omit(targetMappings, ['_meta']), // ._meta property will be updated on a later step
|
||||
mappings: omit(appMappings, ['_meta']), // ._meta property will be updated on a later step
|
||||
})
|
||||
)
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2,22 +2,6 @@
|
|||
|
||||
exports[`buildActiveMappings combines all mappings and includes core mappings 1`] = `
|
||||
Object {
|
||||
"_meta": Object {
|
||||
"migrationMappingPropertyHashes": Object {
|
||||
"aaa": "625b32086eb1d1203564cf85062dd22e",
|
||||
"bbb": "18c78c995965207ed3f6e7fc5c6e55fe",
|
||||
"coreMigrationVersion": "2f4316de49999235636386fe51dc06c1",
|
||||
"created_at": "00da57df13e94e9d98437d13ace4bfe0",
|
||||
"managed": "88cf246b441a6362458cb6a56ca3f7d7",
|
||||
"namespace": "2f4316de49999235636386fe51dc06c1",
|
||||
"namespaces": "2f4316de49999235636386fe51dc06c1",
|
||||
"originId": "2f4316de49999235636386fe51dc06c1",
|
||||
"references": "7997cf5a56cc02bdc9c93361bde732b0",
|
||||
"type": "2f4316de49999235636386fe51dc06c1",
|
||||
"typeMigrationVersion": "539e3ecebb3abc1133618094cc3b7ae7",
|
||||
"updated_at": "00da57df13e94e9d98437d13ace4bfe0",
|
||||
},
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": Object {
|
||||
"aaa": Object {
|
||||
|
|
@ -73,23 +57,6 @@ Object {
|
|||
|
||||
exports[`buildActiveMappings handles the \`dynamic\` property of types 1`] = `
|
||||
Object {
|
||||
"_meta": Object {
|
||||
"migrationMappingPropertyHashes": Object {
|
||||
"coreMigrationVersion": "2f4316de49999235636386fe51dc06c1",
|
||||
"created_at": "00da57df13e94e9d98437d13ace4bfe0",
|
||||
"firstType": "635418ab953d81d93f1190b70a8d3f57",
|
||||
"managed": "88cf246b441a6362458cb6a56ca3f7d7",
|
||||
"namespace": "2f4316de49999235636386fe51dc06c1",
|
||||
"namespaces": "2f4316de49999235636386fe51dc06c1",
|
||||
"originId": "2f4316de49999235636386fe51dc06c1",
|
||||
"references": "7997cf5a56cc02bdc9c93361bde732b0",
|
||||
"secondType": "72d57924f415fbadb3ee293b67d233ab",
|
||||
"thirdType": "510f1f0adb69830cf8a1c5ce2923ed82",
|
||||
"type": "2f4316de49999235636386fe51dc06c1",
|
||||
"typeMigrationVersion": "539e3ecebb3abc1133618094cc3b7ae7",
|
||||
"updated_at": "00da57df13e94e9d98437d13ace4bfe0",
|
||||
},
|
||||
},
|
||||
"dynamic": "strict",
|
||||
"properties": Object {
|
||||
"coreMigrationVersion": Object {
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@
|
|||
*/
|
||||
|
||||
import type {
|
||||
IndexMapping,
|
||||
IndexMappingMeta,
|
||||
SavedObjectsTypeMappingDefinitions,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { buildActiveMappings, diffMappings, getUpdatedHashes } from './build_active_mappings';
|
||||
import { buildActiveMappings, getBaseMappings } from './build_active_mappings';
|
||||
|
||||
describe('buildActiveMappings', () => {
|
||||
test('creates a strict mapping', () => {
|
||||
|
|
@ -58,215 +58,74 @@ describe('buildActiveMappings', () => {
|
|||
expect(buildActiveMappings(typeMappings)).toMatchSnapshot();
|
||||
});
|
||||
|
||||
test('generated hashes are stable', () => {
|
||||
test(`includes the provided override properties, except for 'properties'`, () => {
|
||||
const properties = {
|
||||
aaa: { type: 'keyword', fields: { a: { type: 'keyword' }, b: { type: 'text' } } },
|
||||
bbb: { fields: { b: { type: 'text' }, a: { type: 'keyword' } }, type: 'keyword' },
|
||||
ccc: { fields: { b: { type: 'text' }, a: { type: 'text' } }, type: 'keyword' },
|
||||
} as const;
|
||||
|
||||
const mappings = buildActiveMappings(properties);
|
||||
const hashes = mappings._meta!.migrationMappingPropertyHashes!;
|
||||
const ourExternallyBuiltMeta: IndexMappingMeta = {
|
||||
mappingVersions: {
|
||||
foo: '10.1.0',
|
||||
bar: '10.2.0',
|
||||
baz: '10.3.0',
|
||||
},
|
||||
};
|
||||
|
||||
expect(hashes.aaa).toBeDefined();
|
||||
expect(hashes.aaa).toEqual(hashes.bbb);
|
||||
expect(hashes.aaa).not.toEqual(hashes.ccc);
|
||||
const mappings = buildActiveMappings(properties, ourExternallyBuiltMeta);
|
||||
expect(mappings._meta).toEqual(ourExternallyBuiltMeta);
|
||||
expect(mappings.properties.ddd).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('diffMappings', () => {
|
||||
test('is different if expected contains extra hashes', () => {
|
||||
const actual: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar', baz: 'qux' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)!.changedProp).toEqual('properties.baz');
|
||||
});
|
||||
|
||||
test('does nothing if actual contains extra hashes', () => {
|
||||
const actual: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar', baz: 'qux' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)).toBeUndefined();
|
||||
});
|
||||
|
||||
test('does nothing if actual hashes are identical to expected, but properties differ', () => {
|
||||
const actual: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
describe('getBaseMappings', () => {
|
||||
test('changes in core fields trigger a pickup of all documents, which can be really costly. Update only if you know what you are doing', () => {
|
||||
expect(getBaseMappings()).toEqual({
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
foo: { type: 'keyword' },
|
||||
},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
foo: { type: 'text' },
|
||||
},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)).toBeUndefined();
|
||||
});
|
||||
|
||||
test('is different if meta hashes change', () => {
|
||||
const actual: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'baz' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)!.changedProp).toEqual('properties.foo');
|
||||
});
|
||||
|
||||
test('is different if dynamic is different', () => {
|
||||
const actual: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
// @ts-expect-error
|
||||
dynamic: 'abcde',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)!.changedProp).toEqual('dynamic');
|
||||
});
|
||||
|
||||
test('is different if migrationMappingPropertyHashes is missing from actual', () => {
|
||||
const actual: IndexMapping = {
|
||||
_meta: {},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)!.changedProp).toEqual('_meta');
|
||||
});
|
||||
|
||||
test('is different if _meta is missing from actual', () => {
|
||||
const actual: IndexMapping = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(diffMappings(actual, expected)!.changedProp).toEqual('_meta');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getUpdatedHashes', () => {
|
||||
test('gives all hashes if _meta is missing from actual', () => {
|
||||
const actual: IndexMapping = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar', bar: 'baz' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(getUpdatedHashes({ actual, expected })).toEqual(['foo', 'bar']);
|
||||
});
|
||||
|
||||
test('gives all hashes if migrationMappingPropertyHashes is missing from actual', () => {
|
||||
const actual: IndexMapping = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
_meta: {},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: { foo: 'bar', bar: 'baz' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
expect(getUpdatedHashes({ actual, expected })).toEqual(['foo', 'bar']);
|
||||
});
|
||||
|
||||
test('gives a list of the types with updated hashes', () => {
|
||||
const actual: IndexMapping = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
type1: 'type1hash1',
|
||||
type2: 'type2hash1',
|
||||
type3: 'type3hash1', // will be removed
|
||||
type: {
|
||||
type: 'keyword',
|
||||
},
|
||||
namespace: {
|
||||
type: 'keyword',
|
||||
},
|
||||
namespaces: {
|
||||
type: 'keyword',
|
||||
},
|
||||
originId: {
|
||||
type: 'keyword',
|
||||
},
|
||||
updated_at: {
|
||||
type: 'date',
|
||||
},
|
||||
created_at: {
|
||||
type: 'date',
|
||||
},
|
||||
references: {
|
||||
type: 'nested',
|
||||
properties: {
|
||||
name: {
|
||||
type: 'keyword',
|
||||
},
|
||||
type: {
|
||||
type: 'keyword',
|
||||
},
|
||||
id: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
},
|
||||
coreMigrationVersion: {
|
||||
type: 'keyword',
|
||||
},
|
||||
typeMigrationVersion: {
|
||||
type: 'version',
|
||||
},
|
||||
managed: {
|
||||
type: 'boolean',
|
||||
},
|
||||
},
|
||||
};
|
||||
const expected: IndexMapping = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
type1: 'type1hash1', // remains the same
|
||||
type2: 'type2hash2', // updated
|
||||
type4: 'type4hash1', // new type
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(getUpdatedHashes({ actual, expected })).toEqual(['type2', 'type4']);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,15 +6,11 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file contains logic to build and diff the index mappings for a migration.
|
||||
*/
|
||||
|
||||
import crypto from 'crypto';
|
||||
import { cloneDeep, mapValues } from 'lodash';
|
||||
import { cloneDeep } from 'lodash';
|
||||
import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server';
|
||||
import type {
|
||||
IndexMapping,
|
||||
IndexMappingMeta,
|
||||
SavedObjectsTypeMappingDefinitions,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
|
||||
|
|
@ -25,116 +21,21 @@ import type {
|
|||
* @param typeDefinitions - the type definitions to build mapping from.
|
||||
*/
|
||||
export function buildActiveMappings(
|
||||
typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties
|
||||
typeDefinitions: SavedObjectsTypeMappingDefinitions | SavedObjectsMappingProperties,
|
||||
_meta?: IndexMappingMeta
|
||||
): IndexMapping {
|
||||
const mapping = getBaseMappings();
|
||||
|
||||
const mergedProperties = validateAndMerge(mapping.properties, typeDefinitions);
|
||||
|
||||
return cloneDeep({
|
||||
...mapping,
|
||||
properties: mergedProperties,
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: md5Values(mergedProperties),
|
||||
},
|
||||
properties: validateAndMerge(mapping.properties, typeDefinitions),
|
||||
...(_meta && { _meta }),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Diffs the actual vs expected mappings. The properties are compared using md5 hashes stored in _meta, because
|
||||
* actual and expected mappings *can* differ, but if the md5 hashes stored in actual._meta.migrationMappingPropertyHashes
|
||||
* match our expectations, we don't require a migration. This allows ES to tack on additional mappings that Kibana
|
||||
* doesn't know about or expect, without triggering continual migrations.
|
||||
*/
|
||||
export function diffMappings(actual: IndexMapping, expected: IndexMapping) {
|
||||
if (actual.dynamic !== expected.dynamic) {
|
||||
return { changedProp: 'dynamic' };
|
||||
}
|
||||
|
||||
if (!actual._meta?.migrationMappingPropertyHashes) {
|
||||
return { changedProp: '_meta' };
|
||||
}
|
||||
|
||||
const changedProp = findChangedProp(
|
||||
actual._meta.migrationMappingPropertyHashes,
|
||||
expected._meta!.migrationMappingPropertyHashes
|
||||
);
|
||||
|
||||
return changedProp ? { changedProp: `properties.${changedProp}` } : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares the actual vs expected mappings' hashes.
|
||||
* Returns a list with all the hashes that have been updated.
|
||||
*/
|
||||
export const getUpdatedHashes = ({
|
||||
actual,
|
||||
expected,
|
||||
}: {
|
||||
actual: IndexMapping;
|
||||
expected: IndexMapping;
|
||||
}): string[] => {
|
||||
if (!actual._meta?.migrationMappingPropertyHashes) {
|
||||
return Object.keys(expected._meta!.migrationMappingPropertyHashes!);
|
||||
}
|
||||
|
||||
const updatedHashes = Object.keys(expected._meta!.migrationMappingPropertyHashes!).filter(
|
||||
(key) =>
|
||||
actual._meta!.migrationMappingPropertyHashes![key] !==
|
||||
expected._meta!.migrationMappingPropertyHashes![key]
|
||||
);
|
||||
|
||||
return updatedHashes;
|
||||
};
|
||||
|
||||
// Convert an object to an md5 hash string, using a stable serialization (canonicalStringify)
|
||||
function md5Object(obj: any) {
|
||||
return crypto.createHash('md5').update(canonicalStringify(obj)).digest('hex');
|
||||
}
|
||||
|
||||
// JSON.stringify is non-canonical, meaning the same object may produce slightly
|
||||
// different JSON, depending on compiler optimizations (e.g. object keys
|
||||
// are not guaranteed to be sorted). This function consistently produces the same
|
||||
// string, if passed an object of the same shape. If the outpuf of this function
|
||||
// changes from one release to another, migrations will run, so it's important
|
||||
// that this function remains stable across releases.
|
||||
function canonicalStringify(obj: any): string {
|
||||
if (Array.isArray(obj)) {
|
||||
return `[${obj.map(canonicalStringify)}]`;
|
||||
}
|
||||
|
||||
if (!obj || typeof obj !== 'object') {
|
||||
return JSON.stringify(obj);
|
||||
}
|
||||
|
||||
const keys = Object.keys(obj);
|
||||
|
||||
// This is important for properly handling Date
|
||||
if (!keys.length) {
|
||||
return JSON.stringify(obj);
|
||||
}
|
||||
|
||||
const sortedObj = keys
|
||||
.sort((a, b) => a.localeCompare(b))
|
||||
.map((k) => `${k}: ${canonicalStringify(obj[k])}`);
|
||||
|
||||
return `{${sortedObj}}`;
|
||||
}
|
||||
|
||||
// Convert an object's values to md5 hash strings
|
||||
function md5Values(obj: any) {
|
||||
return mapValues(obj, md5Object);
|
||||
}
|
||||
|
||||
// If something exists in actual, but is missing in expected, we don't
|
||||
// care, as it could be a disabled plugin, etc, and keeping stale stuff
|
||||
// around is better than migrating unecessesarily.
|
||||
function findChangedProp(actual: any, expected: any) {
|
||||
return Object.keys(expected).find((k) => actual[k] !== expected[k]);
|
||||
}
|
||||
|
||||
/**
|
||||
* These mappings are required for any saved object index.
|
||||
* Defines the mappings for the root fields, common to all saved objects.
|
||||
* These are present in all SO indices.
|
||||
*
|
||||
* @returns {IndexMapping}
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -9,23 +9,13 @@
|
|||
import { buildPickupMappingsQuery } from './build_pickup_mappings_query';
|
||||
|
||||
describe('buildPickupMappingsQuery', () => {
|
||||
describe('when no root fields have been updated', () => {
|
||||
it('builds a boolean query to select the updated types', () => {
|
||||
const query = buildPickupMappingsQuery(['type1', 'type2']);
|
||||
it('builds a boolean query to select the updated types', () => {
|
||||
const query = buildPickupMappingsQuery(['type1', 'type2']);
|
||||
|
||||
expect(query).toEqual({
|
||||
bool: {
|
||||
should: [{ term: { type: 'type1' } }, { term: { type: 'type2' } }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when some root fields have been updated', () => {
|
||||
it('returns undefined', () => {
|
||||
const query = buildPickupMappingsQuery(['type1', 'type2', 'namespaces']);
|
||||
|
||||
expect(query).toBeUndefined();
|
||||
expect(query).toEqual({
|
||||
bool: {
|
||||
should: [{ term: { type: 'type1' } }, { term: { type: 'type2' } }],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,22 +6,9 @@
|
|||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { getBaseMappings } from './build_active_mappings';
|
||||
|
||||
export const buildPickupMappingsQuery = (
|
||||
updatedFields: string[]
|
||||
): QueryDslQueryContainer | undefined => {
|
||||
const rootFields = Object.keys(getBaseMappings().properties);
|
||||
|
||||
if (updatedFields.some((field) => rootFields.includes(field))) {
|
||||
// we are updating some root fields, update ALL documents (no filter query)
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// at this point, all updated fields correspond to SO types
|
||||
const updatedTypes = updatedFields;
|
||||
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export const buildPickupMappingsQuery = (updatedTypes: string[]): QueryDslQueryContainer => {
|
||||
return {
|
||||
bool: {
|
||||
should: updatedTypes.map((type) => ({ term: { type } })),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,121 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getBaseMappings } from './build_active_mappings';
|
||||
import { getUpdatedTypes, getUpdatedRootFields } from './compare_mappings';
|
||||
|
||||
describe('getUpdatedTypes', () => {
|
||||
test('returns all types if _meta is missing in indexMappings', () => {
|
||||
const indexTypes = ['foo', 'bar'];
|
||||
const latestMappingsVersions = {};
|
||||
|
||||
expect(getUpdatedTypes({ indexTypes, indexMeta: undefined, latestMappingsVersions })).toEqual([
|
||||
'foo',
|
||||
'bar',
|
||||
]);
|
||||
});
|
||||
|
||||
test('returns all types if migrationMappingPropertyHashes and mappingVersions are missing in indexMappings', () => {
|
||||
const indexTypes = ['foo', 'bar'];
|
||||
const indexMeta: IndexMappingMeta = {};
|
||||
const latestMappingsVersions = {};
|
||||
|
||||
expect(getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions })).toEqual([
|
||||
'foo',
|
||||
'bar',
|
||||
]);
|
||||
});
|
||||
|
||||
describe('when ONLY migrationMappingPropertyHashes exists in indexMappings', () => {
|
||||
test('uses the provided hashToVersionMap to compare changes and return only the types that have changed', async () => {
|
||||
const indexTypes = ['type1', 'type2', 'type4'];
|
||||
const indexMeta: IndexMappingMeta = {
|
||||
migrationMappingPropertyHashes: {
|
||||
type1: 'someHash',
|
||||
type2: 'anotherHash',
|
||||
type3: 'aThirdHash', // will be removed
|
||||
},
|
||||
};
|
||||
|
||||
const hashToVersionMap = {
|
||||
'type1|someHash': '10.1.0',
|
||||
'type2|anotherHash': '10.1.0',
|
||||
'type3|aThirdHash': '10.1.0',
|
||||
};
|
||||
|
||||
const latestMappingsVersions = {
|
||||
type1: '10.1.0',
|
||||
type2: '10.2.0',
|
||||
type4: '10.5.0', // new type, no need to pick it up
|
||||
};
|
||||
|
||||
expect(
|
||||
getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions, hashToVersionMap })
|
||||
).toEqual(['type2']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('when mappingVersions exist in indexMappings', () => {
|
||||
test('compares the modelVersions and returns only the types that have changed', async () => {
|
||||
const indexTypes = ['type1', 'type2', 'type4'];
|
||||
|
||||
const indexMeta: IndexMappingMeta = {
|
||||
mappingVersions: {
|
||||
type1: '10.1.0',
|
||||
type2: '10.1.0',
|
||||
type3: '10.1.0', // will be removed
|
||||
},
|
||||
// ignored, cause mappingVersions is present
|
||||
migrationMappingPropertyHashes: {
|
||||
type1: 'someHash',
|
||||
type2: 'anotherHash',
|
||||
type3: 'aThirdHash',
|
||||
},
|
||||
};
|
||||
|
||||
const latestMappingsVersions = {
|
||||
type1: '10.1.0',
|
||||
type2: '10.2.0',
|
||||
type4: '10.5.0', // new type, no need to pick it up
|
||||
};
|
||||
|
||||
const hashToVersionMap = {
|
||||
// empty on purpose, not used as mappingVersions is present in indexMappings
|
||||
};
|
||||
|
||||
expect(
|
||||
getUpdatedTypes({ indexTypes, indexMeta, latestMappingsVersions, hashToVersionMap })
|
||||
).toEqual(['type2']);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getUpdatedRootFields', () => {
|
||||
it('deep compares provided indexMappings against the current baseMappings()', () => {
|
||||
const updatedFields = getUpdatedRootFields({
|
||||
properties: {
|
||||
...getBaseMappings().properties,
|
||||
namespace: {
|
||||
type: 'text',
|
||||
},
|
||||
references: {
|
||||
type: 'nested',
|
||||
properties: {
|
||||
...getBaseMappings().properties.references.properties,
|
||||
description: {
|
||||
type: 'text',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(updatedFields).toEqual(['namespace', 'references']);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import equals from 'fast-deep-equal';
|
||||
import Semver from 'semver';
|
||||
|
||||
import type {
|
||||
IndexMappingMeta,
|
||||
VirtualVersionMap,
|
||||
IndexMapping,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getBaseMappings } from './build_active_mappings';
|
||||
|
||||
/**
|
||||
* Compare the current mappings for root fields Vs those stored in the SO index.
|
||||
* Relies on getBaseMappings to determine the current mappings.
|
||||
* @param indexMappings The mappings stored in the SO index
|
||||
* @returns A list of the root fields whose mappings have changed
|
||||
*/
|
||||
export const getUpdatedRootFields = (indexMappings: IndexMapping): string[] => {
|
||||
const baseMappings = getBaseMappings();
|
||||
return Object.entries(baseMappings.properties)
|
||||
.filter(
|
||||
([propertyName, propertyValue]) =>
|
||||
!equals(propertyValue, indexMappings.properties[propertyName])
|
||||
)
|
||||
.map(([propertyName]) => propertyName);
|
||||
};
|
||||
|
||||
/**
|
||||
* Compares the current vs stored mappings' hashes or modelVersions.
|
||||
* Returns a list with all the types that have been updated.
|
||||
* @param indexMeta The meta information stored in the SO index
|
||||
* @param knownTypes The list of SO types that belong to the index and are enabled
|
||||
* @param latestMappingsVersions A map holding [type => version] with the latest versions where mappings have changed for each type
|
||||
* @param hashToVersionMap A map holding information about [md5 => modelVersion] equivalence
|
||||
* @returns the list of types that have been updated (in terms of their mappings)
|
||||
*/
|
||||
export const getUpdatedTypes = ({
|
||||
indexMeta,
|
||||
indexTypes,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap = {},
|
||||
}: {
|
||||
indexMeta?: IndexMappingMeta;
|
||||
indexTypes: string[];
|
||||
latestMappingsVersions: VirtualVersionMap;
|
||||
hashToVersionMap?: Record<string, string>;
|
||||
}): string[] => {
|
||||
if (!indexMeta || (!indexMeta.mappingVersions && !indexMeta.migrationMappingPropertyHashes)) {
|
||||
// if we currently do NOT have meta information stored in the index
|
||||
// we consider that all types have been updated
|
||||
return indexTypes;
|
||||
}
|
||||
|
||||
// If something exists in stored, but is missing in current
|
||||
// we don't care, as it could be a disabled plugin, etc
|
||||
// and keeping stale stuff around is better than migrating unecessesarily.
|
||||
return indexTypes.filter((type) =>
|
||||
isTypeUpdated({
|
||||
type,
|
||||
mappingVersion: latestMappingsVersions[type],
|
||||
indexMeta,
|
||||
hashToVersionMap,
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
*
|
||||
* @param type The saved object type to check
|
||||
* @param mappingVersion The most recent model version that includes mappings changes
|
||||
* @param indexMeta The meta information stored in the SO index
|
||||
* @param hashToVersionMap A map holding information about [md5 => modelVersion] equivalence
|
||||
* @returns true if the mappings for the given type have changed since Kibana was last started
|
||||
*/
|
||||
function isTypeUpdated({
|
||||
type,
|
||||
mappingVersion,
|
||||
indexMeta,
|
||||
hashToVersionMap,
|
||||
}: {
|
||||
type: string;
|
||||
mappingVersion: string;
|
||||
indexMeta: IndexMappingMeta;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
}): boolean {
|
||||
const latestMappingsVersion = Semver.parse(mappingVersion);
|
||||
if (!latestMappingsVersion) {
|
||||
throw new Error(
|
||||
`The '${type}' saved object type is not specifying a valid semver: ${mappingVersion}`
|
||||
);
|
||||
}
|
||||
|
||||
if (indexMeta.mappingVersions) {
|
||||
// the SO index is already using mappingVersions (instead of md5 hashes)
|
||||
const indexVersion = indexMeta.mappingVersions[type];
|
||||
if (!indexVersion) {
|
||||
// either a new type, and thus there's not need to update + pickup any docs
|
||||
// or an old re-enabled type, which will be updated on OUTDATED_DOCUMENTS_TRANSFORM
|
||||
return false;
|
||||
}
|
||||
|
||||
// if the last version where mappings have changed is more recent than the one stored in the index
|
||||
// it means that the type has been updated
|
||||
return latestMappingsVersion.compare(indexVersion) === 1;
|
||||
} else if (indexMeta.migrationMappingPropertyHashes) {
|
||||
const latestHash = indexMeta.migrationMappingPropertyHashes?.[type];
|
||||
|
||||
if (!latestHash) {
|
||||
// either a new type, and thus there's not need to update + pickup any docs
|
||||
// or an old re-enabled type, which will be updated on OUTDATED_DOCUMENTS_TRANSFORM
|
||||
return false;
|
||||
}
|
||||
|
||||
const indexEquivalentVersion = hashToVersionMap[`${type}|${latestHash}`];
|
||||
return !indexEquivalentVersion || latestMappingsVersion.compare(indexEquivalentVersion) === 1;
|
||||
}
|
||||
|
||||
// at this point, the mappings do not contain any meta informataion
|
||||
// we consider the type has been updated, out of caution
|
||||
return true;
|
||||
}
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { IndexMapping } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getBaseMappings } from './build_active_mappings';
|
||||
import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings';
|
||||
import { diffMappings } from './diff_mappings';
|
||||
|
||||
jest.mock('./compare_mappings');
|
||||
const getUpdatedRootFieldsMock = getUpdatedRootFields as jest.MockedFn<typeof getUpdatedRootFields>;
|
||||
const getUpdatedTypesMock = getUpdatedTypes as jest.MockedFn<typeof getUpdatedTypes>;
|
||||
|
||||
const dummyMappings: IndexMapping = {
|
||||
_meta: {
|
||||
mappingVersions: { foo: '10.1.0' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
|
||||
const initialMappings: IndexMapping = {
|
||||
_meta: {
|
||||
mappingVersions: { foo: '10.1.0' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
...getBaseMappings().properties,
|
||||
foo: {
|
||||
type: 'text',
|
||||
},
|
||||
},
|
||||
};
|
||||
const updatedMappings: IndexMapping = {
|
||||
_meta: {
|
||||
mappingVersions: { foo: '10.2.0' },
|
||||
},
|
||||
dynamic: 'strict',
|
||||
properties: {
|
||||
...getBaseMappings().properties,
|
||||
foo: {
|
||||
type: 'keyword',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const dummyHashToVersionMap = {
|
||||
'foo|someHash': '10.1.0',
|
||||
};
|
||||
|
||||
describe('diffMappings', () => {
|
||||
beforeEach(() => {
|
||||
getUpdatedRootFieldsMock.mockReset();
|
||||
getUpdatedTypesMock.mockReset();
|
||||
});
|
||||
|
||||
test('is different if dynamic is different', () => {
|
||||
const indexMappings = dummyMappings;
|
||||
const appMappings: IndexMapping = {
|
||||
...dummyMappings,
|
||||
dynamic: false,
|
||||
};
|
||||
|
||||
expect(
|
||||
diffMappings({ indexTypes: ['foo'], appMappings, indexMappings, latestMappingsVersions: {} })!
|
||||
.changedProp
|
||||
).toEqual('dynamic');
|
||||
});
|
||||
|
||||
test('is different if _meta is missing in indexMappings', () => {
|
||||
const indexMappings: IndexMapping = {
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const appMappings: IndexMapping = dummyMappings;
|
||||
|
||||
expect(
|
||||
diffMappings({ indexTypes: ['foo'], appMappings, indexMappings, latestMappingsVersions: {} })!
|
||||
.changedProp
|
||||
).toEqual('_meta');
|
||||
});
|
||||
|
||||
test('is different if migrationMappingPropertyHashes and mappingVersions are missing in indexMappings', () => {
|
||||
const indexMappings: IndexMapping = {
|
||||
_meta: {},
|
||||
dynamic: 'strict',
|
||||
properties: {},
|
||||
};
|
||||
const appMappings: IndexMapping = dummyMappings;
|
||||
|
||||
expect(
|
||||
diffMappings({ indexTypes: ['foo'], appMappings, indexMappings, latestMappingsVersions: {} })!
|
||||
.changedProp
|
||||
).toEqual('_meta');
|
||||
});
|
||||
|
||||
describe('if a root field has changed', () => {
|
||||
test('returns that root field', () => {
|
||||
getUpdatedRootFieldsMock.mockReturnValueOnce(['references']);
|
||||
|
||||
expect(
|
||||
diffMappings({
|
||||
indexTypes: ['foo'],
|
||||
appMappings: updatedMappings,
|
||||
indexMappings: initialMappings,
|
||||
latestMappingsVersions: {},
|
||||
})
|
||||
).toEqual({ changedProp: 'properties.references' });
|
||||
|
||||
expect(getUpdatedRootFieldsMock).toHaveBeenCalledTimes(1);
|
||||
expect(getUpdatedRootFieldsMock).toHaveBeenCalledWith(initialMappings);
|
||||
expect(getUpdatedTypesMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('if some types have changed', () => {
|
||||
test('returns a changed type', () => {
|
||||
getUpdatedRootFieldsMock.mockReturnValueOnce([]);
|
||||
getUpdatedTypesMock.mockReturnValueOnce(['foo', 'bar']);
|
||||
|
||||
expect(
|
||||
diffMappings({
|
||||
indexTypes: ['foo', 'bar', 'baz'],
|
||||
appMappings: updatedMappings,
|
||||
indexMappings: initialMappings,
|
||||
latestMappingsVersions: {
|
||||
foo: '10.1.0',
|
||||
},
|
||||
hashToVersionMap: dummyHashToVersionMap,
|
||||
})
|
||||
).toEqual({ changedProp: 'properties.foo' });
|
||||
|
||||
expect(getUpdatedRootFieldsMock).toHaveBeenCalledTimes(1);
|
||||
expect(getUpdatedRootFieldsMock).toHaveBeenCalledWith(initialMappings);
|
||||
expect(getUpdatedTypesMock).toHaveBeenCalledTimes(1);
|
||||
expect(getUpdatedTypesMock).toHaveBeenCalledWith({
|
||||
indexTypes: ['foo', 'bar', 'baz'],
|
||||
indexMeta: initialMappings._meta,
|
||||
latestMappingsVersions: {
|
||||
foo: '10.1.0',
|
||||
},
|
||||
hashToVersionMap: dummyHashToVersionMap,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('if no root field or types have changed', () => {
|
||||
test('returns undefined', () => {
|
||||
getUpdatedRootFieldsMock.mockReturnValueOnce([]);
|
||||
getUpdatedTypesMock.mockReturnValueOnce([]);
|
||||
|
||||
expect(
|
||||
diffMappings({
|
||||
indexTypes: ['foo', 'bar', 'baz'],
|
||||
appMappings: updatedMappings,
|
||||
indexMappings: initialMappings,
|
||||
latestMappingsVersions: {
|
||||
foo: '10.1.0',
|
||||
},
|
||||
hashToVersionMap: dummyHashToVersionMap,
|
||||
})
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import type { IndexMapping, VirtualVersionMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { getUpdatedRootFields, getUpdatedTypes } from './compare_mappings';
|
||||
|
||||
/**
|
||||
* Diffs the actual vs expected mappings. The properties are compared using md5 hashes stored in _meta, because
|
||||
* actual and expected mappings *can* differ, but if the md5 hashes stored in actual._meta.migrationMappingPropertyHashes
|
||||
* match our expectations, we don't require a migration. This allows ES to tack on additional mappings that Kibana
|
||||
* doesn't know about or expect, without triggering continual migrations.
|
||||
*/
|
||||
export function diffMappings({
|
||||
indexMappings,
|
||||
appMappings,
|
||||
indexTypes,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap = {},
|
||||
}: {
|
||||
indexMappings: IndexMapping;
|
||||
appMappings: IndexMapping;
|
||||
indexTypes: string[];
|
||||
latestMappingsVersions: VirtualVersionMap;
|
||||
hashToVersionMap?: Record<string, string>;
|
||||
}) {
|
||||
if (indexMappings.dynamic !== appMappings.dynamic) {
|
||||
return { changedProp: 'dynamic' };
|
||||
} else if (
|
||||
!indexMappings._meta?.migrationMappingPropertyHashes &&
|
||||
!indexMappings._meta?.mappingVersions
|
||||
) {
|
||||
return { changedProp: '_meta' };
|
||||
} else {
|
||||
const changedProp = findChangedProp({
|
||||
indexMappings,
|
||||
indexTypes,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap,
|
||||
});
|
||||
return changedProp ? { changedProp: `properties.${changedProp}` } : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds a property that has changed its schema with respect to the mappings stored in the SO index
|
||||
* It can either be a root field or a SO type
|
||||
* @returns the name of the property (if any)
|
||||
*/
|
||||
function findChangedProp({
|
||||
indexMappings,
|
||||
indexTypes,
|
||||
hashToVersionMap,
|
||||
latestMappingsVersions,
|
||||
}: {
|
||||
indexMappings: IndexMapping;
|
||||
indexTypes: string[];
|
||||
hashToVersionMap: Record<string, string>;
|
||||
latestMappingsVersions: VirtualVersionMap;
|
||||
}): string | undefined {
|
||||
const updatedFields = getUpdatedRootFields(indexMappings);
|
||||
if (updatedFields.length) {
|
||||
return updatedFields[0];
|
||||
}
|
||||
|
||||
const updatedTypes = getUpdatedTypes({
|
||||
indexMeta: indexMappings._meta,
|
||||
indexTypes,
|
||||
latestMappingsVersions,
|
||||
hashToVersionMap,
|
||||
});
|
||||
if (updatedTypes.length) {
|
||||
return updatedTypes[0];
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
|
@ -8,5 +8,5 @@
|
|||
|
||||
export { KibanaMigrator } from './kibana_migrator';
|
||||
export type { KibanaMigratorOptions } from './kibana_migrator';
|
||||
export { buildActiveMappings, buildTypesMappings } from './core';
|
||||
export { buildActiveMappings, buildTypesMappings, getBaseMappings } from './core';
|
||||
export { DocumentMigrator } from './document_migrator';
|
||||
|
|
|
|||
|
|
@ -31,18 +31,29 @@ const migrationsConfig = {
|
|||
maxReadBatchSizeBytes: ByteSizeValue.parse('500mb'),
|
||||
} as unknown as SavedObjectsMigrationConfigType;
|
||||
|
||||
const indexTypesMap = {
|
||||
'.kibana': ['typeA', 'typeB', 'typeC'],
|
||||
'.kibana_task_manager': ['task'],
|
||||
'.kibana_cases': ['typeD', 'typeE'],
|
||||
};
|
||||
|
||||
const createInitialStateCommonParams = {
|
||||
kibanaVersion: '8.1.0',
|
||||
waitForMigrationCompletion: false,
|
||||
mustRelocateDocuments: true,
|
||||
indexTypesMap: {
|
||||
'.kibana': ['typeA', 'typeB', 'typeC'],
|
||||
'.kibana_task_manager': ['task'],
|
||||
'.kibana_cases': ['typeD', 'typeE'],
|
||||
indexTypes: ['typeA', 'typeB', 'typeC'],
|
||||
indexTypesMap,
|
||||
hashToVersionMap: {
|
||||
'typeA|someHash': '10.1.0',
|
||||
'typeB|someHash': '10.1.0',
|
||||
'typeC|someHash': '10.1.0',
|
||||
},
|
||||
targetMappings: {
|
||||
targetIndexMappings: {
|
||||
dynamic: 'strict',
|
||||
properties: { my_type: { properties: { title: { type: 'text' } } } },
|
||||
_meta: {
|
||||
indexTypesMap,
|
||||
},
|
||||
} as IndexMapping,
|
||||
coreMigrationVersionPerType: {},
|
||||
migrationVersionPerType: {},
|
||||
|
|
@ -58,6 +69,37 @@ describe('createInitialState', () => {
|
|||
|
||||
beforeEach(() => {
|
||||
typeRegistry = new SavedObjectTypeRegistry();
|
||||
typeRegistry.registerType({
|
||||
name: 'foo',
|
||||
hidden: false,
|
||||
mappings: {
|
||||
properties: {},
|
||||
},
|
||||
namespaceType: 'single',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
},
|
||||
},
|
||||
switchToModelVersionAt: '8.10.0',
|
||||
});
|
||||
typeRegistry.registerType({
|
||||
name: 'bar',
|
||||
hidden: false,
|
||||
mappings: {
|
||||
properties: {},
|
||||
},
|
||||
namespaceType: 'single',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
},
|
||||
2: {
|
||||
changes: [{ type: 'mappings_addition', addedMappings: {} }],
|
||||
},
|
||||
},
|
||||
switchToModelVersionAt: '8.10.0',
|
||||
});
|
||||
docLinks = docLinksServiceMock.createSetupContract();
|
||||
logger = mockLogger.get();
|
||||
createInitialStateParams = {
|
||||
|
|
@ -204,7 +246,17 @@ describe('createInitialState', () => {
|
|||
],
|
||||
},
|
||||
},
|
||||
"hashToVersionMap": Object {
|
||||
"typeA|someHash": "10.1.0",
|
||||
"typeB|someHash": "10.1.0",
|
||||
"typeC|someHash": "10.1.0",
|
||||
},
|
||||
"indexPrefix": ".kibana_task_manager",
|
||||
"indexTypes": Array [
|
||||
"typeA",
|
||||
"typeB",
|
||||
"typeC",
|
||||
],
|
||||
"indexTypesMap": Object {
|
||||
".kibana": Array [
|
||||
"typeA",
|
||||
|
|
@ -220,7 +272,14 @@ describe('createInitialState', () => {
|
|||
],
|
||||
},
|
||||
"kibanaVersion": "8.1.0",
|
||||
"knownTypes": Array [],
|
||||
"knownTypes": Array [
|
||||
"foo",
|
||||
"bar",
|
||||
],
|
||||
"latestMappingsVersions": Object {
|
||||
"bar": "10.2.0",
|
||||
"foo": "10.0.0",
|
||||
},
|
||||
"legacyIndex": ".kibana_task_manager",
|
||||
"logs": Array [],
|
||||
"maxBatchSize": 1000,
|
||||
|
|
@ -299,19 +358,6 @@ describe('createInitialState', () => {
|
|||
});
|
||||
|
||||
it('returns state with the correct `knownTypes`', () => {
|
||||
typeRegistry.registerType({
|
||||
name: 'foo',
|
||||
namespaceType: 'single',
|
||||
hidden: false,
|
||||
mappings: { properties: {} },
|
||||
});
|
||||
typeRegistry.registerType({
|
||||
name: 'bar',
|
||||
namespaceType: 'multiple',
|
||||
hidden: true,
|
||||
mappings: { properties: {} },
|
||||
});
|
||||
|
||||
const initialState = createInitialState({
|
||||
...createInitialStateParams,
|
||||
typeRegistry,
|
||||
|
|
@ -325,7 +371,7 @@ describe('createInitialState', () => {
|
|||
it('returns state with the correct `excludeFromUpgradeFilterHooks`', () => {
|
||||
const fooExcludeOnUpgradeHook = jest.fn();
|
||||
typeRegistry.registerType({
|
||||
name: 'foo',
|
||||
name: 'baz',
|
||||
namespaceType: 'single',
|
||||
hidden: false,
|
||||
mappings: { properties: {} },
|
||||
|
|
@ -333,7 +379,7 @@ describe('createInitialState', () => {
|
|||
});
|
||||
|
||||
const initialState = createInitialState(createInitialStateParams);
|
||||
expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ foo: fooExcludeOnUpgradeHook });
|
||||
expect(initialState.excludeFromUpgradeFilterHooks).toEqual({ baz: fooExcludeOnUpgradeHook });
|
||||
});
|
||||
|
||||
it('returns state with a preMigration script', () => {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ import * as Option from 'fp-ts/Option';
|
|||
import type { DocLinksServiceStart } from '@kbn/core-doc-links-server';
|
||||
import type { Logger } from '@kbn/logging';
|
||||
import type { ISavedObjectTypeRegistry } from '@kbn/core-saved-objects-server';
|
||||
import type {
|
||||
import {
|
||||
getLatestMappingsVirtualVersionMap,
|
||||
IndexMapping,
|
||||
IndexTypesMap,
|
||||
SavedObjectsMigrationConfigType,
|
||||
|
|
@ -28,8 +29,10 @@ export interface CreateInitialStateParams extends OutdatedDocumentsQueryParams {
|
|||
kibanaVersion: string;
|
||||
waitForMigrationCompletion: boolean;
|
||||
mustRelocateDocuments: boolean;
|
||||
indexTypes: string[];
|
||||
indexTypesMap: IndexTypesMap;
|
||||
targetMappings: IndexMapping;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
targetIndexMappings: IndexMapping;
|
||||
preMigrationScript?: string;
|
||||
indexPrefix: string;
|
||||
migrationsConfig: SavedObjectsMigrationConfigType;
|
||||
|
|
@ -39,6 +42,16 @@ export interface CreateInitialStateParams extends OutdatedDocumentsQueryParams {
|
|||
esCapabilities: ElasticsearchCapabilities;
|
||||
}
|
||||
|
||||
const TEMP_INDEX_MAPPINGS: IndexMapping = {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
type: { type: 'keyword' },
|
||||
typeMigrationVersion: {
|
||||
type: 'version',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Construct the initial state for the model
|
||||
*/
|
||||
|
|
@ -46,8 +59,10 @@ export const createInitialState = ({
|
|||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypes,
|
||||
indexTypesMap,
|
||||
targetMappings,
|
||||
hashToVersionMap,
|
||||
targetIndexMappings,
|
||||
preMigrationScript,
|
||||
coreMigrationVersionPerType,
|
||||
migrationVersionPerType,
|
||||
|
|
@ -63,16 +78,6 @@ export const createInitialState = ({
|
|||
migrationVersionPerType,
|
||||
});
|
||||
|
||||
const reindexTargetMappings: IndexMapping = {
|
||||
dynamic: false,
|
||||
properties: {
|
||||
type: { type: 'keyword' },
|
||||
typeMigrationVersion: {
|
||||
type: 'version',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const knownTypes = typeRegistry.getAllTypes().map((type) => type.name);
|
||||
const excludeFilterHooks = Object.fromEntries(
|
||||
typeRegistry
|
||||
|
|
@ -101,19 +106,13 @@ export const createInitialState = ({
|
|||
);
|
||||
}
|
||||
|
||||
const targetIndexMappings: IndexMapping = {
|
||||
...targetMappings,
|
||||
_meta: {
|
||||
...targetMappings._meta,
|
||||
indexTypesMap,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
controlState: 'INIT',
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypes,
|
||||
indexTypesMap,
|
||||
hashToVersionMap,
|
||||
indexPrefix,
|
||||
legacyIndex: indexPrefix,
|
||||
currentAlias: indexPrefix,
|
||||
|
|
@ -124,7 +123,7 @@ export const createInitialState = ({
|
|||
kibanaVersion,
|
||||
preMigrationScript: Option.fromNullable(preMigrationScript),
|
||||
targetIndexMappings,
|
||||
tempIndexMappings: reindexTargetMappings,
|
||||
tempIndexMappings: TEMP_INDEX_MAPPINGS,
|
||||
outdatedDocumentsQuery,
|
||||
retryCount: 0,
|
||||
retryDelay: 0,
|
||||
|
|
@ -138,6 +137,7 @@ export const createInitialState = ({
|
|||
logs: [],
|
||||
excludeOnUpgradeQuery: excludeUnusedTypesQuery,
|
||||
knownTypes,
|
||||
latestMappingsVersions: getLatestMappingsVirtualVersionMap(typeRegistry.getAllTypes()),
|
||||
excludeFromUpgradeFilterHooks: excludeFilterHooks,
|
||||
migrationDocLinks,
|
||||
esCapabilities,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import {
|
|||
type MigrationResult,
|
||||
SavedObjectTypeRegistry,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { KibanaMigrator } from './kibana_migrator';
|
||||
import { KibanaMigrator, type KibanaMigratorOptions } from './kibana_migrator';
|
||||
import { DocumentMigrator } from './document_migrator';
|
||||
import { ByteSizeValue } from '@kbn/config-schema';
|
||||
import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks';
|
||||
|
|
@ -235,7 +235,7 @@ describe('KibanaMigrator', () => {
|
|||
});
|
||||
});
|
||||
|
||||
const mockOptions = (algorithm: 'v2' | 'zdt' = 'v2') => {
|
||||
const mockOptions = (algorithm: 'v2' | 'zdt' = 'v2'): KibanaMigratorOptions => {
|
||||
const mockedClient = elasticsearchClientMock.createElasticsearchClient();
|
||||
(mockedClient as any).child = jest.fn().mockImplementation(() => mockedClient);
|
||||
|
||||
|
|
@ -251,6 +251,7 @@ const mockOptions = (algorithm: 'v2' | 'zdt' = 'v2') => {
|
|||
// are moved over to their new index (.my_index)
|
||||
'.my_complementary_index': ['testtype3'],
|
||||
},
|
||||
hashToVersionMap: {},
|
||||
typeRegistry: createRegistry([
|
||||
// typeRegistry depicts an updated index map:
|
||||
// .my_index: ['testtype', 'testtype3'],
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ import type {
|
|||
ElasticsearchClient,
|
||||
ElasticsearchCapabilities,
|
||||
} from '@kbn/core-elasticsearch-server';
|
||||
import {
|
||||
type SavedObjectUnsanitizedDoc,
|
||||
type ISavedObjectTypeRegistry,
|
||||
import type {
|
||||
SavedObjectUnsanitizedDoc,
|
||||
ISavedObjectTypeRegistry,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import {
|
||||
SavedObjectsSerializer,
|
||||
|
|
@ -44,6 +44,7 @@ export interface KibanaMigratorOptions {
|
|||
client: ElasticsearchClient;
|
||||
typeRegistry: ISavedObjectTypeRegistry;
|
||||
defaultIndexTypesMap: IndexTypesMap;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
soMigrationsConfig: SavedObjectsMigrationConfigType;
|
||||
kibanaIndex: string;
|
||||
kibanaVersion: string;
|
||||
|
|
@ -65,6 +66,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
private readonly mappingProperties: SavedObjectsTypeMappingDefinitions;
|
||||
private readonly typeRegistry: ISavedObjectTypeRegistry;
|
||||
private readonly defaultIndexTypesMap: IndexTypesMap;
|
||||
private readonly hashToVersionMap: Record<string, string>;
|
||||
private readonly serializer: SavedObjectsSerializer;
|
||||
private migrationResult?: Promise<MigrationResult[]>;
|
||||
private readonly status$ = new BehaviorSubject<KibanaMigratorStatus>({
|
||||
|
|
@ -87,6 +89,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
typeRegistry,
|
||||
kibanaIndex,
|
||||
defaultIndexTypesMap,
|
||||
hashToVersionMap,
|
||||
soMigrationsConfig,
|
||||
kibanaVersion,
|
||||
logger,
|
||||
|
|
@ -100,7 +103,9 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
this.soMigrationsConfig = soMigrationsConfig;
|
||||
this.typeRegistry = typeRegistry;
|
||||
this.defaultIndexTypesMap = defaultIndexTypesMap;
|
||||
this.hashToVersionMap = hashToVersionMap;
|
||||
this.serializer = new SavedObjectsSerializer(this.typeRegistry);
|
||||
// build mappings.properties for all types, all indices
|
||||
this.mappingProperties = buildTypesMappings(this.typeRegistry.getAllTypes());
|
||||
this.log = logger;
|
||||
this.kibanaVersion = kibanaVersion;
|
||||
|
|
@ -112,8 +117,8 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
});
|
||||
this.waitForMigrationCompletion = waitForMigrationCompletion;
|
||||
this.nodeRoles = nodeRoles;
|
||||
// Building the active mappings (and associated md5sums) is an expensive
|
||||
// operation so we cache the result
|
||||
// we are no longer adding _meta information to the mappings at this level
|
||||
// consumers of the exposed mappings are only accessing the 'properties' field
|
||||
this.activeMappings = buildActiveMappings(this.mappingProperties);
|
||||
this.docLinks = docLinks;
|
||||
this.esCapabilities = esCapabilities;
|
||||
|
|
@ -172,6 +177,7 @@ export class KibanaMigrator implements IKibanaMigrator {
|
|||
kibanaIndexPrefix: this.kibanaIndex,
|
||||
typeRegistry: this.typeRegistry,
|
||||
defaultIndexTypesMap: this.defaultIndexTypesMap,
|
||||
hashToVersionMap: this.hashToVersionMap,
|
||||
logger: this.log,
|
||||
documentMigrator: this.documentMigrator,
|
||||
migrationConfig: this.soMigrationsConfig,
|
||||
|
|
|
|||
|
|
@ -39,12 +39,21 @@ describe('migrationsStateActionMachine', () => {
|
|||
kibanaVersion: '7.11.0',
|
||||
waitForMigrationCompletion: false,
|
||||
mustRelocateDocuments: true,
|
||||
indexTypes: ['typeA', 'typeB', 'typeC'],
|
||||
indexTypesMap: {
|
||||
'.kibana': ['typeA', 'typeB', 'typeC'],
|
||||
'.kibana_task_manager': ['task'],
|
||||
'.kibana_cases': ['typeD', 'typeE'],
|
||||
},
|
||||
targetMappings: { properties: {} },
|
||||
hashToVersionMap: {
|
||||
'typeA|someHash': '10.1.0',
|
||||
'typeB|someHash': '10.1.0',
|
||||
'typeC|someHash': '10.1.0',
|
||||
'task|someHash': '10.1.0',
|
||||
'typeD|someHash': '10.1.0',
|
||||
'typeE|someHash': '10.1.0',
|
||||
},
|
||||
targetIndexMappings: { properties: {} },
|
||||
coreMigrationVersionPerType: {},
|
||||
migrationVersionPerType: {},
|
||||
indexPrefix: '.my-so-index',
|
||||
|
|
|
|||
|
|
@ -44,36 +44,35 @@ export function throwBadResponse(state: { controlState: string }, res: unknown):
|
|||
}
|
||||
|
||||
/**
|
||||
* Merge the _meta.migrationMappingPropertyHashes mappings of an index with
|
||||
* the given target mappings.
|
||||
* Merge the mappings._meta information of an index with the given target mappings.
|
||||
*
|
||||
* @remarks When another instance already completed a migration, the existing
|
||||
* target index might contain documents and mappings created by a plugin that
|
||||
* is disabled in the current Kibana instance performing this migration.
|
||||
* Mapping updates are commutative (deeply merged) by Elasticsearch, except
|
||||
* for the `_meta` key. By merging the `_meta.migrationMappingPropertyHashes`
|
||||
* mappings from the existing target index index into the targetMappings we
|
||||
* ensure that any `migrationPropertyHashes` for disabled plugins aren't lost.
|
||||
*
|
||||
* Right now we don't use these `migrationPropertyHashes` but it could be used
|
||||
* in the future to detect if mappings were changed. If mappings weren't
|
||||
* changed we don't need to reindex but can clone the index to save disk space.
|
||||
* for the `_meta` key. By merging the `_meta` from the existing target index
|
||||
* into the targetMappings we ensure that any versions for disabled plugins aren't lost.
|
||||
*
|
||||
* @param targetMappings
|
||||
* @param indexMappings
|
||||
*/
|
||||
export function mergeMigrationMappingPropertyHashes(
|
||||
targetMappings: IndexMapping,
|
||||
indexMappings: IndexMapping
|
||||
) {
|
||||
export function mergeMappingMeta(targetMappings: IndexMapping, indexMappings: IndexMapping) {
|
||||
const mappingVersions = {
|
||||
...indexMappings._meta?.mappingVersions,
|
||||
...targetMappings._meta?.mappingVersions,
|
||||
};
|
||||
|
||||
const migrationMappingPropertyHashes = {
|
||||
...indexMappings._meta?.migrationMappingPropertyHashes,
|
||||
...targetMappings._meta?.migrationMappingPropertyHashes,
|
||||
};
|
||||
|
||||
return {
|
||||
...targetMappings,
|
||||
_meta: {
|
||||
...targetMappings._meta,
|
||||
migrationMappingPropertyHashes: {
|
||||
...indexMappings._meta?.migrationMappingPropertyHashes,
|
||||
...targetMappings._meta?.migrationMappingPropertyHashes,
|
||||
},
|
||||
...(Object.keys(mappingVersions).length && { mappingVersions }),
|
||||
...(Object.keys(migrationMappingPropertyHashes).length && { migrationMappingPropertyHashes }),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,16 @@ describe('migrations v2 model', () => {
|
|||
],
|
||||
},
|
||||
},
|
||||
indexTypes: ['config'],
|
||||
knownTypes: ['dashboard', 'config'],
|
||||
latestMappingsVersions: {
|
||||
config: '10.3.0',
|
||||
dashboard: '10.3.0',
|
||||
},
|
||||
hashToVersionMap: {
|
||||
'config|someHash': '10.1.0',
|
||||
'dashboard|anotherHash': '10.2.0',
|
||||
},
|
||||
excludeFromUpgradeFilterHooks: {},
|
||||
migrationDocLinks: {
|
||||
resolveMigrationFailures: 'https://someurl.co/',
|
||||
|
|
@ -2602,7 +2611,7 @@ describe('migrations v2 model', () => {
|
|||
describe('reindex migration', () => {
|
||||
it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES if origin mappings did not exist', () => {
|
||||
const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({
|
||||
type: 'actual_mappings_incomplete' as const,
|
||||
type: 'index_mappings_incomplete' as const,
|
||||
});
|
||||
const newState = model(
|
||||
checkTargetMappingsState,
|
||||
|
|
@ -2616,8 +2625,8 @@ describe('migrations v2 model', () => {
|
|||
describe('compatible migration', () => {
|
||||
it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES if core fields have been updated', () => {
|
||||
const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({
|
||||
type: 'compared_mappings_changed' as const,
|
||||
updatedHashes: ['dashboard', 'lens', 'namespaces'],
|
||||
type: 'root_fields_changed' as const,
|
||||
updatedFields: ['references'],
|
||||
});
|
||||
const newState = model(
|
||||
checkTargetMappingsState,
|
||||
|
|
@ -2631,8 +2640,8 @@ describe('migrations v2 model', () => {
|
|||
|
||||
it('CHECK_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_PROPERTIES if only SO types have changed', () => {
|
||||
const res: ResponseType<'CHECK_TARGET_MAPPINGS'> = Either.left({
|
||||
type: 'compared_mappings_changed' as const,
|
||||
updatedHashes: ['dashboard', 'lens'],
|
||||
type: 'types_changed' as const,
|
||||
updatedTypes: ['dashboard', 'lens'],
|
||||
});
|
||||
const newState = model(
|
||||
checkTargetMappingsState,
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ import {
|
|||
getMigrationType,
|
||||
indexBelongsToLaterVersion,
|
||||
indexVersion,
|
||||
mergeMigrationMappingPropertyHashes,
|
||||
mergeMappingMeta,
|
||||
throwBadControlState,
|
||||
throwBadResponse,
|
||||
versionMigrationCompleted,
|
||||
|
|
@ -54,7 +54,6 @@ import {
|
|||
CLUSTER_SHARD_LIMIT_EXCEEDED_REASON,
|
||||
FATAL_REASON_REQUEST_ENTITY_TOO_LARGE,
|
||||
} from '../common/constants';
|
||||
import { getBaseMappings } from '../core';
|
||||
import { buildPickupMappingsQuery } from '../core/build_pickup_mappings_query';
|
||||
|
||||
export const model = (currentState: State, resW: ResponseType<AllActionStates>): State => {
|
||||
|
|
@ -518,7 +517,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
// in this scenario, a .kibana_X.Y.Z_001 index exists that matches the current kibana version
|
||||
// aka we are NOT upgrading to a newer version
|
||||
// we inject the source index's current mappings in the state, to check them later
|
||||
targetIndexMappings: mergeMigrationMappingPropertyHashes(
|
||||
targetIndexMappings: mergeMappingMeta(
|
||||
stateP.targetIndexMappings,
|
||||
stateP.sourceIndexMappings.value
|
||||
),
|
||||
|
|
@ -574,7 +573,7 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
controlState: 'PREPARE_COMPATIBLE_MIGRATION',
|
||||
mustRefresh:
|
||||
stateP.mustRefresh || typeof res.right.deleted === 'undefined' || res.right.deleted > 0,
|
||||
targetIndexMappings: mergeMigrationMappingPropertyHashes(
|
||||
targetIndexMappings: mergeMappingMeta(
|
||||
stateP.targetIndexMappings,
|
||||
stateP.sourceIndexMappings.value
|
||||
),
|
||||
|
|
@ -1430,14 +1429,14 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
} else if (stateP.controlState === 'CHECK_TARGET_MAPPINGS') {
|
||||
const res = resW as ResponseType<typeof stateP.controlState>;
|
||||
if (Either.isRight(res)) {
|
||||
// The md5 of ALL mappings match, so there's no need to update target mappings
|
||||
// The mappings have NOT changed, no need to pick up changes in any documents
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'CHECK_VERSION_INDEX_READY_ACTIONS',
|
||||
};
|
||||
} else {
|
||||
const left = res.left;
|
||||
if (isTypeof(left, 'actual_mappings_incomplete')) {
|
||||
if (isTypeof(left, 'index_mappings_incomplete')) {
|
||||
// reindex migration
|
||||
// some top-level properties have changed, e.g. 'dynamic' or '_meta' (see checkTargetMappings())
|
||||
// we must "pick-up" all documents on the index (by not providing a query)
|
||||
|
|
@ -1446,42 +1445,38 @@ export const model = (currentState: State, resW: ResponseType<AllActionStates>):
|
|||
controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES',
|
||||
updatedTypesQuery: Option.none,
|
||||
};
|
||||
} else if (isTypeof(left, 'compared_mappings_changed')) {
|
||||
const rootFields = Object.keys(getBaseMappings().properties);
|
||||
const updatedRootFields = left.updatedHashes.filter((field) => rootFields.includes(field));
|
||||
const updatedTypesQuery = Option.fromNullable(buildPickupMappingsQuery(left.updatedHashes));
|
||||
} else if (isTypeof(left, 'root_fields_changed')) {
|
||||
// compatible migration: some core fields have been updated
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES',
|
||||
// we must "pick-up" all documents on the index (by not providing a query)
|
||||
updatedTypesQuery: Option.none,
|
||||
logs: [
|
||||
...stateP.logs,
|
||||
{
|
||||
level: 'info',
|
||||
message: `Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: ${left.updatedFields}.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
} else if (isTypeof(left, 'types_changed')) {
|
||||
// compatible migration: some fields have been updated, and they all correspond to SO types
|
||||
const updatedTypesQuery = Option.fromNullable(buildPickupMappingsQuery(left.updatedTypes));
|
||||
|
||||
if (updatedRootFields.length) {
|
||||
// compatible migration: some core fields have been updated
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES',
|
||||
// we must "pick-up" all documents on the index (by not providing a query)
|
||||
updatedTypesQuery,
|
||||
logs: [
|
||||
...stateP.logs,
|
||||
{
|
||||
level: 'info',
|
||||
message: `Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: ${updatedRootFields}.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
} else {
|
||||
// compatible migration: some fields have been updated, and they all correspond to SO types
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES',
|
||||
// we can "pick-up" only the SO types that have changed
|
||||
updatedTypesQuery,
|
||||
logs: [
|
||||
...stateP.logs,
|
||||
{
|
||||
level: 'info',
|
||||
message: `Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings: ${left.updatedHashes}.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
...stateP,
|
||||
controlState: 'UPDATE_TARGET_MAPPINGS_PROPERTIES',
|
||||
// we can "pick-up" only the SO types that have changed
|
||||
updatedTypesQuery,
|
||||
logs: [
|
||||
...stateP.logs,
|
||||
{
|
||||
level: 'info',
|
||||
message: `Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: ${left.updatedTypes}.`,
|
||||
},
|
||||
],
|
||||
};
|
||||
} else {
|
||||
throwBadResponse(stateP, res as never);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -85,16 +85,15 @@ export const nextActionMap = (
|
|||
Actions.fetchIndices({ client, indices: [state.currentAlias, state.versionAlias] }),
|
||||
WAIT_FOR_YELLOW_SOURCE: (state: WaitForYellowSourceState) =>
|
||||
Actions.waitForIndexStatus({ client, index: state.sourceIndex.value, status: 'yellow' }),
|
||||
UPDATE_SOURCE_MAPPINGS_PROPERTIES: ({
|
||||
sourceIndex,
|
||||
sourceIndexMappings,
|
||||
targetIndexMappings,
|
||||
}: UpdateSourceMappingsPropertiesState) =>
|
||||
UPDATE_SOURCE_MAPPINGS_PROPERTIES: (state: UpdateSourceMappingsPropertiesState) =>
|
||||
Actions.updateSourceMappingsProperties({
|
||||
client,
|
||||
sourceIndex: sourceIndex.value,
|
||||
sourceMappings: sourceIndexMappings.value,
|
||||
targetMappings: targetIndexMappings,
|
||||
indexTypes: state.indexTypes,
|
||||
sourceIndex: state.sourceIndex.value,
|
||||
indexMappings: state.sourceIndexMappings.value,
|
||||
appMappings: state.targetIndexMappings,
|
||||
latestMappingsVersions: state.latestMappingsVersions,
|
||||
hashToVersionMap: state.hashToVersionMap,
|
||||
}),
|
||||
CLEANUP_UNKNOWN_AND_EXCLUDED: (state: CleanupUnknownAndExcluded) =>
|
||||
Actions.cleanupUnknownAndExcluded({
|
||||
|
|
@ -206,8 +205,11 @@ export const nextActionMap = (
|
|||
Actions.refreshIndex({ client, index: state.targetIndex }),
|
||||
CHECK_TARGET_MAPPINGS: (state: CheckTargetMappingsState) =>
|
||||
Actions.checkTargetMappings({
|
||||
actualMappings: Option.toUndefined(state.sourceIndexMappings),
|
||||
expectedMappings: state.targetIndexMappings,
|
||||
indexTypes: state.indexTypes,
|
||||
indexMappings: Option.toUndefined(state.sourceIndexMappings),
|
||||
appMappings: state.targetIndexMappings,
|
||||
latestMappingsVersions: state.latestMappingsVersions,
|
||||
hashToVersionMap: state.hashToVersionMap,
|
||||
}),
|
||||
UPDATE_TARGET_MAPPINGS_PROPERTIES: (state: UpdateTargetMappingsPropertiesState) =>
|
||||
Actions.updateAndPickupMappings({
|
||||
|
|
|
|||
|
|
@ -34,6 +34,13 @@ export const indexTypesMapMock = {
|
|||
'.complementary_index': ['testtype3'],
|
||||
};
|
||||
|
||||
export const hashToVersionMapMock = {
|
||||
'testtype|someHash': '10.1.0',
|
||||
'testtype2|anotherHash': '10.2.0',
|
||||
'testtasktype|hashesAreCool': '10.1.0',
|
||||
'testtype3|yetAnotherHash': '10.1.0',
|
||||
};
|
||||
|
||||
export const savedObjectTypeRegistryMock = createRegistry([
|
||||
// typeRegistry depicts an updated index map:
|
||||
// .my_index: ['testtype', 'testtype3'],
|
||||
|
|
|
|||
|
|
@ -18,7 +18,11 @@ import { waitGroup } from './kibana_migrator_utils';
|
|||
import { migrationStateActionMachine } from './migrations_state_action_machine';
|
||||
import { next } from './next';
|
||||
import { runResilientMigrator, type RunResilientMigratorParams } from './run_resilient_migrator';
|
||||
import { indexTypesMapMock, savedObjectTypeRegistryMock } from './run_resilient_migrator.fixtures';
|
||||
import {
|
||||
hashToVersionMapMock,
|
||||
indexTypesMapMock,
|
||||
savedObjectTypeRegistryMock,
|
||||
} from './run_resilient_migrator.fixtures';
|
||||
import type { InitState, State } from './state';
|
||||
import type { Next } from './state_action_machine';
|
||||
|
||||
|
|
@ -70,8 +74,10 @@ describe('runResilientMigrator', () => {
|
|||
kibanaVersion: options.kibanaVersion,
|
||||
waitForMigrationCompletion: options.waitForMigrationCompletion,
|
||||
mustRelocateDocuments: options.mustRelocateDocuments,
|
||||
indexTypes: options.indexTypes,
|
||||
indexTypesMap: options.indexTypesMap,
|
||||
targetMappings: options.targetMappings,
|
||||
hashToVersionMap: options.hashToVersionMap,
|
||||
targetIndexMappings: options.targetIndexMappings,
|
||||
preMigrationScript: options.preMigrationScript,
|
||||
migrationVersionPerType: options.migrationVersionPerType,
|
||||
coreMigrationVersionPerType: options.coreMigrationVersionPerType,
|
||||
|
|
@ -117,8 +123,10 @@ const mockOptions = (): RunResilientMigratorParams => {
|
|||
kibanaVersion: '8.8.0',
|
||||
waitForMigrationCompletion: false,
|
||||
mustRelocateDocuments: true,
|
||||
indexTypes: ['a', 'c'],
|
||||
indexTypesMap: indexTypesMapMock,
|
||||
targetMappings: {
|
||||
hashToVersionMap: hashToVersionMapMock,
|
||||
targetIndexMappings: {
|
||||
properties: {
|
||||
a: { type: 'keyword' },
|
||||
c: { type: 'long' },
|
||||
|
|
|
|||
|
|
@ -49,8 +49,10 @@ export interface RunResilientMigratorParams {
|
|||
kibanaVersion: string;
|
||||
waitForMigrationCompletion: boolean;
|
||||
mustRelocateDocuments: boolean;
|
||||
indexTypes: string[];
|
||||
indexTypesMap: IndexTypesMap;
|
||||
targetMappings: IndexMapping;
|
||||
targetIndexMappings: IndexMapping;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
preMigrationScript?: string;
|
||||
readyToReindex: WaitGroup<void>;
|
||||
doneReindexing: WaitGroup<void>;
|
||||
|
|
@ -76,8 +78,10 @@ export async function runResilientMigrator({
|
|||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypes,
|
||||
indexTypesMap,
|
||||
targetMappings,
|
||||
targetIndexMappings,
|
||||
hashToVersionMap,
|
||||
logger,
|
||||
preMigrationScript,
|
||||
readyToReindex,
|
||||
|
|
@ -96,8 +100,10 @@ export async function runResilientMigrator({
|
|||
kibanaVersion,
|
||||
waitForMigrationCompletion,
|
||||
mustRelocateDocuments,
|
||||
indexTypes,
|
||||
indexTypesMap,
|
||||
targetMappings,
|
||||
hashToVersionMap,
|
||||
targetIndexMappings,
|
||||
preMigrationScript,
|
||||
coreMigrationVersionPerType,
|
||||
migrationVersionPerType,
|
||||
|
|
|
|||
|
|
@ -27,7 +27,11 @@ import {
|
|||
waitGroup,
|
||||
} from './kibana_migrator_utils';
|
||||
import { runResilientMigrator } from './run_resilient_migrator';
|
||||
import { indexTypesMapMock, savedObjectTypeRegistryMock } from './run_resilient_migrator.fixtures';
|
||||
import {
|
||||
hashToVersionMapMock,
|
||||
indexTypesMapMock,
|
||||
savedObjectTypeRegistryMock,
|
||||
} from './run_resilient_migrator.fixtures';
|
||||
|
||||
jest.mock('./core', () => {
|
||||
const actual = jest.requireActual('./core');
|
||||
|
|
@ -248,6 +252,7 @@ const mockOptions = (kibanaVersion = '8.2.3'): RunV2MigrationOpts => {
|
|||
typeRegistry,
|
||||
kibanaIndexPrefix: '.my_index',
|
||||
defaultIndexTypesMap: indexTypesMapMock,
|
||||
hashToVersionMap: hashToVersionMapMock,
|
||||
migrationConfig: {
|
||||
algorithm: 'v2' as const,
|
||||
batchSize: 20,
|
||||
|
|
|
|||
|
|
@ -17,13 +17,16 @@ import type {
|
|||
ISavedObjectsSerializer,
|
||||
SavedObjectsRawDoc,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import type {
|
||||
IndexTypesMap,
|
||||
MigrationResult,
|
||||
SavedObjectsMigrationConfigType,
|
||||
SavedObjectsTypeMappingDefinitions,
|
||||
import {
|
||||
getVirtualVersionMap,
|
||||
type IndexMappingMeta,
|
||||
type IndexTypesMap,
|
||||
type MigrationResult,
|
||||
type SavedObjectsMigrationConfigType,
|
||||
type SavedObjectsTypeMappingDefinitions,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import Semver from 'semver';
|
||||
import { pick } from 'lodash';
|
||||
import type { DocumentMigrator } from './document_migrator';
|
||||
import { buildActiveMappings, createIndexMap } from './core';
|
||||
import {
|
||||
|
|
@ -43,6 +46,8 @@ export interface RunV2MigrationOpts {
|
|||
typeRegistry: ISavedObjectTypeRegistry;
|
||||
/** The map of indices => types to use as a default / baseline state */
|
||||
defaultIndexTypesMap: IndexTypesMap;
|
||||
/** A map that holds [last md5 used => modelVersion] for each of the SO types */
|
||||
hashToVersionMap: Record<string, string>;
|
||||
/** Logger to use for migration output */
|
||||
logger: Logger;
|
||||
/** The document migrator to use to convert the document */
|
||||
|
|
@ -113,6 +118,9 @@ export const runV2Migration = async (options: RunV2MigrationOpts): Promise<Migra
|
|||
// but if their SOs must be relocated to another index, we still need a migrator to do the job
|
||||
indicesWithRelocatingTypes.forEach((index) => migratorIndices.add(index));
|
||||
|
||||
// we will store model versions instead of hashes (to be FIPS compliant)
|
||||
const appVersions = getVirtualVersionMap(options.typeRegistry.getAllTypes());
|
||||
|
||||
const migrators = Array.from(migratorIndices).map((indexName, i) => {
|
||||
return {
|
||||
migrate: (): Promise<MigrationResult> => {
|
||||
|
|
@ -122,14 +130,27 @@ export const runV2Migration = async (options: RunV2MigrationOpts): Promise<Migra
|
|||
// check if this migrator's index is involved in some document redistribution
|
||||
const mustRelocateDocuments = indicesWithRelocatingTypes.includes(indexName);
|
||||
|
||||
// a migrator's index might no longer have any associated types to it
|
||||
const typeDefinitions = indexMap[indexName]?.typeMappings ?? {};
|
||||
|
||||
const indexTypes = Object.keys(typeDefinitions);
|
||||
// store only the model versions of SO types that belong to the index
|
||||
const mappingVersions = pick(appVersions, indexTypes);
|
||||
|
||||
const _meta: IndexMappingMeta = {
|
||||
indexTypesMap,
|
||||
mappingVersions,
|
||||
};
|
||||
|
||||
return runResilientMigrator({
|
||||
client: options.elasticsearchClient,
|
||||
kibanaVersion: options.kibanaVersion,
|
||||
mustRelocateDocuments,
|
||||
indexTypes,
|
||||
indexTypesMap,
|
||||
hashToVersionMap: options.hashToVersionMap,
|
||||
waitForMigrationCompletion: options.waitForMigrationCompletion,
|
||||
// a migrator's index might no longer have any associated types to it
|
||||
targetMappings: buildActiveMappings(indexMap[indexName]?.typeMappings ?? {}),
|
||||
targetIndexMappings: buildActiveMappings(typeDefinitions, _meta),
|
||||
logger: options.logger,
|
||||
preMigrationScript: indexMap[indexName]?.script,
|
||||
readyToReindex,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,11 @@ import type {
|
|||
SavedObjectsRawDoc,
|
||||
SavedObjectTypeExcludeFromUpgradeFilterHook,
|
||||
} from '@kbn/core-saved-objects-server';
|
||||
import type { IndexMapping, IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type {
|
||||
IndexMapping,
|
||||
IndexTypesMap,
|
||||
VirtualVersionMap,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import type { ElasticsearchCapabilities } from '@kbn/core-elasticsearch-server';
|
||||
import type { ControlState } from './state_action_machine';
|
||||
import type { AliasAction } from './actions';
|
||||
|
|
@ -158,6 +162,18 @@ export interface BaseState extends ControlState {
|
|||
* The list of known SO types that are registered.
|
||||
*/
|
||||
readonly knownTypes: string[];
|
||||
/**
|
||||
* Contains a list of the SO types that are currently assigned to this migrator's index
|
||||
*/
|
||||
readonly indexTypes: string[];
|
||||
/**
|
||||
* Contains information about the most recent model version where each type has been modified
|
||||
*/
|
||||
readonly latestMappingsVersions: VirtualVersionMap;
|
||||
/**
|
||||
* Contains a map holding information about [md5 => modelVersion] equivalence
|
||||
*/
|
||||
readonly hashToVersionMap: Record<string, string>;
|
||||
/**
|
||||
* All exclude filter hooks registered for types on this index. Keyed by type name.
|
||||
*/
|
||||
|
|
@ -170,14 +186,12 @@ export interface BaseState extends ControlState {
|
|||
*/
|
||||
readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects'];
|
||||
readonly waitForMigrationCompletion: boolean;
|
||||
|
||||
/**
|
||||
* This flag tells the migrator that SO documents must be redistributed,
|
||||
* i.e. stored in different system indices, compared to where they are currently stored.
|
||||
* This requires reindexing documents.
|
||||
*/
|
||||
readonly mustRelocateDocuments: boolean;
|
||||
|
||||
/**
|
||||
* This object holds a relation of all the types that are stored in each index, e.g.:
|
||||
* {
|
||||
|
|
@ -187,7 +201,6 @@ export interface BaseState extends ControlState {
|
|||
* }
|
||||
*/
|
||||
readonly indexTypesMap: IndexTypesMap;
|
||||
|
||||
/** Capabilities of the ES cluster we're using */
|
||||
readonly esCapabilities: ElasticsearchCapabilities;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,7 +116,7 @@ export const init: ModelStage<
|
|||
});
|
||||
// cloning as we may be mutating it in later stages.
|
||||
let currentIndexMeta = cloneDeep(currentMappings._meta!);
|
||||
if (currentAlgo === 'v2-compatible') {
|
||||
if (currentAlgo === 'v2-compatible' || currentAlgo === 'v2-partially-migrated') {
|
||||
currentIndexMeta = removePropertiesFromV2(currentIndexMeta);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -150,64 +150,33 @@ describe('actions', () => {
|
|||
});
|
||||
|
||||
describe('UPDATE_INDEX_MAPPINGS', () => {
|
||||
describe('when only SO types have been updated', () => {
|
||||
it('calls updateAndPickupMappings with the correct parameters', () => {
|
||||
const state: UpdateIndexMappingsState = {
|
||||
...createPostDocInitState(),
|
||||
controlState: 'UPDATE_INDEX_MAPPINGS',
|
||||
additiveMappingChanges: {
|
||||
it('calls updateAndPickupMappings with the correct parameters', () => {
|
||||
const state: UpdateIndexMappingsState = {
|
||||
...createPostDocInitState(),
|
||||
controlState: 'UPDATE_INDEX_MAPPINGS',
|
||||
additiveMappingChanges: {
|
||||
someToken: {},
|
||||
},
|
||||
};
|
||||
const action = actionMap.UPDATE_INDEX_MAPPINGS;
|
||||
|
||||
action(state);
|
||||
|
||||
expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledTimes(1);
|
||||
expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledWith({
|
||||
client: context.elasticsearchClient,
|
||||
index: state.currentIndex,
|
||||
mappings: {
|
||||
properties: {
|
||||
someToken: {},
|
||||
},
|
||||
};
|
||||
const action = actionMap.UPDATE_INDEX_MAPPINGS;
|
||||
|
||||
action(state);
|
||||
|
||||
expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledTimes(1);
|
||||
expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledWith({
|
||||
client: context.elasticsearchClient,
|
||||
index: state.currentIndex,
|
||||
mappings: {
|
||||
properties: {
|
||||
someToken: {},
|
||||
},
|
||||
},
|
||||
batchSize: context.batchSize,
|
||||
query: {
|
||||
bool: {
|
||||
should: [{ term: { type: 'someToken' } }],
|
||||
},
|
||||
batchSize: context.batchSize,
|
||||
query: {
|
||||
bool: {
|
||||
should: [{ term: { type: 'someToken' } }],
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when core properties have been updated', () => {
|
||||
it('calls updateAndPickupMappings with the correct parameters', () => {
|
||||
const state: UpdateIndexMappingsState = {
|
||||
...createPostDocInitState(),
|
||||
controlState: 'UPDATE_INDEX_MAPPINGS',
|
||||
additiveMappingChanges: {
|
||||
managed: {}, // this is a root field
|
||||
someToken: {},
|
||||
},
|
||||
};
|
||||
const action = actionMap.UPDATE_INDEX_MAPPINGS;
|
||||
|
||||
action(state);
|
||||
|
||||
expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledTimes(1);
|
||||
expect(ActionMocks.updateAndPickupMappings).toHaveBeenCalledWith({
|
||||
client: context.elasticsearchClient,
|
||||
index: state.currentIndex,
|
||||
mappings: {
|
||||
properties: {
|
||||
managed: {},
|
||||
someToken: {},
|
||||
},
|
||||
},
|
||||
batchSize: context.batchSize,
|
||||
});
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -28,22 +28,6 @@ describe('checkIndexCurrentAlgorithm', () => {
|
|||
expect(checkIndexCurrentAlgorithm(mapping)).toEqual('unknown');
|
||||
});
|
||||
|
||||
it('returns `unknown` if both v2 and zdt metas are present', () => {
|
||||
const mapping: IndexMapping = {
|
||||
properties: {},
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
foo: 'someHash',
|
||||
},
|
||||
mappingVersions: {
|
||||
foo: '8.8.0',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(checkIndexCurrentAlgorithm(mapping)).toEqual('unknown');
|
||||
});
|
||||
|
||||
it('returns `zdt` if all zdt metas are present', () => {
|
||||
const mapping: IndexMapping = {
|
||||
properties: {},
|
||||
|
|
@ -73,6 +57,22 @@ describe('checkIndexCurrentAlgorithm', () => {
|
|||
expect(checkIndexCurrentAlgorithm(mapping)).toEqual('v2-partially-migrated');
|
||||
});
|
||||
|
||||
it('returns `unknown` if if mappingVersions and v2 hashes are present', () => {
|
||||
const mapping: IndexMapping = {
|
||||
properties: {},
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
foo: 'someHash',
|
||||
},
|
||||
mappingVersions: {
|
||||
foo: '8.8.0',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(checkIndexCurrentAlgorithm(mapping)).toEqual('v2-partially-migrated');
|
||||
});
|
||||
|
||||
it('returns `v2-incompatible` if v2 hashes are present but not indexTypesMap', () => {
|
||||
const mapping: IndexMapping = {
|
||||
properties: {},
|
||||
|
|
|
|||
|
|
@ -36,19 +36,17 @@ export const checkIndexCurrentAlgorithm = (
|
|||
return 'unknown';
|
||||
}
|
||||
|
||||
const hasV2Meta = !!meta.migrationMappingPropertyHashes;
|
||||
const hasZDTMeta = !!meta.mappingVersions;
|
||||
const hasV2Meta = !!meta.migrationMappingPropertyHashes;
|
||||
|
||||
if (hasV2Meta && hasZDTMeta) {
|
||||
return 'unknown';
|
||||
if (hasZDTMeta) {
|
||||
const isFullZdt = !!meta.docVersions;
|
||||
return isFullZdt ? 'zdt' : 'v2-partially-migrated';
|
||||
}
|
||||
if (hasV2Meta) {
|
||||
const isCompatible = !!meta.indexTypesMap;
|
||||
return isCompatible ? 'v2-compatible' : 'v2-incompatible';
|
||||
}
|
||||
if (hasZDTMeta) {
|
||||
const isFullZdt = !!meta.docVersions;
|
||||
return isFullZdt ? 'zdt' : 'v2-partially-migrated';
|
||||
}
|
||||
|
||||
return 'unknown';
|
||||
};
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
},
|
||||
"include": [
|
||||
"**/*.ts",
|
||||
"src/hash_to_version_map.json"
|
||||
],
|
||||
"kbn_references": [
|
||||
"@kbn/logging",
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import {
|
|||
type SavedObjectsMigrationConfigType,
|
||||
type IKibanaMigrator,
|
||||
DEFAULT_INDEX_TYPES_MAP,
|
||||
HASH_TO_VERSION_MAP,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import {
|
||||
SavedObjectsClient,
|
||||
|
|
@ -389,6 +390,7 @@ export class SavedObjectsService
|
|||
soMigrationsConfig,
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
defaultIndexTypesMap: DEFAULT_INDEX_TYPES_MAP,
|
||||
hashToVersionMap: HASH_TO_VERSION_MAP,
|
||||
client,
|
||||
docLinks,
|
||||
waitForMigrationCompletion,
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ export const prepareModelVersionTestKit = async ({
|
|||
loggerFactory,
|
||||
kibanaIndex,
|
||||
defaultIndexTypesMap: {},
|
||||
hashToVersionMap: {},
|
||||
kibanaVersion,
|
||||
kibanaBranch,
|
||||
nodeRoles: defaultNodeRoles,
|
||||
|
|
@ -213,6 +214,7 @@ const getMigrator = async ({
|
|||
kibanaIndex,
|
||||
typeRegistry,
|
||||
defaultIndexTypesMap,
|
||||
hashToVersionMap,
|
||||
loggerFactory,
|
||||
kibanaVersion,
|
||||
kibanaBranch,
|
||||
|
|
@ -224,6 +226,7 @@ const getMigrator = async ({
|
|||
kibanaIndex: string;
|
||||
typeRegistry: ISavedObjectTypeRegistry;
|
||||
defaultIndexTypesMap: IndexTypesMap;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
loggerFactory: LoggerFactory;
|
||||
kibanaVersion: string;
|
||||
kibanaBranch: string;
|
||||
|
|
@ -250,6 +253,7 @@ const getMigrator = async ({
|
|||
kibanaIndex,
|
||||
typeRegistry,
|
||||
defaultIndexTypesMap,
|
||||
hashToVersionMap,
|
||||
soMigrationsConfig: soConfig.migration,
|
||||
kibanaVersion,
|
||||
logger: loggerFactory.get('savedobjects-service'),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file contains logic to build and diff the index mappings for a migration.
|
||||
*/
|
||||
|
||||
import crypto from 'crypto';
|
||||
import { mapValues } from 'lodash';
|
||||
import {
|
||||
getLatestMappingsVirtualVersionMap,
|
||||
HASH_TO_VERSION_MAP,
|
||||
} from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { buildTypesMappings } from '@kbn/core-saved-objects-migration-server-internal';
|
||||
import { getCurrentVersionTypeRegistry } from '../kibana_migrator_test_kit';
|
||||
|
||||
describe('transition from md5 hashes to model versions', () => {
|
||||
// this short-lived test is here to ensure no changes are introduced after the creation of the HASH_TO_VERSION_MAP
|
||||
it('ensures the hashToVersionMap does not miss any mappings changes', async () => {
|
||||
const typeRegistry = await getCurrentVersionTypeRegistry({ oss: false });
|
||||
const mappingProperties = buildTypesMappings(typeRegistry.getAllTypes());
|
||||
const hashes = md5Values(mappingProperties);
|
||||
const versions = getLatestMappingsVirtualVersionMap(typeRegistry.getAllTypes());
|
||||
|
||||
const currentHashToVersionMap = Object.entries(hashes).reduce<Record<string, string>>(
|
||||
(acc, [type, hash]) => {
|
||||
acc[`${type}|${hash}`] = versions[type];
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
expect(currentHashToVersionMap).toEqual(HASH_TO_VERSION_MAP);
|
||||
});
|
||||
});
|
||||
|
||||
// Convert an object to an md5 hash string, using a stable serialization (canonicalStringify)
|
||||
function md5Object(obj: any) {
|
||||
return crypto.createHash('md5').update(canonicalStringify(obj)).digest('hex');
|
||||
}
|
||||
|
||||
// JSON.stringify is non-canonical, meaning the same object may produce slightly
|
||||
// different JSON, depending on compiler optimizations (e.g. object keys
|
||||
// are not guaranteed to be sorted). This function consistently produces the same
|
||||
// string, if passed an object of the same shape. If the outpuf of this function
|
||||
// changes from one release to another, migrations will run, so it's important
|
||||
// that this function remains stable across releases.
|
||||
function canonicalStringify(obj: any): string {
|
||||
if (Array.isArray(obj)) {
|
||||
return `[${obj.map(canonicalStringify)}]`;
|
||||
}
|
||||
|
||||
if (!obj || typeof obj !== 'object') {
|
||||
return JSON.stringify(obj);
|
||||
}
|
||||
|
||||
const keys = Object.keys(obj);
|
||||
|
||||
// This is important for properly handling Date
|
||||
if (!keys.length) {
|
||||
return JSON.stringify(obj);
|
||||
}
|
||||
|
||||
const sortedObj = keys
|
||||
.sort((a, b) => a.localeCompare(b))
|
||||
.map((k) => `${k}: ${canonicalStringify(obj[k])}`);
|
||||
|
||||
return `{${sortedObj}}`;
|
||||
}
|
||||
|
||||
// Convert an object's values to md5 hash strings
|
||||
function md5Values(obj: any) {
|
||||
return mapValues(obj, md5Object);
|
||||
}
|
||||
|
|
@ -0,0 +1,275 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||
* Side Public License, v 1.
|
||||
*/
|
||||
|
||||
import Path from 'path';
|
||||
import type { Metadata } from '@elastic/elasticsearch/lib/api/types';
|
||||
import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
|
||||
import type { MigrationResult } from '@kbn/core-saved-objects-base-server-internal';
|
||||
import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import {
|
||||
clearLog,
|
||||
deleteSavedObjectIndices,
|
||||
getKibanaMigratorTestKit,
|
||||
readLog,
|
||||
startElasticsearch,
|
||||
} from '../kibana_migrator_test_kit';
|
||||
import { delay, createType } from '../test_utils';
|
||||
import '../jest_matchers';
|
||||
|
||||
const logFilePath = Path.join(__dirname, 'v2_md5_to_mv.test.log');
|
||||
|
||||
const SOME_TYPE = createType({
|
||||
switchToModelVersionAt: '8.10.0',
|
||||
name: 'some-type',
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
},
|
||||
},
|
||||
mappings: {
|
||||
properties: {
|
||||
field1: { type: 'text' },
|
||||
field2: { type: 'text' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const ANOTHER_TYPE = createType({
|
||||
switchToModelVersionAt: '8.10.0',
|
||||
name: 'another-type',
|
||||
modelVersions: {
|
||||
'1': {
|
||||
changes: [],
|
||||
},
|
||||
},
|
||||
mappings: {
|
||||
properties: {
|
||||
field1: { type: 'integer' },
|
||||
field2: { type: 'integer' },
|
||||
},
|
||||
},
|
||||
});
|
||||
const ANOTHER_TYPE_UPDATED = createType({
|
||||
switchToModelVersionAt: '8.10.0',
|
||||
name: 'another-type',
|
||||
modelVersions: {
|
||||
'1': {
|
||||
changes: [],
|
||||
},
|
||||
'2': {
|
||||
changes: [
|
||||
{
|
||||
type: 'mappings_addition',
|
||||
addedMappings: {
|
||||
field3: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
mappings: {
|
||||
properties: {
|
||||
field1: { type: 'integer' },
|
||||
field2: { type: 'integer' },
|
||||
field3: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const TYPE_WITHOUT_MODEL_VERSIONS = createType({
|
||||
name: 'no-mv-type',
|
||||
mappings: {
|
||||
properties: {
|
||||
title: { type: 'text' },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const SOME_TYPE_HASH = 'someLongHashThatWeCanImagineWasCalculatedUsingMd5';
|
||||
const ANOTHER_TYPE_HASH = 'differentFromTheOneAboveAsTheRelatedTypeFieldsAreIntegers';
|
||||
const A_THIRD_HASH = 'yetAnotherHashUsedByTypeWithoutModelVersions';
|
||||
const HASH_TO_VERSION_MAP: Record<string, string> = {};
|
||||
HASH_TO_VERSION_MAP[`some-type|${SOME_TYPE_HASH}`] = '10.1.0';
|
||||
// simulate that transition to modelVersion happened before 'another-type' was updated
|
||||
HASH_TO_VERSION_MAP[`another-type|${ANOTHER_TYPE_HASH}`] = '10.1.0';
|
||||
HASH_TO_VERSION_MAP[`no-mv-type|${A_THIRD_HASH}`] = '0.0.0';
|
||||
|
||||
describe('V2 algorithm', () => {
|
||||
let esServer: TestElasticsearchUtils['es'];
|
||||
let esClient: ElasticsearchClient;
|
||||
let result: MigrationResult[];
|
||||
|
||||
const getMappingMeta = async () => {
|
||||
const mapping = await esClient.indices.getMapping({ index: MAIN_SAVED_OBJECT_INDEX });
|
||||
return Object.values(mapping)[0].mappings._meta;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
// clean ES startup
|
||||
esServer = await startElasticsearch();
|
||||
});
|
||||
|
||||
describe('when started on a fresh ES deployment', () => {
|
||||
beforeAll(async () => {
|
||||
const { runMigrations, client } = await getKibanaMigratorTestKit({
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
types: [SOME_TYPE, ANOTHER_TYPE, TYPE_WITHOUT_MODEL_VERSIONS],
|
||||
logFilePath,
|
||||
});
|
||||
esClient = client;
|
||||
|
||||
// misc cleanup
|
||||
await clearLog(logFilePath);
|
||||
await deleteSavedObjectIndices(client);
|
||||
|
||||
result = await runMigrations();
|
||||
});
|
||||
|
||||
it('creates the SO indices, storing modelVersions in meta.mappingVersions', async () => {
|
||||
expect(result[0].status === 'skipped');
|
||||
expect(await getMappingMeta()).toEqual({
|
||||
indexTypesMap: {
|
||||
'.kibana': ['another-type', 'no-mv-type', 'some-type'],
|
||||
},
|
||||
mappingVersions: {
|
||||
'another-type': '10.1.0',
|
||||
'no-mv-type': '0.0.0',
|
||||
'some-type': '10.1.0',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
describe('when upgrading to a more recent version', () => {
|
||||
let indexMetaAfterMigration: Metadata | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
// start the migrator again, which will update meta with modelVersions
|
||||
const { runMigrations: restartKibana } = await getKibanaMigratorTestKit({
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
// note that we are updating 'another-type'
|
||||
types: [SOME_TYPE, ANOTHER_TYPE_UPDATED, TYPE_WITHOUT_MODEL_VERSIONS],
|
||||
hashToVersionMap: HASH_TO_VERSION_MAP,
|
||||
logFilePath,
|
||||
});
|
||||
|
||||
result = await restartKibana();
|
||||
|
||||
indexMetaAfterMigration = await getMappingMeta();
|
||||
});
|
||||
|
||||
it('performs a compatible (non-reindexing) migration', () => {
|
||||
expect(result[0].status).toEqual('patched');
|
||||
});
|
||||
|
||||
it('updates the SO indices meta.mappingVersions with the appropriate model versions', () => {
|
||||
expect(indexMetaAfterMigration?.mappingVersions).toEqual({
|
||||
'some-type': '10.1.0',
|
||||
'another-type': '10.2.0',
|
||||
'no-mv-type': '0.0.0',
|
||||
});
|
||||
});
|
||||
|
||||
it('stores a breakdown of indices => types in the meta', () => {
|
||||
expect(indexMetaAfterMigration?.indexTypesMap).toEqual({
|
||||
'.kibana': ['another-type', 'no-mv-type', 'some-type'],
|
||||
});
|
||||
});
|
||||
|
||||
it('only "picks up" the types that have changed', async () => {
|
||||
const logs = await readLog(logFilePath);
|
||||
expect(logs).toMatch(
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: another-type.'
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('when SO indices still contain md5 hashes', () => {
|
||||
let indexMetaAfterMigration: Metadata | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
const { runMigrations, client } = await getKibanaMigratorTestKit({
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
types: [SOME_TYPE, ANOTHER_TYPE, TYPE_WITHOUT_MODEL_VERSIONS],
|
||||
logFilePath,
|
||||
});
|
||||
esClient = client;
|
||||
|
||||
// misc cleanup
|
||||
await clearLog(logFilePath);
|
||||
await deleteSavedObjectIndices(client);
|
||||
|
||||
await runMigrations();
|
||||
|
||||
// we update the mappings to mimic an "md5 state"
|
||||
await client.indices.putMapping({
|
||||
index: MAIN_SAVED_OBJECT_INDEX,
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: {
|
||||
'some-type': SOME_TYPE_HASH,
|
||||
'another-type': ANOTHER_TYPE_HASH,
|
||||
'no-mv-type': A_THIRD_HASH,
|
||||
},
|
||||
},
|
||||
allow_no_indices: true,
|
||||
});
|
||||
|
||||
// we then start the migrator again, which will update meta with modelVersions
|
||||
const { runMigrations: restartKibana } = await getKibanaMigratorTestKit({
|
||||
kibanaIndex: MAIN_SAVED_OBJECT_INDEX,
|
||||
// note that we are updating 'another-type'
|
||||
types: [SOME_TYPE, ANOTHER_TYPE_UPDATED, TYPE_WITHOUT_MODEL_VERSIONS],
|
||||
hashToVersionMap: HASH_TO_VERSION_MAP,
|
||||
logFilePath,
|
||||
});
|
||||
|
||||
result = await restartKibana();
|
||||
|
||||
indexMetaAfterMigration = await getMappingMeta();
|
||||
});
|
||||
|
||||
it('performs a compatible (non-reindexing) migration', () => {
|
||||
expect(result[0].status).toEqual('patched');
|
||||
});
|
||||
|
||||
it('preserves the SO indices meta.migrationMappingPropertyHashes (although they are no longer up to date / in use)', () => {
|
||||
expect(indexMetaAfterMigration?.migrationMappingPropertyHashes).toEqual({
|
||||
'another-type': 'differentFromTheOneAboveAsTheRelatedTypeFieldsAreIntegers',
|
||||
'no-mv-type': 'yetAnotherHashUsedByTypeWithoutModelVersions',
|
||||
'some-type': 'someLongHashThatWeCanImagineWasCalculatedUsingMd5',
|
||||
});
|
||||
});
|
||||
|
||||
it('adds the mappingVersions with the current modelVersions', () => {
|
||||
expect(indexMetaAfterMigration?.mappingVersions).toEqual({
|
||||
'another-type': '10.2.0',
|
||||
'no-mv-type': '0.0.0',
|
||||
'some-type': '10.1.0',
|
||||
});
|
||||
});
|
||||
|
||||
it('stores a breakdown of indices => types in the meta', () => {
|
||||
expect(indexMetaAfterMigration?.indexTypesMap).toEqual({
|
||||
'.kibana': ['another-type', 'no-mv-type', 'some-type'],
|
||||
});
|
||||
});
|
||||
|
||||
it('only "picks up" the types that have changed', async () => {
|
||||
const logs = await readLog(logFilePath);
|
||||
expect(logs).toMatch(
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: another-type.'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await esServer?.stop();
|
||||
await delay(10);
|
||||
});
|
||||
});
|
||||
|
|
@ -138,7 +138,9 @@ describe('V2 algorithm - using model versions - upgrade without stack version in
|
|||
indexTypesMap: {
|
||||
'.kibana': ['test_mv'],
|
||||
},
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
mappingVersions: {
|
||||
test_mv: '10.2.0',
|
||||
},
|
||||
});
|
||||
|
||||
const { saved_objects: testMvDocs } = await savedObjectsRepository.find({
|
||||
|
|
|
|||
|
|
@ -206,7 +206,10 @@ describe('V2 algorithm - using model versions - stack version bump scenario', ()
|
|||
indexTypesMap: {
|
||||
'.kibana': ['test_mv', 'test_switch'],
|
||||
},
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
mappingVersions: {
|
||||
test_mv: '10.2.0',
|
||||
test_switch: '10.1.0',
|
||||
},
|
||||
});
|
||||
|
||||
const { saved_objects: testSwitchDocs } = await savedObjectsRepository.find({
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ describe('split .kibana index into multiple system indices', () => {
|
|||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
mappingVersions: expect.any(Object),
|
||||
indexTypesMap: expect.any(Object),
|
||||
},
|
||||
properties: expect.any(Object),
|
||||
|
|
@ -149,7 +149,7 @@ describe('split .kibana index into multiple system indices', () => {
|
|||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
mappingVersions: expect.any(Object),
|
||||
indexTypesMap: expect.any(Object),
|
||||
},
|
||||
properties: expect.any(Object),
|
||||
|
|
@ -164,7 +164,7 @@ describe('split .kibana index into multiple system indices', () => {
|
|||
mappings: {
|
||||
dynamic: 'strict',
|
||||
_meta: {
|
||||
migrationMappingPropertyHashes: expect.any(Object),
|
||||
mappingVersions: expect.any(Object),
|
||||
indexTypesMap: expect.any(Object),
|
||||
},
|
||||
properties: expect.any(Object),
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import type { TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server';
|
|||
import {
|
||||
clearLog,
|
||||
createBaseline,
|
||||
currentVersion,
|
||||
defaultKibanaIndex,
|
||||
defaultLogFilePath,
|
||||
getCompatibleMappingsMigrator,
|
||||
|
|
@ -20,7 +19,6 @@ import {
|
|||
} from '../kibana_migrator_test_kit';
|
||||
import '../jest_matchers';
|
||||
import { delay, parseLogFile } from '../test_utils';
|
||||
import { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal';
|
||||
|
||||
export const logFilePath = Path.join(__dirname, 'pickup_updated_types_only.test.log');
|
||||
|
||||
|
|
@ -45,7 +43,7 @@ describe('pickupUpdatedMappings', () => {
|
|||
const logs = await parseLogFile(defaultLogFilePath);
|
||||
|
||||
expect(logs).not.toContainLogEntry(
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings'
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -59,7 +57,7 @@ describe('pickupUpdatedMappings', () => {
|
|||
const logs = await parseLogFile(defaultLogFilePath);
|
||||
|
||||
expect(logs).toContainLogEntry(
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings: complex.'
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings: complex.'
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -68,21 +66,21 @@ describe('pickupUpdatedMappings', () => {
|
|||
|
||||
// we tamper the baseline mappings to simulate some root fields changes
|
||||
const baselineMappings = await client.indices.getMapping({ index: defaultKibanaIndex });
|
||||
const _meta = baselineMappings[`${defaultKibanaIndex}_${currentVersion}_001`].mappings
|
||||
._meta as IndexMappingMeta;
|
||||
_meta.migrationMappingPropertyHashes!.namespace =
|
||||
_meta.migrationMappingPropertyHashes!.namespace + '_tampered';
|
||||
await client.indices.putMapping({ index: defaultKibanaIndex, _meta });
|
||||
const properties = Object.values(baselineMappings)[0].mappings.properties!;
|
||||
(properties.references as any).properties.description = {
|
||||
type: 'text',
|
||||
};
|
||||
await client.indices.putMapping({ index: defaultKibanaIndex, properties });
|
||||
|
||||
await runMigrations();
|
||||
|
||||
const logs = await parseLogFile(defaultLogFilePath);
|
||||
|
||||
expect(logs).toContainLogEntry(
|
||||
'Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: namespace.'
|
||||
'Kibana is performing a compatible upgrade and the mappings of some root fields have been changed. For Elasticsearch to pickup these mappings, all saved objects need to be updated. Updated root fields: references.'
|
||||
);
|
||||
expect(logs).not.toContainLogEntry(
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been udpated. Kibana will update the following SO types so that ES can pickup the updated mappings'
|
||||
'Kibana is performing a compatible upgrade and NO root fields have been updated. Kibana will update the following SO types so that ES can pickup the updated mappings'
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -18,6 +18,12 @@ const defaultType: SavedObjectsType<any> = {
|
|||
name: { type: 'keyword' },
|
||||
},
|
||||
},
|
||||
modelVersions: {
|
||||
1: {
|
||||
changes: [],
|
||||
},
|
||||
},
|
||||
switchToModelVersionAt: '8.10.0',
|
||||
migrations: {},
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ export interface KibanaMigratorTestKitParams {
|
|||
settings?: Record<string, any>;
|
||||
types?: Array<SavedObjectsType<any>>;
|
||||
defaultIndexTypesMap?: IndexTypesMap;
|
||||
hashToVersionMap?: Record<string, string>;
|
||||
logFilePath?: string;
|
||||
clientWrapperFactory?: ElasticsearchClientWrapperFactory;
|
||||
}
|
||||
|
|
@ -131,6 +132,7 @@ export const getKibanaMigratorTestKit = async ({
|
|||
settings = {},
|
||||
kibanaIndex = defaultKibanaIndex,
|
||||
defaultIndexTypesMap = {}, // do NOT assume any types are stored in any index by default
|
||||
hashToVersionMap = {}, // allows testing the md5 => modelVersion transition
|
||||
kibanaVersion = currentVersion,
|
||||
kibanaBranch = currentBranch,
|
||||
types = [],
|
||||
|
|
@ -163,6 +165,7 @@ export const getKibanaMigratorTestKit = async ({
|
|||
loggerFactory,
|
||||
kibanaIndex,
|
||||
defaultIndexTypesMap,
|
||||
hashToVersionMap,
|
||||
kibanaVersion,
|
||||
kibanaBranch,
|
||||
nodeRoles,
|
||||
|
|
@ -275,6 +278,7 @@ interface GetMigratorParams {
|
|||
kibanaIndex: string;
|
||||
typeRegistry: ISavedObjectTypeRegistry;
|
||||
defaultIndexTypesMap: IndexTypesMap;
|
||||
hashToVersionMap: Record<string, string>;
|
||||
loggerFactory: LoggerFactory;
|
||||
kibanaVersion: string;
|
||||
kibanaBranch: string;
|
||||
|
|
@ -288,6 +292,7 @@ const getMigrator = async ({
|
|||
kibanaIndex,
|
||||
typeRegistry,
|
||||
defaultIndexTypesMap,
|
||||
hashToVersionMap,
|
||||
loggerFactory,
|
||||
kibanaVersion,
|
||||
kibanaBranch,
|
||||
|
|
@ -314,6 +319,7 @@ const getMigrator = async ({
|
|||
kibanaIndex,
|
||||
typeRegistry,
|
||||
defaultIndexTypesMap,
|
||||
hashToVersionMap,
|
||||
soMigrationsConfig: soConfig.migration,
|
||||
kibanaVersion,
|
||||
logger: loggerFactory.get('savedobjects-service'),
|
||||
|
|
@ -324,6 +330,17 @@ const getMigrator = async ({
|
|||
});
|
||||
};
|
||||
|
||||
export const deleteSavedObjectIndices = async (
|
||||
client: ElasticsearchClient,
|
||||
index: string[] = ALL_SAVED_OBJECT_INDICES
|
||||
) => {
|
||||
const indices = await client.indices.get({ index, allow_no_indices: true }, { ignore: [404] });
|
||||
return await client.indices.delete(
|
||||
{ index: Object.keys(indices), allow_no_indices: true },
|
||||
{ ignore: [404] }
|
||||
);
|
||||
};
|
||||
|
||||
export const getAggregatedTypesCount = async (
|
||||
client: ElasticsearchClient,
|
||||
index: string
|
||||
|
|
@ -458,11 +475,23 @@ export const getCompatibleMappingsMigrator = async ({
|
|||
...type,
|
||||
mappings: {
|
||||
properties: {
|
||||
name: { type: 'text' },
|
||||
value: { type: 'integer' },
|
||||
...type.mappings.properties,
|
||||
createdAt: { type: 'date' },
|
||||
},
|
||||
},
|
||||
modelVersions: {
|
||||
...type.modelVersions,
|
||||
2: {
|
||||
changes: [
|
||||
{
|
||||
type: 'mappings_addition',
|
||||
addedMappings: {
|
||||
createdAt: { type: 'date' },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
} else {
|
||||
return type;
|
||||
|
|
@ -486,11 +515,28 @@ export const getIncompatibleMappingsMigrator = async ({
|
|||
...type,
|
||||
mappings: {
|
||||
properties: {
|
||||
name: { type: 'keyword' },
|
||||
value: { type: 'long' },
|
||||
...type.mappings.properties,
|
||||
value: { type: 'text' }, // we're forcing an incompatible udpate (number => text)
|
||||
createdAt: { type: 'date' },
|
||||
},
|
||||
},
|
||||
modelVersions: {
|
||||
...type.modelVersions,
|
||||
2: {
|
||||
changes: [
|
||||
{
|
||||
type: 'data_removal', // not true (we're testing reindex migrations, and modelVersions do not support breaking changes)
|
||||
removedAttributePaths: ['complex.properties.value'],
|
||||
},
|
||||
{
|
||||
type: 'mappings_addition',
|
||||
addedMappings: {
|
||||
createdAt: { type: 'date' },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
} else {
|
||||
return type;
|
||||
|
|
|
|||
|
|
@ -103,8 +103,9 @@ describe('ZDT upgrades - switching from v2 algorithm', () => {
|
|||
const records = await parseLogFile(logFilePath);
|
||||
expect(records).toContainLogEntries(
|
||||
[
|
||||
'INIT: current algo check result: v2-compatible',
|
||||
'INIT -> UPDATE_INDEX_MAPPINGS',
|
||||
'INIT: current algo check result: v2-partially-migrated',
|
||||
'INIT: mapping version check result: equal',
|
||||
'INIT -> INDEX_STATE_UPDATE_DONE',
|
||||
'INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT',
|
||||
'Migration completed',
|
||||
],
|
||||
|
|
@ -117,13 +118,17 @@ describe('ZDT upgrades - switching from v2 algorithm', () => {
|
|||
it('fails and throws an explicit error', async () => {
|
||||
const { client } = await createBaseline({ kibanaVersion: '8.7.0' });
|
||||
|
||||
// even when specifying an older version, the `indexTypeMap` will be present on the index's meta,
|
||||
// so we have to manually remove it there.
|
||||
// even when specifying an older version, `indexTypeMap` and `mappingVersions` will be present on the index's meta,
|
||||
// so we have to manually remove them.
|
||||
const indices = await client.indices.get({
|
||||
index: '.kibana_8.7.0_001',
|
||||
});
|
||||
const meta = indices['.kibana_8.7.0_001'].mappings!._meta! as IndexMappingMeta;
|
||||
delete meta.indexTypesMap;
|
||||
delete meta.mappingVersions;
|
||||
meta.migrationMappingPropertyHashes = {
|
||||
sample_a: 'sampleAHash',
|
||||
};
|
||||
await client.indices.putMapping({
|
||||
index: '.kibana_8.7.0_001',
|
||||
_meta: meta,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue