mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[UA] Use new _create_from ES API (#207114)
This commit is contained in:
Alejandro Fernández Haro
GitHub
parent
35794a00af
commit
fb8a17ba04
5 changed files with 42 additions and 217 deletions
|
|
@ -8,124 +8,11 @@
|
|||
import { versionService } from '../version';
|
||||
import { getMockVersionInfo } from '../__fixtures__/version';
|
||||
|
||||
import {
|
||||
generateNewIndexName,
|
||||
getReindexWarnings,
|
||||
sourceNameForIndex,
|
||||
transformFlatSettings,
|
||||
} from './index_settings';
|
||||
import { generateNewIndexName, getReindexWarnings, sourceNameForIndex } from './index_settings';
|
||||
|
||||
const { currentMajor, prevMajor } = getMockVersionInfo();
|
||||
|
||||
describe('transformFlatSettings', () => {
|
||||
it('does not blow up for empty mappings', () => {
|
||||
expect(
|
||||
transformFlatSettings({
|
||||
settings: {},
|
||||
mappings: {},
|
||||
})
|
||||
).toEqual({
|
||||
settings: {},
|
||||
mappings: {},
|
||||
});
|
||||
});
|
||||
|
||||
it('removes settings that cannot be set on a new index', () => {
|
||||
expect(
|
||||
transformFlatSettings({
|
||||
settings: {
|
||||
// Settings that should get preserved
|
||||
'index.number_of_replicas': '1',
|
||||
'index.number_of_shards': '5',
|
||||
|
||||
// Blacklisted settings
|
||||
'index.allocation.existing_shards_allocator': 'gateway_allocator',
|
||||
'index.blocks.write': 'true',
|
||||
'index.creation_date': '1547052614626',
|
||||
'index.frozen': 'true',
|
||||
'index.history.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae9279234gh',
|
||||
'index.merge.enabled': 'true',
|
||||
'index.provided_name': 'test1',
|
||||
'index.resize.source.name': 'resizeName',
|
||||
'index.resize.source.uuid': 'k34b9149a-00ee-42d9-8ca1-85ae9279234zs',
|
||||
'index.routing.allocation.initial_recovery._id': '1',
|
||||
'index.search.throttled': 'true',
|
||||
'index.source_only': 'true',
|
||||
'index.shrink.source.name': 'shrinkSourceName',
|
||||
'index.shrink.source.uuid': 'q34b9149a-00ee-42d9-8ca1-85ae234324df',
|
||||
'index.store.snapshot.repository_name': 'repoName',
|
||||
'index.store.snapshot.snapshot_name': 'snapshotName',
|
||||
'index.store.snapshot.snapshot_uuid': 'f345c9149a-00ee-42d9-8ca1-85ae234324df',
|
||||
'index.store.snapshot.index_name': 'snapshotIndexName',
|
||||
'index.store.snapshot.index_uuid': 'h764f9149a-00ee-42d9-8ca1-85ae234324af',
|
||||
'index.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae927924bf',
|
||||
'index.verified_before_close': 'true',
|
||||
'index.version.created': '123123',
|
||||
'index.version.upgraded': '123123',
|
||||
'index.mapper.dynamic': 'true',
|
||||
|
||||
// Deprecated settings
|
||||
'index.force_memory_term_dictionary': '1024',
|
||||
'index.max_adjacency_matrix_filters': 'true',
|
||||
'index.soft_deletes.enabled': 'true',
|
||||
},
|
||||
mappings: {},
|
||||
})
|
||||
).toEqual({
|
||||
settings: {
|
||||
'index.number_of_replicas': '1',
|
||||
'index.number_of_shards': '5',
|
||||
},
|
||||
mappings: {},
|
||||
});
|
||||
});
|
||||
|
||||
it('removes index.translog.retention.size if soft deletes is enabled', () => {
|
||||
expect(
|
||||
transformFlatSettings({
|
||||
settings: {
|
||||
// Settings that should get preserved
|
||||
'index.number_of_replicas': '1',
|
||||
'index.number_of_shards': '5',
|
||||
|
||||
// Deprecated settings
|
||||
'index.soft_deletes.enabled': 'true',
|
||||
'index.translog.retention.size': '5b',
|
||||
},
|
||||
mappings: {},
|
||||
})
|
||||
).toEqual({
|
||||
settings: {
|
||||
'index.number_of_replicas': '1',
|
||||
'index.number_of_shards': '5',
|
||||
},
|
||||
mappings: {},
|
||||
});
|
||||
});
|
||||
|
||||
it('removes index.translog.retention.age if soft deletes is enabled', () => {
|
||||
expect(
|
||||
transformFlatSettings({
|
||||
settings: {
|
||||
// Settings that should get preserved
|
||||
'index.number_of_replicas': '1',
|
||||
'index.number_of_shards': '5',
|
||||
|
||||
// Deprecated settings
|
||||
'index.soft_deletes.enabled': 'true',
|
||||
'index.translog.retention.age': '5d',
|
||||
},
|
||||
mappings: {},
|
||||
})
|
||||
).toEqual({
|
||||
settings: {
|
||||
'index.number_of_replicas': '1',
|
||||
'index.number_of_shards': '5',
|
||||
},
|
||||
mappings: {},
|
||||
});
|
||||
});
|
||||
|
||||
describe('index settings', () => {
|
||||
describe('sourceNameForIndex', () => {
|
||||
beforeEach(() => {
|
||||
versionService.setup('8.0.0');
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { flow, omit } from 'lodash';
|
||||
import { ReindexWarning } from '../../../common/types';
|
||||
import { versionService } from '../version';
|
||||
import { FlatSettings } from './types';
|
||||
|
|
@ -27,17 +26,6 @@ const deprecatedSettings = [
|
|||
'index.soft_deletes.enabled',
|
||||
];
|
||||
|
||||
/**
|
||||
* Validates, and updates deprecated settings and mappings to be applied to the
|
||||
* new updated index.
|
||||
*/
|
||||
export const transformFlatSettings = (flatSettings: FlatSettings) => {
|
||||
const settings = transformSettings(flatSettings.settings);
|
||||
const mappings = transformMappings(flatSettings.mappings);
|
||||
|
||||
return { settings, mappings };
|
||||
};
|
||||
|
||||
/**
|
||||
* Provides the assumed source of the index name stripping any prefixing
|
||||
* introduced by the upgrade assistant
|
||||
|
|
@ -125,63 +113,3 @@ export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[]
|
|||
|
||||
return warnings;
|
||||
};
|
||||
|
||||
const removeUnsettableSettings = (settings: FlatSettings['settings']) =>
|
||||
omit(settings, [
|
||||
// Private ES settings
|
||||
'index.allocation.existing_shards_allocator',
|
||||
'index.blocks.write',
|
||||
'index.creation_date',
|
||||
'index.frozen',
|
||||
'index.history.uuid',
|
||||
'index.merge.enabled',
|
||||
'index.provided_name',
|
||||
'index.resize.source.name',
|
||||
'index.resize.source.uuid',
|
||||
'index.routing.allocation.initial_recovery._id',
|
||||
'index.search.throttled',
|
||||
'index.source_only',
|
||||
'index.shrink.source.name',
|
||||
'index.shrink.source.uuid',
|
||||
'index.store.snapshot.repository_name',
|
||||
'index.store.snapshot.snapshot_name',
|
||||
'index.store.snapshot.snapshot_uuid',
|
||||
'index.store.snapshot.index_name',
|
||||
'index.store.snapshot.index_uuid',
|
||||
'index.uuid',
|
||||
'index.verified_before_close',
|
||||
'index.version.created',
|
||||
|
||||
// Ignored since 6.x and forbidden in 7.x
|
||||
'index.mapper.dynamic',
|
||||
|
||||
// Deprecated in 9.0
|
||||
'index.version.upgraded',
|
||||
]);
|
||||
|
||||
const removeDeprecatedSettings = (settings: FlatSettings['settings']) => {
|
||||
const updatedSettings = { ...settings };
|
||||
|
||||
// Translog settings are only marked as deprecated if soft deletes is enabled
|
||||
if (updatedSettings['index.soft_deletes.enabled'] === 'true') {
|
||||
if (updatedSettings['index.translog.retention.size']) {
|
||||
delete updatedSettings['index.translog.retention.size'];
|
||||
}
|
||||
|
||||
// @ts-expect-error @elastic/elasticsearch doesn't declare such a setting
|
||||
if (settings['index.translog.retention.age']) {
|
||||
delete updatedSettings['index.translog.retention.age'];
|
||||
}
|
||||
}
|
||||
|
||||
return omit(updatedSettings, deprecatedSettings);
|
||||
};
|
||||
|
||||
// Use `flow` to pipe the settings through each function.
|
||||
const transformSettings = flow(removeUnsettableSettings, removeDeprecatedSettings);
|
||||
|
||||
const updateFixableMappings = (mappings: FlatSettings['mappings']) => {
|
||||
return mappings;
|
||||
};
|
||||
|
||||
const transformMappings = flow(updateFixableMappings);
|
||||
|
|
|
|||
|
|
@ -457,19 +457,21 @@ describe('reindexService', () => {
|
|||
// The more intricate details of how the settings are chosen are test separately.
|
||||
it('creates new index with settings and mappings and updates lastCompletedStep', async () => {
|
||||
actions.getFlatSettings.mockResolvedValueOnce(settingsMappings);
|
||||
clusterClient.asCurrentUser.indices.create.mockResponse(
|
||||
// @ts-expect-error not full interface
|
||||
{ acknowledged: true }
|
||||
);
|
||||
clusterClient.asCurrentUser.transport.request.mockResolvedValueOnce({ acknowledged: true });
|
||||
const updatedOp = await service.processNextStep(reindexOp);
|
||||
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.newIndexCreated);
|
||||
expect(clusterClient.asCurrentUser.indices.create).toHaveBeenCalledWith({
|
||||
index: 'myIndex-reindex-0',
|
||||
// index.blocks.write should be removed from the settings for the new index.
|
||||
// index.number_of_replicas and index.refresh_interval are stored to be set at a later stage.
|
||||
// Setting to 0 and -1, respectively, right now.
|
||||
settings: { 'index.number_of_replicas': 0, 'index.refresh_interval': -1 },
|
||||
mappings: settingsMappings.mappings,
|
||||
expect(clusterClient.asCurrentUser.transport.request).toHaveBeenCalledWith({
|
||||
method: 'POST',
|
||||
path: `_create_from/myIndex/myIndex-reindex-0`,
|
||||
body: {
|
||||
settings_override: {
|
||||
'index.blocks.read_only': null,
|
||||
'index.blocks.read_only_allow_delete': null,
|
||||
'index.blocks.write': null,
|
||||
'index.number_of_replicas': 0,
|
||||
'index.refresh_interval': -1,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -19,12 +19,7 @@ import {
|
|||
|
||||
import { esIndicesStateCheck } from '../es_indices_state_check';
|
||||
|
||||
import {
|
||||
generateNewIndexName,
|
||||
getReindexWarnings,
|
||||
sourceNameForIndex,
|
||||
transformFlatSettings,
|
||||
} from './index_settings';
|
||||
import { generateNewIndexName, getReindexWarnings, sourceNameForIndex } from './index_settings';
|
||||
|
||||
import { ReindexActions } from './reindex_actions';
|
||||
|
||||
|
|
@ -190,7 +185,7 @@ export const reindexServiceFactory = (
|
|||
throw error.indexNotFound(`Index ${indexName} does not exist.`);
|
||||
}
|
||||
|
||||
const { settings, mappings } = transformFlatSettings(flatSettings);
|
||||
const { settings = {} } = flatSettings;
|
||||
|
||||
// Backup the current settings to restore them after the reindex
|
||||
// https://github.com/elastic/kibana/issues/201605
|
||||
|
|
@ -201,16 +196,31 @@ export const reindexServiceFactory = (
|
|||
|
||||
let createIndex;
|
||||
try {
|
||||
createIndex = await esClient.indices.create({
|
||||
index: newIndexName,
|
||||
settings: {
|
||||
...settings,
|
||||
// Reindexing optimizations
|
||||
'index.number_of_replicas': 0,
|
||||
'index.refresh_interval': -1,
|
||||
createIndex = await esClient.transport.request<{ acknowledged: boolean }>({
|
||||
method: 'POST',
|
||||
path: `_create_from/${indexName}/${newIndexName}`,
|
||||
body: {
|
||||
// Settings overrides copied from ES datastream reindex logic
|
||||
// https://github.com/elastic/elasticsearch/blob/9c0709f386fee4154e930cb61a02868adebe8572/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java#L195-L210
|
||||
settings_override: {
|
||||
// Remove read-only settings if they exist
|
||||
'index.blocks.read_only': null,
|
||||
'index.blocks.read_only_allow_delete': null,
|
||||
'index.blocks.write': null,
|
||||
// Reindexing optimizations
|
||||
'index.number_of_replicas': 0,
|
||||
'index.refresh_interval': -1,
|
||||
},
|
||||
},
|
||||
mappings,
|
||||
});
|
||||
/**
|
||||
* Response is expected to be:
|
||||
* {
|
||||
* "acknowledged": true,
|
||||
* "shards_acknowledged": true,
|
||||
* "index": "test-copy"
|
||||
* }
|
||||
*/
|
||||
} catch (err) {
|
||||
// If for any reason the new index name generated by the `generateNewIndexName` already
|
||||
// exists (this could happen if kibana is restarted during reindexing), we can just go
|
||||
|
|
|
|||
|
|
@ -48,10 +48,8 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
try {
|
||||
await es.indices.create({
|
||||
index: indexName,
|
||||
body: {
|
||||
settings: {
|
||||
index: indexSettings,
|
||||
},
|
||||
settings: {
|
||||
index: indexSettings,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue