github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Reindexing prepends to index name instead of appending (#30114) (#30166)

Browse source 
Similar to what is done in the Index Lifecycle Management, we prepend as to
avoid possible issues with conflicts in index patterns, templates, etc.

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>
This commit is contained in:
Tyler Smalley 2019-02-05 18:48:54 -08:00 • committed by GitHub
parent f8c7320277
commit c20bb3a0d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 171 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

48
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts
View file

@ -4,7 +4,11 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { getReindexWarnings, transformFlatSettings } from './index_settings';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
} from 'x-pack/plugins/upgrade_assistant/common/version';
import { getReindexWarnings, parseIndexName, transformFlatSettings } from './index_settings';
describe('transformFlatSettings', () => {
it('does not blow up for empty mappings', () => {
@ -49,6 +53,48 @@ describe('transformFlatSettings', () => {
});
});
describe('parseIndexName', () => {
it('parses internal indices', () => {
expect(parseIndexName('.watches').baseName).toBe('watches');
});
it('parses non-internal indices', () => {
expect(parseIndexName('myIndex').baseName).toBe('myIndex');
});
it('excludes appended v5 reindexing string from newIndexName', () => {
expect(parseIndexName('myIndex-reindexed-v5')).toEqual({
baseName: 'myIndex-reindexed-v5',
cleanBaseName: 'myIndex',
cleanIndexName: 'myIndex',
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
});
expect(parseIndexName('.myInternalIndex-reindexed-v5')).toEqual({
baseName: 'myInternalIndex-reindexed-v5',
cleanBaseName: 'myInternalIndex',
cleanIndexName: '.myInternalIndex',
newIndexName: `.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`,
});
});
it('replaces reindexed-v${PREV_MAJOR_VERSION} with reindexed-v${CURRENT_MAJOR_VERSION} in newIndexName', () => {
expect(parseIndexName(`reindexed-v${PREV_MAJOR_VERSION}-myIndex`)).toEqual({
baseName: `reindexed-v${PREV_MAJOR_VERSION}-myIndex`,
cleanBaseName: 'myIndex',
cleanIndexName: 'myIndex',
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
});
expect(parseIndexName(`.reindexed-v${PREV_MAJOR_VERSION}-myInternalIndex`)).toEqual({
baseName: `reindexed-v${PREV_MAJOR_VERSION}-myInternalIndex`,
cleanBaseName: 'myInternalIndex',
cleanIndexName: '.myInternalIndex',
newIndexName: `.reindexed-v${CURRENT_MAJOR_VERSION}-myInternalIndex`,
});
});
});
describe('getReindexWarnings', () => {
it('does not blow up for empty mappings', () => {
expect(

36
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts
View file

@ -5,9 +5,20 @@
*/
import { flow, omit } from 'lodash';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
} from 'x-pack/plugins/upgrade_assistant/common/version';
import { ReindexWarning } from '../../../common/types';
import { FlatSettings } from './types';
export interface ParsedIndexName {
cleanIndexName: string;
baseName: string;
newIndexName: string;
cleanBaseName: string;
}
/**
* Validates, and updates deprecated settings and mappings to be applied to the
* new updated index.
@ -19,6 +30,31 @@ export const transformFlatSettings = (flatSettings: FlatSettings) => {
return { settings, mappings };
};
/**
* Parses an index name
* @param indexName
*/
export const parseIndexName = (indexName: string): ParsedIndexName => {
const matches = indexName.match(/^([\.])?(.*)$/) || [];
const internal = matches[1] || '';
const baseName = matches[2];
const currentVersion = `reindexed-v${CURRENT_MAJOR_VERSION}`;
// in 5.6 the upgrade assistant appended to the index, in 6.7+ we prepend to
// avoid conflicts with index patterns/templates/etc
const reindexedMatcher = new RegExp(`(-reindexed-v5$|reindexed-v${PREV_MAJOR_VERSION}-)`, 'g');
const cleanBaseName = baseName.replace(reindexedMatcher, '');
return {
cleanIndexName: `${internal}${cleanBaseName}`,
baseName,
cleanBaseName,
newIndexName: `${internal}${currentVersion}-${cleanBaseName}`,
};
};
/**
* Returns an array of warnings that should be displayed to user before reindexing begins.
* @param flatSettings

44
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts
View file

@ -49,11 +49,11 @@ describe('ReindexActions', () => {
describe('createReindexOp', () => {
beforeEach(() => client.create.mockResolvedValue());
it(`appends -reindexed-v${CURRENT_MAJOR_VERSION} to new name`, async () => {
it(`prepends reindexed-v${CURRENT_MAJOR_VERSION} to new name`, async () => {
await actions.createReindexOp('myIndex');
expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, {
indexName: 'myIndex',
newIndexName: `myIndex-reindexed-v${CURRENT_MAJOR_VERSION}`,
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,
@ -64,11 +64,43 @@ describe('ReindexActions', () => {
});
});
it(`replaces -reindexed-v${PREV_MAJOR_VERSION} with -reindexed-v${CURRENT_MAJOR_VERSION}`, async () => {
await actions.createReindexOp(`myIndex-reindexed-v${PREV_MAJOR_VERSION}`);
it(`prepends reindexed-v${CURRENT_MAJOR_VERSION} to new name, preserving leading period`, async () => {
await actions.createReindexOp('.internalIndex');
expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, {
indexName: `myIndex-reindexed-v${PREV_MAJOR_VERSION}`,
newIndexName: `myIndex-reindexed-v${CURRENT_MAJOR_VERSION}`,
indexName: '.internalIndex',
newIndexName: `.reindexed-v${CURRENT_MAJOR_VERSION}-internalIndex`,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,
reindexTaskId: null,
reindexTaskPercComplete: null,
errorMessage: null,
runningReindexCount: null,
});
});
// in v5.6, the upgrade assistant appended to the index name instead of prepending
it(`prepends reindexed-v${CURRENT_MAJOR_VERSION}- and removes reindex appended in v5`, async () => {
const indexName = 'myIndex-reindexed-v5';
await actions.createReindexOp(indexName);
expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, {
indexName,
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,
reindexTaskId: null,
reindexTaskPercComplete: null,
errorMessage: null,
runningReindexCount: null,
});
});
it(`replaces reindexed-v${PREV_MAJOR_VERSION} with reindexed-v${CURRENT_MAJOR_VERSION}`, async () => {
await actions.createReindexOp(`reindexed-v${PREV_MAJOR_VERSION}-myIndex`);
expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, {
indexName: `reindexed-v${PREV_MAJOR_VERSION}-myIndex`,
newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,

23
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts
View file

@ -11,10 +11,6 @@ import {
FindResponse,
SavedObjectsClient,
} from 'src/server/saved_objects/service/saved_objects_client';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
} from 'x-pack/plugins/upgrade_assistant/common/version';
import {
IndexGroup,
REINDEX_OP_TYPE,
@ -23,6 +19,7 @@ import {
ReindexStatus,
ReindexStep,
} from '../../../common/types';
import { parseIndexName } from './index_settings';
import { FlatSettings } from './types';
// TODO: base on elasticsearch.requestTimeout?
@ -120,22 +117,6 @@ export const reindexActionsFactory = (
callCluster: CallCluster
): ReindexActions => {
// ----- Internal functions
/**
* Generates a new index name for the new index. Iterates until it finds an index
* that doesn't already exist.
* @param indexName
*/
const getNewIndexName = (indexName: string) => {
const prevVersionSuffix = `-reindexed-v${PREV_MAJOR_VERSION}`;
const currentVersionSuffix = `-reindexed-v${CURRENT_MAJOR_VERSION}`;
if (indexName.endsWith(prevVersionSuffix)) {
return indexName.replace(new RegExp(`${prevVersionSuffix}$`), currentVersionSuffix);
} else {
return `${indexName}${currentVersionSuffix}`;
}
};
const isLocked = (reindexOp: ReindexSavedObject) => {
if (reindexOp.attributes.locked) {
const now = moment();
@ -176,7 +157,7 @@ export const reindexActionsFactory = (
async createReindexOp(indexName: string) {
return client.create<ReindexOperation>(REINDEX_OP_TYPE, {
indexName,
newIndexName: getNewIndexName(indexName),
newIndexName: parseIndexName(indexName).newIndexName,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,

40
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts
View file

@ -5,6 +5,10 @@
*/
import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
} from 'x-pack/plugins/upgrade_assistant/common/version';
import {
IndexGroup,
ReindexOperation,
@ -84,7 +88,7 @@ describe('reindexService', () => {
cluster: ['manage'],
index: [
{
names: [`anIndex*`],
names: ['anIndex', `reindexed-v${CURRENT_MAJOR_VERSION}-anIndex`],
privileges: ['all'],
},
{
@ -107,7 +111,37 @@ describe('reindexService', () => {
cluster: ['manage', 'manage_ml'],
index: [
{
names: [`.ml-anomalies*`],
names: ['.ml-anomalies', `.reindexed-v${CURRENT_MAJOR_VERSION}-ml-anomalies`],
privileges: ['all'],
},
{
names: ['.tasks'],
privileges: ['read', 'delete'],
},
],
},
});
});
it('includes checking for permissions on the baseName which could be an alias', async () => {
callCluster.mockResolvedValueOnce({ has_all_requested: true });
const hasRequired = await service.hasRequiredPrivileges(
`reindexed-v${PREV_MAJOR_VERSION}-anIndex`
);
expect(hasRequired).toBe(true);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_security/user/_has_privileges',
method: 'POST',
body: {
cluster: ['manage'],
index: [
{
names: [
`reindexed-v${PREV_MAJOR_VERSION}-anIndex`,
`reindexed-v${CURRENT_MAJOR_VERSION}-anIndex`,
'anIndex',
],
privileges: ['all'],
},
{
@ -130,7 +164,7 @@ describe('reindexService', () => {
cluster: ['manage', 'manage_watcher'],
index: [
{
names: [`.watches*`],
names: ['.watches', `.reindexed-v${CURRENT_MAJOR_VERSION}-watches`],
privileges: ['all'],
},
{

13
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts
View file

@ -15,7 +15,7 @@ import {
ReindexStep,
ReindexWarning,
} from '../../../common/types';
import { getReindexWarnings, transformFlatSettings } from './index_settings';
import { getReindexWarnings, parseIndexName, transformFlatSettings } from './index_settings';
import { ReindexActions } from './reindex_actions';
const VERSION_REGEX = new RegExp(/^([1-9]+)\.([0-9]+)\.([0-9]+)/);
@ -440,12 +440,21 @@ export const reindexServiceFactory = (
return true;
}
const index = parseIndexName(indexName);
const names = [indexName, index.newIndexName];
// if we have re-indexed this in the past, there will be an
// underlying alias we will also need to update.
if (index.cleanIndexName !== indexName) {
names.push(index.cleanIndexName);
}
// Otherwise, query for required privileges for this index.
const body = {
cluster: ['manage'],
index: [
{
names: [`${indexName}*`],
names,
privileges: ['all'],
},
{