github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add reindex warning for mapping type changes (#32081) (#33088)

Browse source
This commit is contained in:
Josh Dover 2019-03-13 01:34:35 -05:00 • committed by Tyler Smalley
parent 43352cb07f
commit b68ba34bec
11 changed files with 122 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

5
x-pack/plugins/upgrade_assistant/common/types.ts
View file

@ -49,11 +49,12 @@ export type ReindexSavedObject = SavedObject<ReindexOperation>;
export enum ReindexWarning {
// 6.0 -> 7.0 warnings, now unused
allField = 0,
booleanFields = 1,
allField,
booleanFields,
// 7.0 -> 8.0 warnings
apmReindex,
customTypeName,
}
export enum IndexGroup {

75
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap
View file

@ -29,92 +29,67 @@ exports[`WarningsFlyoutStep renders 1`] = `
<WarningCheckbox
checkedIds={
Object {
"reindexWarning-0": false,
"reindexWarning-1": false,
"reindexWarning-2": false,
"reindexWarning-3": false,
}
}
description={
<FormattedMessage
defaultMessage="The {allField} meta field is no longer supported in 7.0. Reindexing removes
the {allField} field in the new index. Ensure that no application code or scripts reply on
this field."
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningDetail"
defaultMessage="Mapping types are no longer supported in 8.x. This index mapping does not use the
default type name, {defaultType}, and will be updated when reindexed. Ensure no application code
or scripts rely on a different type."
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningDetail"
values={
Object {
"allField": <EuiCode>
_all
"defaultType": <EuiCode>
_doc
</EuiCode>,
}
}
/>
}
documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html"
label={
<FormattedMessage
defaultMessage="{allField} will be removed"
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningTitle"
defaultMessage="Mapping type will be changed to {defaultType}"
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningTitle"
values={
Object {
"allField": <EuiCode>
_all
"defaultType": <EuiCode>
_doc
</EuiCode>,
}
}
/>
}
onChange={[Function]}
warning={0}
warning={3}
/>
<WarningCheckbox
checkedIds={
Object {
"reindexWarning-0": false,
"reindexWarning-1": false,
"reindexWarning-2": false,
"reindexWarning-3": false,
}
}
description={
<FormattedMessage
defaultMessage="If a document contain a boolean field that is neither {true} or {false}
(for example, {yes}, {on}, {one}), reindexing converts these fields to {true} or {false}.
Ensure that no application code or scripts rely on boolean fields in the deprecated format."
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningDetail"
values={
Object {
"false": <EuiCode>
false
</EuiCode>,
"on": <EuiCode>
"on"
</EuiCode>,
"one": <EuiCode>
1
</EuiCode>,
"true": <EuiCode>
true
</EuiCode>,
"yes": <EuiCode>
"yes"
</EuiCode>,
}
}
defaultMessage="Starting in version 7.0.0, APM data will be represented in the Elastic Common Schema.
Historical APM data will not visible until it's reindexed."
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningDetail"
values={Object {}}
/>
}
documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_field"
documentationUrl="https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html"
label={
<FormattedMessage
defaultMessage="Boolean data in {_source} might change"
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningTitle"
values={
Object {
"_source": <EuiCode>
_source
</EuiCode>,
}
}
defaultMessage="This index will be converted to ECS format"
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningTitle"
values={Object {}}
/>
}
onChange={[Function]}
warning={1}
warning={2}
/>
</EuiFlyoutBody>
<EuiFlyoutFooter>

6
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx
View file

@ -14,7 +14,7 @@ import { idForWarning, WarningsFlyoutStep } from './warnings_step';
describe('WarningsFlyoutStep', () => {
const defaultProps = {
advanceNextStep: jest.fn(),
warnings: [ReindexWarning.allField, ReindexWarning.booleanFields],
warnings: [ReindexWarning.customTypeName, ReindexWarning.apmReindex],
closeFlyout: jest.fn(),
};
@ -33,11 +33,11 @@ describe('WarningsFlyoutStep', () => {
button.simulate('click');
expect(defaultProps.advanceNextStep).not.toHaveBeenCalled();
wrapper.find(`input#${idForWarning(ReindexWarning.allField)}`).simulate('change');
wrapper.find(`input#${idForWarning(ReindexWarning.apmReindex)}`).simulate('change');
button.simulate('click');
expect(defaultProps.advanceNextStep).not.toHaveBeenCalled();
wrapper.find(`input#${idForWarning(ReindexWarning.booleanFields)}`).simulate('change');
wrapper.find(`input#${idForWarning(ReindexWarning.customTypeName)}`).simulate('change');
button.simulate('click');
expect(defaultProps.advanceNextStep).toHaveBeenCalled();
});

53
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx
View file

@ -123,32 +123,32 @@ export class WarningsFlyoutStep extends React.Component<
<EuiSpacer />
{warnings.includes(ReindexWarning.allField) && (
{warnings.includes(ReindexWarning.customTypeName) && (
<WarningCheckbox
checkedIds={checkedIds}
onChange={this.onChange}
warning={ReindexWarning.allField}
warning={ReindexWarning.customTypeName}
label={
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningTitle"
defaultMessage="{allField} will be removed"
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningTitle"
defaultMessage="Mapping type will be changed to {defaultType}"
values={{
allField: <EuiCode>_all</EuiCode>,
defaultType: <EuiCode>_doc</EuiCode>,
}}
/>
}
description={
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningDetail"
defaultMessage="The {allField} meta field is no longer supported in 7.0. Reindexing removes
the {allField} field in the new index. Ensure that no application code or scripts reply on
this field."
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningDetail"
defaultMessage="Mapping types are no longer supported in 8.x. This index mapping does not use the
default type name, {defaultType}, and will be updated when reindexed. Ensure no application code
or scripts rely on a different type."
values={{
allField: <EuiCode>_all</EuiCode>,
defaultType: <EuiCode>_doc</EuiCode>,
}}
/>
}
documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html"
/>
)}
@ -173,37 +173,6 @@ export class WarningsFlyoutStep extends React.Component<
documentationUrl="https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html"
/>
)}
{warnings.includes(ReindexWarning.booleanFields) && (
<WarningCheckbox
checkedIds={checkedIds}
onChange={this.onChange}
warning={ReindexWarning.booleanFields}
label={
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningTitle"
defaultMessage="Boolean data in {_source} might change"
values={{ _source: <EuiCode>_source</EuiCode> }}
/>
}
description={
<FormattedMessage
id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningDetail"
defaultMessage="If a document contain a boolean field that is neither {true} or {false}
(for example, {yes}, {on}, {one}), reindexing converts these fields to {true} or {false}.
Ensure that no application code or scripts rely on boolean fields in the deprecated format."
values={{
true: <EuiCode>true</EuiCode>,
false: <EuiCode>false</EuiCode>,
yes: <EuiCode>"yes"</EuiCode>,
on: <EuiCode>"on"</EuiCode>,
one: <EuiCode>1</EuiCode>,
}}
/>
}
documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_field"
/>
)}
</EuiFlyoutBody>
<EuiFlyoutFooter>
<EuiFlexGroup justifyContent="spaceBetween">

19
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts
View file

@ -4,6 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { ReindexWarning } from 'x-pack/plugins/upgrade_assistant/common/types';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
@ -121,4 +122,22 @@ describe('getReindexWarnings', () => {
})
).toEqual([]);
});
it('returns customTypeName for non-_doc mapping types', () => {
expect(
getReindexWarnings({
settings: {},
mappings: { doc: {} },
})
).toEqual([ReindexWarning.customTypeName]);
});
it('does not return customTypeName for _doc mapping types', () => {
expect(
getReindexWarnings({
settings: {},
mappings: { _doc: {} },
})
).toEqual([]);
});
});

28
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts
View file

@ -11,7 +11,9 @@ import {
} from 'x-pack/plugins/upgrade_assistant/common/version';
import { ReindexWarning } from '../../../common/types';
import { isLegacyApmIndex } from '../apm';
import { FlatSettings } from './types';
import { FlatSettings, FlatSettingsWithTypeName } from './types';
export const DEFAULT_TYPE_NAME = '_doc';
export interface ParsedIndexName {
cleanIndexName: string;
@ -74,19 +76,33 @@ export const generateNewIndexName = (indexName: string): string => {
* @param flatSettings
*/
export const getReindexWarnings = (
flatSettings: FlatSettings,
flatSettings: FlatSettingsWithTypeName,
apmIndexPatterns: string[] = []
): ReindexWarning[] => {
const indexName = flatSettings.settings['index.provided_name'];
const apmReindexWarning = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings);
const typeName = Object.getOwnPropertyNames(flatSettings.mappings)[0];
const apmReindexWarning = isLegacyApmIndex(
indexName,
apmIndexPatterns,
flatSettings.mappings[typeName]
);
const typeNameWarning = usesCustomTypeName(flatSettings);
const warnings = [[ReindexWarning.apmReindex, apmReindexWarning]] as Array<
[ReindexWarning, boolean]
>;
const warnings = [
[ReindexWarning.apmReindex, apmReindexWarning],
[ReindexWarning.customTypeName, typeNameWarning],
] as Array<[ReindexWarning, boolean]>;
return warnings.filter(([_, applies]) => applies).map(([warning, _]) => warning);
};
const usesCustomTypeName = (flatSettings: FlatSettingsWithTypeName) => {
// In 7+ it's not possible to have more than one type anyways, so always grab the first
// (and only) key.
const typeName = Object.getOwnPropertyNames(flatSettings.mappings)[0];
return typeName && typeName !== DEFAULT_TYPE_NAME;
};
const removeUnsettableSettings = (settings: FlatSettings['settings']) =>
omit(settings, [
'index.uuid',

20
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts
View file

@ -20,7 +20,7 @@ import {
ReindexStep,
} from '../../../common/types';
import { generateNewIndexName } from './index_settings';
import { FlatSettings } from './types';
import { FlatSettings, FlatSettingsWithTypeName } from './types';
// TODO: base on elasticsearch.requestTimeout?
export const LOCK_WINDOW = moment.duration(90, 'seconds');
@ -84,6 +84,12 @@ export interface ReindexActions {
*/
getFlatSettings(indexName: string): Promise<FlatSettings | null>;
/**
* Retrieve index settings (in flat, dot-notation style) and mappings with the type name included.
* @param indexName
*/
getFlatSettingsWithTypeName(indexName: string): Promise<FlatSettingsWithTypeName | null>;
// ----- Functions below are for enforcing locks around groups of indices like ML or Watcher
/**
@ -244,6 +250,18 @@ export const reindexActionsFactory = (
return flatSettings[indexName];
},
async getFlatSettingsWithTypeName(indexName: string) {
const flatSettings = (await callCluster('transport.request', {
path: `/${encodeURIComponent(indexName)}?flat_settings=true&include_type_name=true`,
})) as { [indexName: string]: FlatSettingsWithTypeName };
if (!flatSettings[indexName]) {
return null;
}
return flatSettings[indexName];
},
async _fetchAndLockIndexGroupDoc(indexGroup) {
const fetchDoc = async () => {
try {

7
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts
View file

@ -48,6 +48,7 @@ describe('reindexService', () => {
findReindexOperations: jest.fn(unimplemented('findReindexOperations')),
findAllByStatus: jest.fn(unimplemented('findAllInProgressOperations')),
getFlatSettings: jest.fn(unimplemented('getFlatSettings')),
getFlatSettingsWithTypeName: jest.fn(unimplemented('getFlatSettingsWithTypeName')),
cleanupChanges: jest.fn(),
incrementIndexGroupReindexes: jest.fn(unimplemented('incrementIndexGroupReindexes')),
decrementIndexGroupReindexes: jest.fn(unimplemented('decrementIndexGroupReindexes')),
@ -186,12 +187,12 @@ describe('reindexService', () => {
describe('detectReindexWarnings', () => {
it('fetches reindex warnings from flat settings', async () => {
const indexName = 'myIndex';
actions.getFlatSettings.mockResolvedValueOnce({
actions.getFlatSettingsWithTypeName.mockResolvedValueOnce({
settings: {
'index.provided_name': indexName,
},
mappings: {
properties: { https: { type: 'boolean' } },
_doc: { properties: { https: { type: 'boolean' } } },
},
});
@ -200,7 +201,7 @@ describe('reindexService', () => {
});
it('returns null if index does not exist', async () => {
actions.getFlatSettings.mockResolvedValueOnce(null);
actions.getFlatSettingsWithTypeName.mockResolvedValueOnce(null);
const reindexWarnings = await service.detectReindexWarnings('myIndex');
expect(reindexWarnings).toBeNull();
});

2
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts
View file

@ -508,7 +508,7 @@ export const reindexServiceFactory = (
},
async detectReindexWarnings(indexName: string) {
const flatSettings = await actions.getFlatSettings(indexName);
const flatSettings = await actions.getFlatSettingsWithTypeName(indexName);
if (!flatSettings) {
return null;
} else {

12
x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts
View file

@ -26,3 +26,15 @@ export interface FlatSettings {
_meta?: MetaProperties;
};
}
export interface FlatSettingsWithTypeName {
settings: {
[key: string]: string;
};
mappings: {
[typeName: string]: {
properties?: MappingProperties;
_meta?: MetaProperties;
};
};
}

6
x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js
View file

@ -7,7 +7,7 @@
import expect from 'expect.js';
import { ReindexStatus, REINDEX_OP_TYPE } from '../../../plugins/upgrade_assistant/common/types';
import { ReindexStatus, ReindexWarning, REINDEX_OP_TYPE } from '../../../plugins/upgrade_assistant/common/types';
export default function ({ getService }) {
const supertest = getService('supertest');
@ -128,9 +128,9 @@ export default function ({ getService }) {
});
});
it('shows no warnings', async () => {
it('returns a warning for `doc` type', async () => {
const resp = await supertest.get(`/api/upgrade_assistant/reindex/6.0-data`);
expect(resp.body.warnings.length).to.be(0);
expect(resp.body.warnings.includes(ReindexWarning.customTypeName)).to.be(true);
});
it('reindexes old 6.0 index', async () => {