mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 09:19:04 -04:00
Support for reindexing APM indices (#29845)
Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>
This commit is contained in:
Tyler Smalley
GitHub
parent
6ad036b187
commit
c16849dc3b
25 changed files with 2446 additions and 111 deletions
22
src/legacy/core_plugins/apm_oss/index.d.ts
vendored
Normal file
22
src/legacy/core_plugins/apm_oss/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch B.V. under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch B.V. licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
export interface ApmOssPlugin {
|
||||
indexPatterns: string[];
|
||||
}
|
||||
|
|
@ -17,6 +17,8 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import _ from 'lodash';
|
||||
|
||||
export default function apmOss(kibana) {
|
||||
return new kibana.Plugin({
|
||||
id: 'apm_oss',
|
||||
|
|
@ -30,12 +32,24 @@ export default function apmOss(kibana) {
|
|||
indexPattern: Joi.string().default('apm-*'),
|
||||
|
||||
// ES Indices
|
||||
sourcemapIndices: Joi.string().default('apm-*'),
|
||||
errorIndices: Joi.string().default('apm-*'),
|
||||
onboardingIndices: Joi.string().default('apm-*'),
|
||||
spanIndices: Joi.string().default('apm-*'),
|
||||
transactionIndices: Joi.string().default('apm-*'),
|
||||
spanIndices: Joi.string().default('apm-*'),
|
||||
metricsIndices: Joi.string().default('apm-*'),
|
||||
onboardingIndices: Joi.string().default('apm-*'),
|
||||
}).default();
|
||||
},
|
||||
|
||||
init(server) {
|
||||
server.expose('indexPatterns', _.uniq([
|
||||
'sourcemapIndices',
|
||||
'errorIndices',
|
||||
'transactionIndices',
|
||||
'spanIndices',
|
||||
'metricsIndices',
|
||||
'onboardingIndices'
|
||||
].map(type => server.config().get(`apm_oss.${type}`))));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -202,13 +202,15 @@ export interface DeprecationInfo {
|
|||
details?: string;
|
||||
}
|
||||
|
||||
export interface IndexSettingsDeprecationInfo {
|
||||
[indexName: string]: DeprecationInfo[];
|
||||
}
|
||||
|
||||
export interface DeprecationAPIResponse {
|
||||
cluster_settings: DeprecationInfo[];
|
||||
ml_settings: DeprecationInfo[];
|
||||
node_settings: DeprecationInfo[];
|
||||
index_settings: {
|
||||
[indexName: string]: DeprecationInfo[];
|
||||
};
|
||||
index_settings: IndexSettingsDeprecationInfo;
|
||||
}
|
||||
|
||||
export interface CallClusterOptions {
|
||||
|
|
|
|||
3
src/server/kbn_server.d.ts
vendored
3
src/server/kbn_server.d.ts
vendored
|
|
@ -19,7 +19,9 @@
|
|||
|
||||
import { Server } from 'hapi';
|
||||
|
||||
import { ApmOssPlugin } from '../legacy/core_plugins/apm_oss';
|
||||
import { CallClusterWithRequest, ElasticsearchPlugin } from '../legacy/core_plugins/elasticsearch';
|
||||
|
||||
import { IndexPatternsServiceFactory } from './index_patterns';
|
||||
import { SavedObjectsClient, SavedObjectsService } from './saved_objects';
|
||||
|
||||
|
|
@ -33,6 +35,7 @@ declare module 'hapi' {
|
|||
elasticsearch: ElasticsearchPlugin;
|
||||
kibana: any;
|
||||
spaces: any;
|
||||
apm_oss: ApmOssPlugin;
|
||||
// add new plugin types here
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@ export enum ReindexWarning {
|
|||
booleanFields = 1,
|
||||
|
||||
// 7.0 -> 8.0 warnings
|
||||
apmReindex,
|
||||
}
|
||||
|
||||
export enum IndexGroup {
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { Server } from 'hapi';
|
||||
import Joi from 'joi';
|
||||
import { Legacy } from 'kibana';
|
||||
import { resolve } from 'path';
|
||||
import mappings from './mappings.json';
|
||||
import { initServer } from './server';
|
||||
|
|
@ -35,7 +35,7 @@ export function upgradeAssistant(kibana: any) {
|
|||
}).default();
|
||||
},
|
||||
|
||||
init(server: Server) {
|
||||
init(server: Legacy.Server) {
|
||||
// Add server routes and initialize the plugin here
|
||||
initServer(server);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ export class IndexDeprecationTableUI extends React.Component<
|
|||
// NOTE: this naive implementation assumes all indices in the table are
|
||||
// should show the reindex button. This should work for known usecases.
|
||||
const { indices } = this.props;
|
||||
if (!indices.find(i => i.reindex)) {
|
||||
if (!indices.find(i => i.reindex === true)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -143,7 +143,7 @@ export class IndexDeprecationTableUI extends React.Component<
|
|||
actions: [
|
||||
{
|
||||
render(indexDep: IndexDeprecationDetails) {
|
||||
return <ReindexButton indexName={indexDep.index} />;
|
||||
return <ReindexButton indexName={indexDep.index!} />;
|
||||
},
|
||||
},
|
||||
],
|
||||
|
|
|
|||
|
|
@ -10,13 +10,10 @@ import { DeprecationInfo } from 'src/legacy/core_plugins/elasticsearch';
|
|||
import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis';
|
||||
import { GroupByOption } from '../../../types';
|
||||
|
||||
import { CURRENT_MAJOR_VERSION } from 'x-pack/plugins/upgrade_assistant/common/version';
|
||||
import { COLOR_MAP, LEVEL_MAP } from '../constants';
|
||||
import { DeprecationCell } from './cell';
|
||||
import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table';
|
||||
|
||||
const OLD_INDEX_MESSAGE = `Index created before ${CURRENT_MAJOR_VERSION}.0`;
|
||||
|
||||
const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => {
|
||||
return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]);
|
||||
};
|
||||
|
|
@ -37,7 +34,7 @@ const MessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprecationI
|
|||
<DeprecationCell
|
||||
headline={deprecation.message}
|
||||
healthColor={COLOR_MAP[deprecation.level]}
|
||||
reindexIndexName={deprecation.message === OLD_INDEX_MESSAGE ? deprecation.index! : undefined}
|
||||
reindexIndexName={deprecation.reindex ? deprecation.index! : undefined}
|
||||
docUrl={deprecation.url}
|
||||
items={items}
|
||||
/>
|
||||
|
|
@ -91,9 +88,8 @@ export const DeprecationList: StatelessComponent<{
|
|||
const indices = deprecations.map(dep => ({
|
||||
index: dep.index!,
|
||||
details: dep.details,
|
||||
reindex: dep.message === OLD_INDEX_MESSAGE,
|
||||
reindex: dep.reindex === true,
|
||||
}));
|
||||
|
||||
return <IndexDeprecation indices={indices} deprecation={deprecations[0]} />;
|
||||
} else if (currentGroupBy === GroupByOption.index) {
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -108,8 +108,7 @@ exports[`WarningsFlyoutStep renders 1`] = `
|
|||
<EuiCode>
|
||||
1
|
||||
</EuiCode>
|
||||
), reindexing converts these fields to
|
||||
|
||||
), reindexing converts these fields to
|
||||
<EuiCode>
|
||||
true
|
||||
</EuiCode>
|
||||
|
|
@ -129,6 +128,9 @@ exports[`WarningsFlyoutStep renders 1`] = `
|
|||
</EuiLink>
|
||||
</p>
|
||||
</EuiText>
|
||||
<EuiSpacer
|
||||
size="l"
|
||||
/>
|
||||
</EuiFlyoutBody>
|
||||
<EuiFlyoutFooter>
|
||||
<EuiFlexGroup
|
||||
|
|
|
|||
|
|
@ -79,61 +79,93 @@ export class WarningsFlyoutStep extends React.Component<
|
|||
<EuiSpacer />
|
||||
|
||||
{warnings.includes(ReindexWarning.allField) && (
|
||||
<EuiText>
|
||||
<EuiCheckbox
|
||||
id={idForWarning(ReindexWarning.allField)}
|
||||
label={
|
||||
<strong>
|
||||
<EuiCode>_all</EuiCode> field will be removed
|
||||
</strong>
|
||||
}
|
||||
checked={checkedIds[idForWarning(ReindexWarning.allField)]}
|
||||
onChange={this.onChange}
|
||||
/>
|
||||
<p className="upgWarningsStep__warningDescription">
|
||||
The <EuiCode>_all</EuiCode> meta field is no longer supported in 7.0. Reindexing
|
||||
removes the <EuiCode>_all</EuiCode> field in the new index. Ensure that no
|
||||
application code or scripts reply on this field.
|
||||
<br />
|
||||
<EuiLink
|
||||
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
|
||||
target="_blank"
|
||||
>
|
||||
Documentation
|
||||
</EuiLink>
|
||||
</p>
|
||||
</EuiText>
|
||||
<Fragment>
|
||||
<EuiText>
|
||||
<EuiCheckbox
|
||||
id={idForWarning(ReindexWarning.allField)}
|
||||
label={
|
||||
<strong>
|
||||
<EuiCode>_all</EuiCode> field will be removed
|
||||
</strong>
|
||||
}
|
||||
checked={checkedIds[idForWarning(ReindexWarning.allField)]}
|
||||
onChange={this.onChange}
|
||||
/>
|
||||
<p className="upgWarningsStep__warningDescription">
|
||||
The <EuiCode>_all</EuiCode> meta field is no longer supported in 7.0. Reindexing
|
||||
removes the <EuiCode>_all</EuiCode> field in the new index. Ensure that no
|
||||
application code or scripts reply on this field.
|
||||
<br />
|
||||
<EuiLink
|
||||
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
|
||||
target="_blank"
|
||||
>
|
||||
Documentation
|
||||
</EuiLink>
|
||||
</p>
|
||||
</EuiText>
|
||||
|
||||
<EuiSpacer />
|
||||
</Fragment>
|
||||
)}
|
||||
|
||||
<EuiSpacer />
|
||||
{warnings.includes(ReindexWarning.apmReindex) && (
|
||||
<Fragment>
|
||||
<EuiText>
|
||||
<EuiCheckbox
|
||||
id={idForWarning(ReindexWarning.apmReindex)}
|
||||
label={<strong>This index will be converted to ECS format</strong>}
|
||||
checked={checkedIds[idForWarning(ReindexWarning.apmReindex)]}
|
||||
onChange={this.onChange}
|
||||
/>
|
||||
<p className="upgWarningsStep__warningDescription">
|
||||
Starting in version 7.0.0, APM data will be represented in the Elastic Common
|
||||
Schema. Historical APM data will not visible until it's reindexed.
|
||||
<br />
|
||||
<EuiLink
|
||||
href="https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html"
|
||||
target="_blank"
|
||||
>
|
||||
Documentation
|
||||
</EuiLink>
|
||||
</p>
|
||||
</EuiText>
|
||||
|
||||
<EuiSpacer />
|
||||
</Fragment>
|
||||
)}
|
||||
|
||||
{warnings.includes(ReindexWarning.booleanFields) && (
|
||||
<EuiText>
|
||||
<EuiCheckbox
|
||||
id={idForWarning(ReindexWarning.booleanFields)}
|
||||
label={
|
||||
<strong>
|
||||
Boolean data in <EuiCode>_source</EuiCode> might change
|
||||
</strong>
|
||||
}
|
||||
checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]}
|
||||
onChange={this.onChange}
|
||||
/>
|
||||
<p className="upgWarningsStep__warningDescription">
|
||||
If a documents contain a boolean field that is neither <EuiCode>true</EuiCode> or{' '}
|
||||
<EuiCode>false</EuiCode> (for example, <EuiCode>"yes"</EuiCode>,{' '}
|
||||
<EuiCode>"on"</EuiCode>, <EuiCode>1</EuiCode>), reindexing converts these fields to{' '}
|
||||
<EuiCode>true</EuiCode> or <EuiCode>false</EuiCode>. Ensure that no application code
|
||||
or scripts rely on boolean fields in the deprecated format.
|
||||
<br />
|
||||
<EuiLink
|
||||
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields"
|
||||
target="_blank"
|
||||
>
|
||||
Documentation
|
||||
</EuiLink>
|
||||
</p>
|
||||
</EuiText>
|
||||
<Fragment>
|
||||
<EuiText>
|
||||
<EuiCheckbox
|
||||
id={idForWarning(ReindexWarning.booleanFields)}
|
||||
label={
|
||||
<strong>
|
||||
Boolean data in <EuiCode>_source</EuiCode> might change
|
||||
</strong>
|
||||
}
|
||||
checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]}
|
||||
onChange={this.onChange}
|
||||
/>
|
||||
<p className="upgWarningsStep__warningDescription">
|
||||
If a documents contain a boolean field that is neither <EuiCode>true</EuiCode> or{' '}
|
||||
<EuiCode>false</EuiCode> (for example, <EuiCode>"yes"</EuiCode>,{' '}
|
||||
<EuiCode>"on"</EuiCode>, <EuiCode>1</EuiCode>), reindexing converts these fields
|
||||
to <EuiCode>true</EuiCode> or <EuiCode>false</EuiCode>. Ensure that no application
|
||||
code or scripts rely on boolean fields in the deprecated format.
|
||||
<br />
|
||||
<EuiLink
|
||||
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields"
|
||||
target="_blank"
|
||||
>
|
||||
Documentation
|
||||
</EuiLink>
|
||||
</p>
|
||||
</EuiText>
|
||||
|
||||
<EuiSpacer />
|
||||
</Fragment>
|
||||
)}
|
||||
</EuiFlyoutBody>
|
||||
<EuiFlyoutFooter>
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ Object {
|
|||
"index": ".monitoring-es-6-2018.11.07",
|
||||
"level": "warning",
|
||||
"message": "Coercion of boolean fields",
|
||||
"reindex": false,
|
||||
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
|
||||
},
|
||||
Object {
|
||||
|
|
@ -41,6 +42,7 @@ Object {
|
|||
"index": "twitter",
|
||||
"level": "warning",
|
||||
"message": "Coercion of boolean fields",
|
||||
"reindex": false,
|
||||
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
|
||||
},
|
||||
Object {
|
||||
|
|
@ -48,6 +50,7 @@ Object {
|
|||
"index": ".kibana",
|
||||
"level": "warning",
|
||||
"message": "Coercion of boolean fields",
|
||||
"reindex": false,
|
||||
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
|
||||
},
|
||||
Object {
|
||||
|
|
@ -55,6 +58,7 @@ Object {
|
|||
"index": ".watcher-history-6-2018.11.07",
|
||||
"level": "warning",
|
||||
"message": "Coercion of boolean fields",
|
||||
"reindex": false,
|
||||
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
|
||||
},
|
||||
Object {
|
||||
|
|
@ -62,6 +66,7 @@ Object {
|
|||
"index": ".monitoring-kibana-6-2018.11.07",
|
||||
"level": "warning",
|
||||
"message": "Coercion of boolean fields",
|
||||
"reindex": false,
|
||||
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
|
||||
},
|
||||
Object {
|
||||
|
|
@ -69,6 +74,7 @@ Object {
|
|||
"index": "twitter2",
|
||||
"level": "warning",
|
||||
"message": "Coercion of boolean fields",
|
||||
"reindex": false,
|
||||
"url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
|
||||
},
|
||||
],
|
||||
|
|
|
|||
117
x-pack/plugins/upgrade_assistant/server/lib/apm/index.test.ts
Normal file
117
x-pack/plugins/upgrade_assistant/server/lib/apm/index.test.ts
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { getDeprecatedApmIndices, isLegacyApmIndex } from './';
|
||||
|
||||
function mockedCallWithRequest() {
|
||||
return jest.fn().mockImplementation(async () => {
|
||||
return {
|
||||
'foo-1': {
|
||||
mappings: {},
|
||||
},
|
||||
'foo-2': {
|
||||
mappings: {
|
||||
_meta: {
|
||||
version: '6.7.0',
|
||||
},
|
||||
},
|
||||
},
|
||||
'foo-3': {
|
||||
mappings: {
|
||||
_meta: {
|
||||
version: '7.0.0',
|
||||
},
|
||||
},
|
||||
},
|
||||
'foo-4': {
|
||||
mappings: {
|
||||
_meta: {
|
||||
version: '7.1.0',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
describe('getDeprecatedApmIndices', () => {
|
||||
it('calls indices.getMapping', async () => {
|
||||
const callWithRequest = mockedCallWithRequest();
|
||||
await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*', 'bar-*']);
|
||||
|
||||
expect(callWithRequest).toHaveBeenCalledWith({}, 'indices.getMapping', {
|
||||
index: 'foo-*,bar-*',
|
||||
filterPath: '*.mappings._meta.version,*.mappings.properties.@timestamp',
|
||||
});
|
||||
});
|
||||
|
||||
it('includes mappings not yet at 7.0.0', async () => {
|
||||
const callWithRequest = mockedCallWithRequest();
|
||||
const deprecations = await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*']);
|
||||
|
||||
expect(deprecations).toHaveLength(2);
|
||||
expect(deprecations[0].index).toEqual('foo-1');
|
||||
expect(deprecations[1].index).toEqual('foo-2');
|
||||
});
|
||||
|
||||
it('formats the deprecations', async () => {
|
||||
const callWithRequest = mockedCallWithRequest();
|
||||
// @ts-ignore
|
||||
const [deprecation, _] = await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*']);
|
||||
|
||||
expect(deprecation.level).toEqual('warning');
|
||||
expect(deprecation.message).toEqual('APM index needs converted to 7.x format');
|
||||
expect(deprecation.url).toEqual(
|
||||
'https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html'
|
||||
);
|
||||
expect(deprecation.details).toEqual('This index was created prior to 7.0');
|
||||
expect(deprecation.reindex).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isLegacyApmIndex', () => {
|
||||
it('is true when for no version', () => {
|
||||
expect(isLegacyApmIndex('foo-1', ['foo-*'], {})).toEqual(true);
|
||||
});
|
||||
|
||||
it('is true when version is less than 7.0.0', () => {
|
||||
expect(
|
||||
isLegacyApmIndex('foo-1', ['foo-*'], {
|
||||
_meta: { version: '6.7.0' },
|
||||
})
|
||||
).toEqual(true);
|
||||
});
|
||||
|
||||
it('is false when version is 7.0.0', () => {
|
||||
expect(
|
||||
isLegacyApmIndex('foo-1', ['foo-*'], {
|
||||
_meta: { version: '7.0.0' },
|
||||
})
|
||||
).toEqual(false);
|
||||
});
|
||||
|
||||
it('is false when version is greater than 7.0.0', () => {
|
||||
expect(
|
||||
isLegacyApmIndex('foo-1', ['foo-*'], {
|
||||
_meta: { version: '7.1.0' },
|
||||
})
|
||||
).toEqual(false);
|
||||
});
|
||||
|
||||
it('handles multiple index patterns', () => {
|
||||
expect(
|
||||
isLegacyApmIndex('bar-1', ['foo-*', 'bar-*'], {
|
||||
_meta: { version: '6.7.0' },
|
||||
})
|
||||
).toEqual(true);
|
||||
|
||||
expect(
|
||||
isLegacyApmIndex('bar-1', ['foo-*', 'bar-*'], {
|
||||
_meta: { version: '7.0.0' },
|
||||
})
|
||||
).toEqual(false);
|
||||
});
|
||||
});
|
||||
362
x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts
Normal file
362
x-pack/plugins/upgrade_assistant/server/lib/apm/index.ts
Normal file
|
|
@ -0,0 +1,362 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { Request } from 'hapi';
|
||||
import { get } from 'lodash';
|
||||
import minimatch from 'minimatch';
|
||||
import semver from 'semver';
|
||||
import { CallClusterWithRequest } from 'src/legacy/core_plugins/elasticsearch';
|
||||
import pkg from '../../../../../package.json';
|
||||
|
||||
import { EnrichedDeprecationInfo } from '../es_migration_apis';
|
||||
import { FlatSettings } from '../reindexing/types';
|
||||
|
||||
export async function getDeprecatedApmIndices(
|
||||
callWithRequest: CallClusterWithRequest,
|
||||
request: Request,
|
||||
indexPatterns: string[] = []
|
||||
): Promise<EnrichedDeprecationInfo[]> {
|
||||
const indices = await callWithRequest(request, 'indices.getMapping', {
|
||||
index: indexPatterns.join(','),
|
||||
// we include @timestamp to prevent filtering mappings without a version
|
||||
// since @timestamp is expected to always exist
|
||||
filterPath: '*.mappings._meta.version,*.mappings.properties.@timestamp',
|
||||
});
|
||||
|
||||
return Object.keys(indices).reduce((deprecations: EnrichedDeprecationInfo[], index) => {
|
||||
if (semver.lt(get(indices[index], 'mappings._meta.version', '0.0.0'), pkg.version)) {
|
||||
deprecations.push({
|
||||
level: 'warning',
|
||||
message: 'APM index needs converted to 7.x format',
|
||||
url: 'https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html',
|
||||
details: 'This index was created prior to 7.0',
|
||||
reindex: true,
|
||||
index,
|
||||
});
|
||||
}
|
||||
|
||||
return deprecations;
|
||||
}, []);
|
||||
}
|
||||
|
||||
export const isLegacyApmIndex = (
|
||||
indexName: string,
|
||||
apmIndexPatterns: string[] = [],
|
||||
mappings: FlatSettings['mappings']
|
||||
) => {
|
||||
const clientVersion = get(mappings, '_meta.version', '0.0.0');
|
||||
|
||||
const find = apmIndexPatterns.find(pattern => {
|
||||
return minimatch(indexName, pattern) && semver.lt(clientVersion, pkg.version); // no client version or version < 7.0
|
||||
});
|
||||
|
||||
return Boolean(find);
|
||||
};
|
||||
|
||||
// source: https://github.com/elastic/apm-integration-testing/blob/master/tests/server/test_upgrade.py
|
||||
export const apmReindexScript = `
|
||||
// add ecs version
|
||||
ctx._source.ecs = ['version': '1.0.0-beta2'];
|
||||
|
||||
// beat -> observer
|
||||
def beat = ctx._source.remove("beat");
|
||||
if (beat != null) {
|
||||
beat.remove("name");
|
||||
ctx._source.observer = beat;
|
||||
ctx._source.observer.type = "apm-server";
|
||||
}
|
||||
|
||||
if (! ctx._source.containsKey("observer")) {
|
||||
ctx._source.observer = new HashMap();
|
||||
}
|
||||
|
||||
// observer.major_version
|
||||
ctx._source.observer.version_major = 7;
|
||||
|
||||
def listening = ctx._source.remove("listening");
|
||||
if (listening != null) {
|
||||
ctx._source.observer.listening = listening;
|
||||
}
|
||||
|
||||
// remove host[.name]
|
||||
// clarify if we can simply delete this or it will be set somewhere else in 7.0
|
||||
ctx._source.remove("host");
|
||||
|
||||
// docker.container -> container
|
||||
def docker = ctx._source.remove("docker");
|
||||
if (docker != null && docker.containsKey("container")) {
|
||||
ctx._source.container = docker.container;
|
||||
}
|
||||
|
||||
// rip up context
|
||||
HashMap context = ctx._source.remove("context");
|
||||
if (context != null) {
|
||||
// context.process -> process
|
||||
if (context.containsKey("process")) {
|
||||
ctx._source.process = context.remove("process");
|
||||
ctx._source.process.args = ctx._source.process.remove("argv");
|
||||
}
|
||||
|
||||
// context.response -> http.response
|
||||
HashMap resp = context.remove("response");
|
||||
if (resp != null) {
|
||||
if (! ctx._source.containsKey("http")) {
|
||||
ctx._source.http = new HashMap();
|
||||
}
|
||||
ctx._source.http.response = resp;
|
||||
}
|
||||
|
||||
// context.request -> http & url
|
||||
HashMap request = context.remove("request");
|
||||
if (request != null) {
|
||||
if (! ctx._source.containsKey("http")) {
|
||||
ctx._source.http = new HashMap();
|
||||
}
|
||||
|
||||
// context.request.http_version -> http.version
|
||||
def http_version = request.remove("http_version");
|
||||
if (http_version != null) {
|
||||
ctx._source.http.version = http_version;
|
||||
}
|
||||
|
||||
ctx._source.http.request = new HashMap();
|
||||
|
||||
|
||||
// context.request.url -> url
|
||||
HashMap url = request.remove("url");
|
||||
def fragment = url.remove("hash");
|
||||
if (fragment != null) {
|
||||
url.fragment = fragment;
|
||||
}
|
||||
def domain = url.remove("hostname");
|
||||
if (domain != null) {
|
||||
url.domain = domain;
|
||||
}
|
||||
def path = url.remove("pathname");
|
||||
if (path != null) {
|
||||
url.path = path;
|
||||
}
|
||||
def scheme = url.remove("protocol");
|
||||
if (scheme != null) {
|
||||
def end = scheme.lastIndexOf(":");
|
||||
if (end > -1) {
|
||||
scheme = scheme.substring(0, end);
|
||||
}
|
||||
url.scheme = scheme
|
||||
}
|
||||
def original = url.remove("raw");
|
||||
if (original != null) {
|
||||
url.original = original;
|
||||
}
|
||||
def port = url.remove("port");
|
||||
if (port != null) {
|
||||
try {
|
||||
int portNum = Integer.parseInt(port);
|
||||
url.port = portNum;
|
||||
} catch (Exception e) {
|
||||
// toss port
|
||||
}
|
||||
}
|
||||
def query = url.remove("search");
|
||||
if (query != null) {
|
||||
url.query = query;
|
||||
}
|
||||
ctx._source.url = url;
|
||||
|
||||
// restore what is left of request, under http
|
||||
|
||||
def body = request.remove("body");
|
||||
|
||||
ctx._source.http.request = request;
|
||||
ctx._source.http.request.method = ctx._source.http.request.method?.toLowerCase();
|
||||
|
||||
// context.request.body -> http.request.body.original
|
||||
if (body != null) {
|
||||
ctx._source.http.request.body = new HashMap();
|
||||
ctx._source.http.request.body.original = body;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// context.service.agent -> agent
|
||||
HashMap service = context.remove("service");
|
||||
ctx._source.agent = service.remove("agent");
|
||||
|
||||
// context.service -> service
|
||||
ctx._source.service = service;
|
||||
|
||||
// context.system -> host
|
||||
def system = context.remove("system");
|
||||
if (system != null) {
|
||||
system.os = new HashMap();
|
||||
system.os.platform = system.remove("platform");
|
||||
ctx._source.host = system;
|
||||
}
|
||||
|
||||
// context.tags -> labels
|
||||
def tags = context.remove("tags");
|
||||
if (tags != null) {
|
||||
ctx._source.labels = tags;
|
||||
}
|
||||
|
||||
// context.user -> user & user_agent
|
||||
if (context.containsKey("user")) {
|
||||
HashMap user = context.remove("user");
|
||||
// user.username -> user.name
|
||||
def username = user.remove("username");
|
||||
if (username != null) {
|
||||
user.name = username;
|
||||
}
|
||||
|
||||
// context.user.ip -> client.ip
|
||||
if (user.containsKey("ip")) {
|
||||
ctx._source.client = new HashMap();
|
||||
ctx._source.client.ip = user.remove("ip");
|
||||
}
|
||||
|
||||
def ua = user.remove("user-agent");
|
||||
if (ua != null) {
|
||||
ctx._source.user_agent = new HashMap();
|
||||
// setting original and original.text is not possible in painless
|
||||
// as original is a keyword in ES template we cannot set it to a HashMap here,
|
||||
// so the following is the only possible solution:
|
||||
ctx._source.user_agent.original = ua.substring(0, Integer.min(1024, ua.length()));
|
||||
}
|
||||
|
||||
def pua = user.remove("user_agent");
|
||||
if (pua != null) {
|
||||
if (ctx._source.user_agent == null){
|
||||
ctx._source.user_agent = new HashMap();
|
||||
}
|
||||
def os = pua.remove("os");
|
||||
def osminor = pua.remove("os_minor");
|
||||
def osmajor = pua.remove("os_major");
|
||||
def osname = pua.remove("os_name");
|
||||
if (osminor != null || osmajor != null || osname != null){
|
||||
ctx._source.user_agent.os = new HashMap();
|
||||
ctx._source.user_agent.os.full = os;
|
||||
ctx._source.user_agent.os.version = osmajor + "." + osminor;
|
||||
ctx._source.user_agent.os.name = osname;
|
||||
}
|
||||
|
||||
def device = pua.remove("device");
|
||||
if (device != null){
|
||||
ctx._source.user_agent.device = new HashMap();
|
||||
ctx._source.user_agent.device.name = device;
|
||||
}
|
||||
// not exactly reflecting 7.0, but the closes we can get
|
||||
def patch = pua.remove("patch");
|
||||
def minor = pua.remove("minor");
|
||||
def major = pua.remove("major");
|
||||
if (patch != null || minor != null || major != null){
|
||||
ctx._source.user_agent.version = major + "." + minor + "." + patch;
|
||||
}
|
||||
}
|
||||
|
||||
ctx._source.user = user;
|
||||
}
|
||||
|
||||
// context.custom -> error,transaction,span.custom
|
||||
def custom = context.remove("custom");
|
||||
if (custom != null) {
|
||||
if (ctx._source.processor.event == "span") {
|
||||
ctx._source.span.custom = custom;
|
||||
} else if (ctx._source.processor.event == "transaction") {
|
||||
ctx._source.transaction.custom = custom;
|
||||
} else if (ctx._source.processor.event == "error") {
|
||||
ctx._source.error.custom = custom;
|
||||
}
|
||||
}
|
||||
|
||||
// context.db -> span.db
|
||||
def db = context.remove("db");
|
||||
if (db != null) {
|
||||
ctx._source.span.db = db;
|
||||
}
|
||||
|
||||
// context.http -> span.http
|
||||
def http = context.remove("http");
|
||||
if (http != null) {
|
||||
// context.http.url -> span.http.url.original
|
||||
def url = http.remove("url");
|
||||
if (url != null) {
|
||||
http.url = ["original": url];
|
||||
}
|
||||
// context.http.status_code -> span.http.response.status_code
|
||||
def status_code = http.remove("status_code");
|
||||
if (status_code != null) {
|
||||
http.response = ["status_code": status_code];
|
||||
}
|
||||
ctx._source.span.http = http;
|
||||
}
|
||||
}
|
||||
|
||||
if (ctx._source.processor.event == "span") {
|
||||
// bump timestamp.us by span.start.us for spans
|
||||
// shouldn't @timestamp this already be a Date?
|
||||
def ts = ctx._source.get("@timestamp");
|
||||
if (ts != null && !ctx._source.containsKey("timestamp")) {
|
||||
// add span.start to @timestamp for rum documents v1
|
||||
if (ctx._source.context.service.agent.name == "js-base" && ctx._source.span.start.containsKey("us")) {
|
||||
ts += ctx._source.span.start.us/1000;
|
||||
}
|
||||
}
|
||||
if (ctx._source.span.containsKey("hex_id")) {
|
||||
ctx._source.span.id = ctx._source.span.remove("hex_id");
|
||||
}
|
||||
def parent = ctx._source.span.remove("parent");
|
||||
if (parent != null && ctx._source.parent == null) {
|
||||
ctx._source.parent = ["id": parent];
|
||||
}
|
||||
}
|
||||
|
||||
// create trace.id
|
||||
if (ctx._source.processor.event == "transaction" || ctx._source.processor.event == "span" || ctx._source.processor.event == "error") {
|
||||
if (ctx._source.containsKey("transaction")) {
|
||||
def tr_id = ctx._source.transaction.get("id");
|
||||
if (ctx._source.trace == null && tr_id != null) {
|
||||
// create a trace id from the transaction.id
|
||||
// v1 transaction.id was a UUID, should have 122 random bits or so
|
||||
ctx._source.trace = new HashMap();
|
||||
ctx._source.trace.id = tr_id.replace("-", "");
|
||||
}
|
||||
}
|
||||
|
||||
// create timestamp.us from @timestamp
|
||||
def ts = ctx._source.get("@timestamp");
|
||||
if (ts != null && !ctx._source.containsKey("timestamp")) {
|
||||
//set timestamp.microseconds to @timestamp
|
||||
ctx._source.timestamp = new HashMap();
|
||||
ctx._source.timestamp.us = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").parse(ts).getTime()*1000;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// transaction.span_count.dropped.total -> transaction.span_count.dropped
|
||||
if (ctx._source.processor.event == "transaction") {
|
||||
// transaction.span_count.dropped.total -> transaction.span_count.dropped
|
||||
if (ctx._source.transaction.containsKey("span_count")) {
|
||||
def dropped = ctx._source.transaction.span_count.remove("dropped");
|
||||
if (dropped != null) {
|
||||
ctx._source.transaction.span_count.dropped = dropped.total;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ctx._source.processor.event == "error") {
|
||||
// culprit is now a keyword, so trim it down to 1024 chars
|
||||
def culprit = ctx._source.error.remove("culprit");
|
||||
if (culprit != null) {
|
||||
ctx._source.error.culprit = culprit.substring(0, Integer.min(1024, culprit.length()));
|
||||
}
|
||||
|
||||
// error.exception is now a list (exception chain)
|
||||
def exception = ctx._source.error.remove("exception");
|
||||
if (exception != null) {
|
||||
ctx._source.error.exception = [exception];
|
||||
}
|
||||
}
|
||||
`;
|
||||
1598
x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json
Normal file
1598
x-pack/plugins/upgrade_assistant/server/lib/apm/mapping.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -16,6 +16,8 @@ describe('getUpgradeAssistantStatus', () => {
|
|||
const callWithRequest = jest.fn().mockImplementation(async (req, api, { path }) => {
|
||||
if (path === '/_migration/deprecations') {
|
||||
return deprecationsResponse;
|
||||
} else if (api === 'indices.getMapping') {
|
||||
return {};
|
||||
} else {
|
||||
throw new Error(`Unexpected API call: ${path}`);
|
||||
}
|
||||
|
|
@ -26,7 +28,7 @@ describe('getUpgradeAssistantStatus', () => {
|
|||
});
|
||||
|
||||
it('calls /_migration/deprecations', async () => {
|
||||
await getUpgradeAssistantStatus(callWithRequest, {} as any, false);
|
||||
await getUpgradeAssistantStatus(callWithRequest, {} as any, false, []);
|
||||
expect(callWithRequest).toHaveBeenCalledWith({}, 'transport.request', {
|
||||
path: '/_migration/deprecations',
|
||||
method: 'GET',
|
||||
|
|
@ -34,7 +36,7 @@ describe('getUpgradeAssistantStatus', () => {
|
|||
});
|
||||
|
||||
it('returns the correct shape of data', async () => {
|
||||
const resp = await getUpgradeAssistantStatus(callWithRequest, {} as any, false);
|
||||
const resp = await getUpgradeAssistantStatus(callWithRequest, {} as any, false, []);
|
||||
expect(resp).toMatchSnapshot();
|
||||
});
|
||||
|
||||
|
|
@ -47,7 +49,7 @@ describe('getUpgradeAssistantStatus', () => {
|
|||
};
|
||||
|
||||
await expect(
|
||||
getUpgradeAssistantStatus(callWithRequest, {} as any, false)
|
||||
getUpgradeAssistantStatus(callWithRequest, {} as any, false, [])
|
||||
).resolves.toHaveProperty('readyForUpgrade', false);
|
||||
});
|
||||
|
||||
|
|
@ -60,7 +62,7 @@ describe('getUpgradeAssistantStatus', () => {
|
|||
};
|
||||
|
||||
await expect(
|
||||
getUpgradeAssistantStatus(callWithRequest, {} as any, false)
|
||||
getUpgradeAssistantStatus(callWithRequest, {} as any, false, [])
|
||||
).resolves.toHaveProperty('readyForUpgrade', true);
|
||||
});
|
||||
|
||||
|
|
@ -78,7 +80,7 @@ describe('getUpgradeAssistantStatus', () => {
|
|||
index_settings: {},
|
||||
};
|
||||
|
||||
const result = await getUpgradeAssistantStatus(callWithRequest, {} as any, true);
|
||||
const result = await getUpgradeAssistantStatus(callWithRequest, {} as any, true, []);
|
||||
|
||||
expect(result).toHaveProperty('readyForUpgrade', true);
|
||||
expect(result).toHaveProperty('cluster', []);
|
||||
|
|
|
|||
|
|
@ -13,9 +13,12 @@ import {
|
|||
DeprecationInfo,
|
||||
} from 'src/legacy/core_plugins/elasticsearch';
|
||||
|
||||
import { getDeprecatedApmIndices } from './apm';
|
||||
|
||||
export interface EnrichedDeprecationInfo extends DeprecationInfo {
|
||||
index?: string;
|
||||
node?: string;
|
||||
reindex?: boolean;
|
||||
}
|
||||
|
||||
export interface UpgradeAssistantStatus {
|
||||
|
|
@ -27,15 +30,19 @@ export interface UpgradeAssistantStatus {
|
|||
export async function getUpgradeAssistantStatus(
|
||||
callWithRequest: CallClusterWithRequest,
|
||||
req: Request,
|
||||
isCloudEnabled: boolean
|
||||
isCloudEnabled: boolean,
|
||||
apmIndices: string[]
|
||||
): Promise<UpgradeAssistantStatus> {
|
||||
const deprecations = await callWithRequest(req, 'transport.request', {
|
||||
path: '/_migration/deprecations',
|
||||
method: 'GET',
|
||||
});
|
||||
const [deprecations, apmIndexDeprecations] = await Promise.all([
|
||||
(await callWithRequest(req, 'transport.request', {
|
||||
path: '/_migration/deprecations',
|
||||
method: 'GET',
|
||||
})) as DeprecationAPIResponse,
|
||||
getDeprecatedApmIndices(callWithRequest, req, apmIndices),
|
||||
]);
|
||||
|
||||
const cluster = getClusterDeprecations(deprecations, isCloudEnabled);
|
||||
const indices = getCombinedIndexInfos(deprecations);
|
||||
const indices = getCombinedIndexInfos(deprecations, apmIndexDeprecations);
|
||||
|
||||
const criticalWarnings = cluster.concat(indices).filter(d => d.level === 'critical');
|
||||
|
||||
|
|
@ -47,17 +54,35 @@ export async function getUpgradeAssistantStatus(
|
|||
}
|
||||
|
||||
// Reformats the index deprecations to an array of deprecation warnings extended with an index field.
|
||||
const getCombinedIndexInfos = (deprecations: DeprecationAPIResponse) =>
|
||||
Object.keys(deprecations.index_settings).reduce(
|
||||
(indexDeprecations, indexName) => {
|
||||
return indexDeprecations.concat(
|
||||
deprecations.index_settings[indexName].map(
|
||||
d => ({ ...d, index: indexName } as EnrichedDeprecationInfo)
|
||||
)
|
||||
);
|
||||
},
|
||||
[] as EnrichedDeprecationInfo[]
|
||||
);
|
||||
const getCombinedIndexInfos = (
|
||||
deprecations: DeprecationAPIResponse,
|
||||
apmIndexDeprecations: EnrichedDeprecationInfo[]
|
||||
) => {
|
||||
const apmIndices = apmIndexDeprecations.reduce((acc, dep) => acc.add(dep.index), new Set());
|
||||
|
||||
return Object.keys(deprecations.index_settings)
|
||||
.reduce(
|
||||
(indexDeprecations, indexName) => {
|
||||
// prevent APM indices from showing up for general re-indexing
|
||||
if (apmIndices.has(indexName)) {
|
||||
return indexDeprecations;
|
||||
}
|
||||
|
||||
return indexDeprecations.concat(
|
||||
deprecations.index_settings[indexName].map(
|
||||
d =>
|
||||
({
|
||||
...d,
|
||||
index: indexName,
|
||||
reindex: /Index created before/.test(d.message),
|
||||
} as EnrichedDeprecationInfo)
|
||||
)
|
||||
);
|
||||
},
|
||||
[] as EnrichedDeprecationInfo[]
|
||||
)
|
||||
.concat(apmIndexDeprecations);
|
||||
};
|
||||
|
||||
const getClusterDeprecations = (deprecations: DeprecationAPIResponse, isCloudEnabled: boolean) => {
|
||||
const combined = deprecations.cluster_settings
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import {
|
|||
PREV_MAJOR_VERSION,
|
||||
} from 'x-pack/plugins/upgrade_assistant/common/version';
|
||||
import { ReindexWarning } from '../../../common/types';
|
||||
import { isLegacyApmIndex } from '../apm';
|
||||
import { FlatSettings } from './types';
|
||||
|
||||
export interface ParsedIndexName {
|
||||
|
|
@ -59,10 +60,16 @@ export const parseIndexName = (indexName: string): ParsedIndexName => {
|
|||
* Returns an array of warnings that should be displayed to user before reindexing begins.
|
||||
* @param flatSettings
|
||||
*/
|
||||
export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[] => {
|
||||
const warnings = [
|
||||
// No warnings yet for 7.0 -> 8.0
|
||||
] as Array<[ReindexWarning, boolean]>;
|
||||
export const getReindexWarnings = (
|
||||
flatSettings: FlatSettings,
|
||||
apmIndexPatterns: string[] = []
|
||||
): ReindexWarning[] => {
|
||||
const indexName = flatSettings.settings['index.provided_name'];
|
||||
const apmReindexWarning = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings);
|
||||
|
||||
const warnings = [[ReindexWarning.apmReindex, apmReindexWarning]] as Array<
|
||||
[ReindexWarning, boolean]
|
||||
>;
|
||||
|
||||
return warnings.filter(([_, applies]) => applies).map(([warning, _]) => warning);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ import {
|
|||
ReindexStatus,
|
||||
ReindexStep,
|
||||
} from '../../../common/types';
|
||||
import { apmReindexScript } from '../apm';
|
||||
import apmMappings from '../apm/mapping.json';
|
||||
import { ReindexService, reindexServiceFactory } from './reindex_service';
|
||||
|
||||
describe('reindexService', () => {
|
||||
|
|
@ -58,7 +60,7 @@ describe('reindexService', () => {
|
|||
},
|
||||
})),
|
||||
};
|
||||
service = reindexServiceFactory(callCluster, xpackInfo as any, actions);
|
||||
service = reindexServiceFactory(callCluster, xpackInfo as any, actions, ['apm-*']);
|
||||
});
|
||||
|
||||
describe('hasRequiredPrivileges', () => {
|
||||
|
|
@ -179,14 +181,17 @@ describe('reindexService', () => {
|
|||
|
||||
describe('detectReindexWarnings', () => {
|
||||
it('fetches reindex warnings from flat settings', async () => {
|
||||
const indexName = 'myIndex';
|
||||
actions.getFlatSettings.mockResolvedValueOnce({
|
||||
settings: {},
|
||||
settings: {
|
||||
'index.provided_name': indexName,
|
||||
},
|
||||
mappings: {
|
||||
properties: { https: { type: 'boolean' } },
|
||||
},
|
||||
});
|
||||
|
||||
const reindexWarnings = await service.detectReindexWarnings('myIndex');
|
||||
const reindexWarnings = await service.detectReindexWarnings(indexName);
|
||||
expect(reindexWarnings).toEqual([]);
|
||||
});
|
||||
|
||||
|
|
@ -727,6 +732,43 @@ describe('reindexService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('used APM mapping for legacy APM index', async () => {
|
||||
const indexName = 'apm-1';
|
||||
const newIndexName = 'apm-1-reindexed';
|
||||
|
||||
actions.getFlatSettings.mockResolvedValueOnce({
|
||||
settings: {
|
||||
'index.number_of_replicas': 5,
|
||||
},
|
||||
mappings: {
|
||||
_meta: {
|
||||
version: '6.7.0',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
callCluster.mockResolvedValueOnce({ acknowledged: true }); // indices.create
|
||||
await service.processNextStep({
|
||||
id: '1',
|
||||
attributes: {
|
||||
...defaultAttributes,
|
||||
indexName,
|
||||
newIndexName,
|
||||
lastCompletedStep: ReindexStep.readonly,
|
||||
},
|
||||
} as ReindexSavedObject);
|
||||
|
||||
expect(callCluster).toHaveBeenCalledWith('indices.create', {
|
||||
index: newIndexName,
|
||||
body: {
|
||||
mappings: apmMappings,
|
||||
settings: {
|
||||
'index.number_of_replicas': 5,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('fails if create index is not acknowledged', async () => {
|
||||
callCluster
|
||||
.mockResolvedValueOnce({ myIndex: settingsMappings })
|
||||
|
|
@ -761,6 +803,13 @@ describe('reindexService', () => {
|
|||
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.newIndexCreated },
|
||||
} as ReindexSavedObject;
|
||||
|
||||
beforeEach(() => {
|
||||
actions.getFlatSettings.mockResolvedValueOnce({
|
||||
settings: {},
|
||||
mappings: {},
|
||||
});
|
||||
});
|
||||
|
||||
it('starts reindex, saves taskId, and updates lastCompletedStep', async () => {
|
||||
callCluster.mockResolvedValueOnce({ task: 'xyz' }); // reindex
|
||||
const updatedOp = await service.processNextStep(reindexOp);
|
||||
|
|
@ -777,6 +826,43 @@ describe('reindexService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('uses APM script for legacy APM index', async () => {
|
||||
const indexName = 'apm-1';
|
||||
const newIndexName = 'apm-1-reindexed';
|
||||
|
||||
callCluster.mockResolvedValueOnce({ task: 'xyz' }); // reindex
|
||||
actions.getFlatSettings.mockResolvedValueOnce({
|
||||
settings: {},
|
||||
mappings: {
|
||||
_meta: {
|
||||
version: '6.7.0',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await service.processNextStep({
|
||||
id: '1',
|
||||
attributes: {
|
||||
...defaultAttributes,
|
||||
indexName,
|
||||
newIndexName,
|
||||
lastCompletedStep: ReindexStep.newIndexCreated,
|
||||
},
|
||||
} as ReindexSavedObject);
|
||||
expect(callCluster).toHaveBeenLastCalledWith('reindex', {
|
||||
refresh: true,
|
||||
waitForCompletion: false,
|
||||
body: {
|
||||
source: { index: indexName },
|
||||
dest: { index: newIndexName },
|
||||
script: {
|
||||
lang: 'painless',
|
||||
source: apmReindexScript,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('fails if starting reindex fails', async () => {
|
||||
callCluster.mockRejectedValueOnce(new Error('blah!')).mockResolvedValueOnce({});
|
||||
const updatedOp = await service.processNextStep(reindexOp);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import {
|
|||
ReindexStep,
|
||||
ReindexWarning,
|
||||
} from '../../../common/types';
|
||||
import { apmReindexScript, isLegacyApmIndex } from '../apm';
|
||||
import apmMappings from '../apm/mapping.json';
|
||||
import { getReindexWarnings, parseIndexName, transformFlatSettings } from './index_settings';
|
||||
import { ReindexActions } from './reindex_actions';
|
||||
|
||||
|
|
@ -91,7 +93,8 @@ export interface ReindexService {
|
|||
export const reindexServiceFactory = (
|
||||
callCluster: CallCluster,
|
||||
xpackInfo: XPackInfo,
|
||||
actions: ReindexActions
|
||||
actions: ReindexActions,
|
||||
apmIndexPatterns: string[] = []
|
||||
): ReindexService => {
|
||||
// ------ Utility functions
|
||||
|
||||
|
|
@ -279,11 +282,13 @@ export const reindexServiceFactory = (
|
|||
}
|
||||
|
||||
const { settings, mappings } = transformFlatSettings(flatSettings);
|
||||
const legacyApmIndex = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings);
|
||||
|
||||
const createIndex = await callCluster('indices.create', {
|
||||
index: newIndexName,
|
||||
body: {
|
||||
settings,
|
||||
mappings,
|
||||
mappings: legacyApmIndex ? apmMappings : mappings,
|
||||
},
|
||||
});
|
||||
|
||||
|
|
@ -302,13 +307,29 @@ export const reindexServiceFactory = (
|
|||
*/
|
||||
const startReindexing = async (reindexOp: ReindexSavedObject) => {
|
||||
const { indexName } = reindexOp.attributes;
|
||||
|
||||
const reindexBody = {
|
||||
source: { index: indexName },
|
||||
dest: { index: reindexOp.attributes.newIndexName },
|
||||
} as any;
|
||||
|
||||
const flatSettings = await actions.getFlatSettings(indexName);
|
||||
if (!flatSettings) {
|
||||
throw Boom.notFound(`Index ${indexName} does not exist.`);
|
||||
}
|
||||
|
||||
const legacyApmIndex = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings);
|
||||
if (legacyApmIndex) {
|
||||
reindexBody.script = {
|
||||
lang: 'painless',
|
||||
source: apmReindexScript,
|
||||
};
|
||||
}
|
||||
|
||||
const startReindex = (await callCluster('reindex', {
|
||||
refresh: true,
|
||||
waitForCompletion: false,
|
||||
body: {
|
||||
source: { index: indexName },
|
||||
dest: { index: reindexOp.attributes.newIndexName },
|
||||
},
|
||||
body: reindexBody,
|
||||
})) as any;
|
||||
|
||||
return actions.updateReindexOp(reindexOp, {
|
||||
|
|
@ -486,7 +507,7 @@ export const reindexServiceFactory = (
|
|||
if (!flatSettings) {
|
||||
return null;
|
||||
} else {
|
||||
return getReindexWarnings(flatSettings);
|
||||
return getReindexWarnings(flatSettings, apmIndexPatterns);
|
||||
}
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,16 @@ export interface MappingProperties {
|
|||
[key: string]: Mapping;
|
||||
}
|
||||
|
||||
interface MetaProperties {
|
||||
[key: string]: string;
|
||||
}
|
||||
|
||||
export interface FlatSettings {
|
||||
settings: {
|
||||
[key: string]: string;
|
||||
};
|
||||
mappings: {
|
||||
properties?: MappingProperties;
|
||||
_meta?: MetaProperties;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,16 +48,20 @@ export class ReindexWorker {
|
|||
private callWithRequest: CallClusterWithRequest,
|
||||
private callWithInternalUser: CallCluster,
|
||||
private xpackInfo: XPackInfo,
|
||||
private readonly log: Server['log']
|
||||
private readonly log: Server['log'],
|
||||
private apmIndexPatterns: string[]
|
||||
) {
|
||||
if (ReindexWorker.workerSingleton) {
|
||||
throw new Error(`More than one ReindexWorker cannot be created.`);
|
||||
}
|
||||
|
||||
this.apmIndexPatterns = apmIndexPatterns;
|
||||
|
||||
this.reindexService = reindexServiceFactory(
|
||||
this.callWithInternalUser,
|
||||
this.xpackInfo,
|
||||
reindexActionsFactory(this.client, this.callWithInternalUser)
|
||||
reindexActionsFactory(this.client, this.callWithInternalUser),
|
||||
apmIndexPatterns
|
||||
);
|
||||
|
||||
ReindexWorker.workerSingleton = this;
|
||||
|
|
@ -161,7 +165,12 @@ export class ReindexWorker {
|
|||
const fakeRequest = { headers: credential } as Request;
|
||||
const callCluster = this.callWithRequest.bind(null, fakeRequest) as CallCluster;
|
||||
const actions = reindexActionsFactory(this.client, callCluster);
|
||||
const service = reindexServiceFactory(callCluster, this.xpackInfo, actions);
|
||||
const service = reindexServiceFactory(
|
||||
callCluster,
|
||||
this.xpackInfo,
|
||||
actions,
|
||||
this.apmIndexPatterns
|
||||
);
|
||||
reindexOp = await swallowExceptions(service.processNextStep, this.log)(reindexOp);
|
||||
|
||||
// Update credential store with most recent state.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ describe('cluster checkup API', () => {
|
|||
elasticsearch: {
|
||||
getCluster: () => ({ callWithRequest: jest.fn() } as any),
|
||||
} as any,
|
||||
apm_oss: {
|
||||
indexPatterns: ['apm-*'],
|
||||
},
|
||||
cloud: {
|
||||
isCloudEnabled: false,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -18,7 +18,14 @@ export function registerClusterCheckupRoutes(server: Legacy.Server) {
|
|||
method: 'GET',
|
||||
async handler(request) {
|
||||
try {
|
||||
return await getUpgradeAssistantStatus(callWithRequest, request, isCloudEnabled);
|
||||
const apmIndexPatterns = server.plugins.apm_oss.indexPatterns;
|
||||
|
||||
return await getUpgradeAssistantStatus(
|
||||
callWithRequest,
|
||||
request,
|
||||
isCloudEnabled,
|
||||
apmIndexPatterns
|
||||
);
|
||||
} catch (e) {
|
||||
if (e.status === 403) {
|
||||
return Boom.forbidden(e.message);
|
||||
|
|
|
|||
|
|
@ -42,6 +42,9 @@ describe('reindex API', () => {
|
|||
xpack_main: {
|
||||
info: {},
|
||||
},
|
||||
apm_oss: {
|
||||
indexPatterns: ['apm-*'],
|
||||
},
|
||||
} as any;
|
||||
server.config = () => ({ get: () => '' } as any);
|
||||
server.decorate('request', 'getSavedObjectsClient', () => jest.fn());
|
||||
|
|
|
|||
|
|
@ -40,7 +40,8 @@ export function registerReindexWorker(server: Server, credentialStore: Credentia
|
|||
callWithRequest,
|
||||
callWithInternalUser,
|
||||
xpackInfo,
|
||||
log
|
||||
log,
|
||||
server.plugins.apm_oss.indexPatterns
|
||||
);
|
||||
|
||||
// Wait for ES connection before starting the polling loop.
|
||||
|
|
@ -59,6 +60,7 @@ export function registerReindexIndicesRoutes(
|
|||
) {
|
||||
const { callWithRequest } = server.plugins.elasticsearch.getCluster('admin');
|
||||
const xpackInfo = server.plugins.xpack_main.info;
|
||||
const apmIndexPatterns = server.plugins.apm_oss.indexPatterns;
|
||||
const BASE_PATH = '/api/upgrade_assistant/reindex';
|
||||
|
||||
// Start reindex for an index
|
||||
|
|
@ -70,7 +72,12 @@ export function registerReindexIndicesRoutes(
|
|||
const { indexName } = request.params;
|
||||
const callCluster = callWithRequest.bind(null, request) as CallCluster;
|
||||
const reindexActions = reindexActionsFactory(client, callCluster);
|
||||
const reindexService = reindexServiceFactory(callCluster, xpackInfo, reindexActions);
|
||||
const reindexService = reindexServiceFactory(
|
||||
callCluster,
|
||||
xpackInfo,
|
||||
reindexActions,
|
||||
apmIndexPatterns
|
||||
);
|
||||
|
||||
try {
|
||||
if (!(await reindexService.hasRequiredPrivileges(indexName))) {
|
||||
|
|
@ -111,7 +118,12 @@ export function registerReindexIndicesRoutes(
|
|||
const { indexName } = request.params;
|
||||
const callCluster = callWithRequest.bind(null, request) as CallCluster;
|
||||
const reindexActions = reindexActionsFactory(client, callCluster);
|
||||
const reindexService = reindexServiceFactory(callCluster, xpackInfo, reindexActions);
|
||||
const reindexService = reindexServiceFactory(
|
||||
callCluster,
|
||||
xpackInfo,
|
||||
reindexActions,
|
||||
apmIndexPatterns
|
||||
);
|
||||
|
||||
try {
|
||||
const hasRequiredPrivileges = await reindexService.hasRequiredPrivileges(indexName);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue