github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add reindex feature to Upgrade Assistant (#27457)

Browse source 
* Fixup saved objects

* WIP reindex state machine

* WIP UI

* Separate status from current step

* Add error messages

* Copy types to CallWithInternalUser

* Use backend worker

* Add progress bar for large indices

* Cleanup worker implementation

* Add tests for ReindexService

* Fix types

* Fix CI

* Add support for mapping coerced boolean fields

* Add basic functional integration test

* Cleanup reindexing backend, add more checks

* Cleanup frontend code and add tests

* Support removal of _default_ mapping + add tests

* Add reindex warnings to reindex service

* Add confirmation modal for reindex warnings

* Cleanup flyout appearance

* Generate new index name intelligently

* Design tweaks

* Show reindex button in both grouping modes

* Add data archive integration tests

* Change flyout design to two step process

* Reorganize flyout files

* Use custom steps component

* Allow writes to indices if anything fails

* Misc cleanup

* Design edits

* Only show warnings panel when reindex has not begun

* Fix types

* Handle moving existing aliases

* Move integration tests to separate config

* Fix ReindexButton bugs when changing pages

* Fix polling service tests

* Refactor ReindexService and ML handling (backend only)

* Add credential caching and paused state to backend

* Add paused state to frontend

* Fix ts errors

* Update copy

* Add check for node version before ML step

* Update data archive format

* Update snapshots for React upgrade

* Handle _default_ mappings correctly in warning checks

* Use hashed state of reindex operation for credential storage

* Address tyler's comments

* Add watcher stop/starting

* Use json-stable-stringify and sha256 for CredentialStore

* Add types for json-stable-stringify
This commit is contained in:
Josh Dover 2019-01-29 10:14:32 -06:00 • committed by GitHub
parent 55dbc66f74
commit 468eabbbb8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
64 changed files with 5214 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

157
src/legacy/core_plugins/elasticsearch/index.d.ts vendored
View file

@ -149,7 +149,7 @@ import {
export class Cluster {
public callWithRequest: CallClusterWithRequest;
public callWithInternalUser: CallClusterWithInternalUser;
public callWithInternalUser: CallCluster;
public constructor(config: ClusterConfig);
}
@ -376,11 +376,156 @@ export interface CallClusterWithRequest {
): Promise<T>;
}
export type CallClusterWithInternalUser = <T = any>(
endpoint: string,
clientParams: GenericParams,
options?: CallClusterOptions
) => Promise<T>;
export interface CallCluster {
/* tslint:disable */
(endpoint: 'bulk', params: BulkIndexDocumentsParams, options?: CallClusterOptions): ReturnType<ESClient['bulk']>;
(endpoint: 'clearScroll', params: ClearScrollParams, options?: CallClusterOptions): ReturnType<ESClient['clearScroll']>;
(endpoint: 'count', params: CountParams, options?: CallClusterOptions): ReturnType<ESClient['count']>;
(endpoint: 'create', params: CreateDocumentParams, options?: CallClusterOptions): ReturnType<ESClient['create']>;
(endpoint: 'delete', params: DeleteDocumentParams, options?: CallClusterOptions): ReturnType<ESClient['delete']>;
(endpoint: 'deleteByQuery', params: DeleteDocumentByQueryParams, options?: CallClusterOptions): ReturnType<ESClient['deleteByQuery']>;
(endpoint: 'deleteScript', params: DeleteScriptParams, options?: CallClusterOptions): ReturnType<ESClient['deleteScript']>;
(endpoint: 'deleteTemplate', params: DeleteTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['deleteTemplate']>;
(endpoint: 'exists', params: ExistsParams, options?: CallClusterOptions): ReturnType<ESClient['exists']>;
(endpoint: 'explain', params: ExplainParams, options?: CallClusterOptions): ReturnType<ESClient['explain']>;
(endpoint: 'fieldStats', params: FieldStatsParams, options?: CallClusterOptions): ReturnType<ESClient['fieldStats']>;
// Generic types cannot be properly looked up with ReturnType. Hard code these explicitly.
<T>(endpoint: 'get', params: GetParams, options?: CallClusterOptions): Promise<GetResponse<T>>;
(endpoint: 'getScript', params: GetScriptParams, options?: CallClusterOptions): ReturnType<ESClient['getScript']>;
(endpoint: 'getSource', params: GetSourceParams, options?: CallClusterOptions): ReturnType<ESClient['getSource']>;
(endpoint: 'getTemplate', params: GetTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['getTemplate']>;
// Generic types cannot be properly looked up with ReturnType. Hard code these explicitly.
<T>(endpoint: 'index', params: IndexDocumentParams<T>, options?: CallClusterOptions): ReturnType<ESClient['index']>;
(endpoint: 'info', params: InfoParams, options?: CallClusterOptions): ReturnType<ESClient['info']>;
// Generic types cannot be properly looked up with ReturnType. Hard code these explicitly.
<T>(endpoint: 'mget', params: MGetParams, options?: CallClusterOptions): Promise<MGetResponse<T>>;
<T>(endpoint: 'msearch', params: MSearchParams, options?: CallClusterOptions): Promise<MSearchResponse<T>>;
<T>(endpoint: 'msearchTemplate', params: MSearchTemplateParams, options?: CallClusterOptions): Promise<MSearchResponse<T>>;
(endpoint: 'mtermvectors', params: MTermVectorsParams, options?: CallClusterOptions): ReturnType<ESClient['mtermvectors']>;
(endpoint: 'ping', params: PingParams, options?: CallClusterOptions): ReturnType<ESClient['ping']>;
(endpoint: 'putScript', params: PutScriptParams, options?: CallClusterOptions): ReturnType<ESClient['putScript']>;
(endpoint: 'putTemplate', params: PutTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['putTemplate']>;
(endpoint: 'reindex', params: ReindexParams, options?: CallClusterOptions): ReturnType<ESClient['reindex']>;
(endpoint: 'reindexRethrottle', params: ReindexRethrottleParams, options?: CallClusterOptions): ReturnType<ESClient['reindexRethrottle']>;
(endpoint: 'renderSearchTemplate', params: RenderSearchTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['renderSearchTemplate']>;
// Generic types cannot be properly looked up with ReturnType. Hard code these explicitly.
<T>(endpoint: 'scroll', params: ScrollParams, options?: CallClusterOptions): Promise<SearchResponse<T>>;
<T>(endpoint: 'search', params: SearchParams, options?: CallClusterOptions): Promise<SearchResponse<T>>;
(endpoint: 'searchShards', params: SearchShardsParams, options?: CallClusterOptions): ReturnType<ESClient['searchShards']>;
(endpoint: 'searchTemplate', params: SearchTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['searchTemplate']>;
(endpoint: 'suggest', params: SuggestParams, options?: CallClusterOptions): ReturnType<ESClient['suggest']>;
(endpoint: 'termvectors', params: TermvectorsParams, options?: CallClusterOptions): ReturnType<ESClient['termvectors']>;
(endpoint: 'update', params: UpdateDocumentParams, options?: CallClusterOptions): ReturnType<ESClient['update']>;
(endpoint: 'updateByQuery', params: UpdateDocumentByQueryParams, options?: CallClusterOptions): ReturnType<ESClient['updateByQuery']>;
// cat namespace
(endpoint: 'cat.aliases', params: CatAliasesParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['aliases']>;
(endpoint: 'cat.allocation', params: CatAllocationParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['allocation']>;
(endpoint: 'cat.count', params: CatAllocationParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['count']>;
(endpoint: 'cat.fielddata', params: CatFielddataParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['fielddata']>;
(endpoint: 'cat.health', params: CatHealthParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['health']>;
(endpoint: 'cat.help', params: CatHelpParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['help']>;
(endpoint: 'cat.indices', params: CatIndicesParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['indices']>;
(endpoint: 'cat.master', params: CatCommonParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['master']>;
(endpoint: 'cat.nodeattrs', params: CatCommonParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['nodeattrs']>;
(endpoint: 'cat.nodes', params: CatCommonParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['nodes']>;
(endpoint: 'cat.pendingTasks', params: CatCommonParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['pendingTasks']>;
(endpoint: 'cat.plugins', params: CatCommonParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['plugins']>;
(endpoint: 'cat.recovery', params: CatRecoveryParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['recovery']>;
(endpoint: 'cat.repositories', params: CatCommonParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['repositories']>;
(endpoint: 'cat.segments', params: CatSegmentsParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['segments']>;
(endpoint: 'cat.shards', params: CatShardsParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['shards']>;
(endpoint: 'cat.snapshots', params: CatSnapshotsParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['snapshots']>;
(endpoint: 'cat.tasks', params: CatTasksParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['tasks']>;
(endpoint: 'cat.threadPool', params: CatThreadPoolParams, options?: CallClusterOptions): ReturnType<ESClient['cat']['threadPool']>;
// cluster namespace
(endpoint: 'cluster.allocationExplain', params: ClusterAllocationExplainParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['allocationExplain']>;
(endpoint: 'cluster.getSettings', params: ClusterGetSettingsParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['getSettings']>;
(endpoint: 'cluster.health', params: ClusterHealthParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['health']>;
(endpoint: 'cluster.pendingTasks', params: ClusterPendingTasksParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['pendingTasks']>;
(endpoint: 'cluster.putSettings', params: ClusterPutSettingsParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['putSettings']>;
(endpoint: 'cluster.reroute', params: ClusterRerouteParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['reroute']>;
(endpoint: 'cluster.state', params: ClusterStateParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['state']>;
(endpoint: 'cluster.stats', params: ClusterStatsParams, options?: CallClusterOptions): ReturnType<ESClient['cluster']['stats']>;
// indices namespace
(endpoint: 'indices.analyze', params: IndicesAnalyzeParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['analyze']>;
(endpoint: 'indices.clearCache', params: IndicesClearCacheParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['clearCache']>;
(endpoint: 'indices.close', params: IndicesCloseParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['close']>;
(endpoint: 'indices.create', params: IndicesCreateParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['create']>;
(endpoint: 'indices.delete', params: IndicesDeleteParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['delete']>;
(endpoint: 'indices.deleteAlias', params: IndicesDeleteAliasParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['deleteAlias']>;
(endpoint: 'indices.deleteTemplate', params: IndicesDeleteTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['deleteTemplate']>;
(endpoint: 'indices.exists', params: IndicesExistsParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['exists']>;
(endpoint: 'indices.existsAlias', params: IndicesExistsAliasParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['existsAlias']>;
(endpoint: 'indices.existsTemplate', params: IndicesExistsTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['existsTemplate']>;
(endpoint: 'indices.existsType', params: IndicesExistsTypeParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['existsType']>;
(endpoint: 'indices.flush', params: IndicesFlushParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['flush']>;
(endpoint: 'indices.flushSynced', params: IndicesFlushSyncedParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['flushSynced']>;
(endpoint: 'indices.forcemerge', params: IndicesForcemergeParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['forcemerge']>;
(endpoint: 'indices.get', params: IndicesGetParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['get']>;
(endpoint: 'indices.getAlias', params: IndicesGetAliasParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['getAlias']>;
(endpoint: 'indices.getFieldMapping', params: IndicesGetFieldMappingParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['getFieldMapping']>;
(endpoint: 'indices.getMapping', params: IndicesGetMappingParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['getMapping']>;
(endpoint: 'indices.getSettings', params: IndicesGetSettingsParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['getSettings']>;
(endpoint: 'indices.getTemplate', params: IndicesGetTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['getTemplate']>;
(endpoint: 'indices.getUpgrade', params: IndicesGetUpgradeParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['getUpgrade']>;
(endpoint: 'indices.open', params: IndicesOpenParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['open']>;
(endpoint: 'indices.putAlias', params: IndicesPutAliasParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['putAlias']>;
(endpoint: 'indices.putMapping', params: IndicesPutMappingParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['putMapping']>;
(endpoint: 'indices.putSettings', params: IndicesPutSettingsParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['putSettings']>;
(endpoint: 'indices.putTemplate', params: IndicesPutTemplateParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['putTemplate']>;
(endpoint: 'indices.recovery', params: IndicesRecoveryParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['recovery']>;
(endpoint: 'indices.refresh', params: IndicesRefreshParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['refresh']>;
(endpoint: 'indices.rollover', params: IndicesRolloverParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['rollover']>;
(endpoint: 'indices.segments', params: IndicesSegmentsParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['segments']>;
(endpoint: 'indices.shardStores', params: IndicesShardStoresParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['shardStores']>;
(endpoint: 'indices.shrink', params: IndicesShrinkParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['shrink']>;
(endpoint: 'indices.stats', params: IndicesStatsParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['stats']>;
(endpoint: 'indices.updateAliases', params: IndicesUpdateAliasesParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['updateAliases']>;
(endpoint: 'indices.upgrade', params: IndicesUpgradeParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['upgrade']>;
(endpoint: 'indices.validateQuery', params: IndicesValidateQueryParams, options?: CallClusterOptions): ReturnType<ESClient['indices']['validateQuery']>;
// ingest namepsace
(endpoint: 'ingest.deletePipeline', params: IngestDeletePipelineParams, options?: CallClusterOptions): ReturnType<ESClient['ingest']['deletePipeline']>;
(endpoint: 'ingest.getPipeline', params: IngestGetPipelineParams, options?: CallClusterOptions): ReturnType<ESClient['ingest']['getPipeline']>;
(endpoint: 'ingest.putPipeline', params: IngestPutPipelineParams, options?: CallClusterOptions): ReturnType<ESClient['ingest']['putPipeline']>;
(endpoint: 'ingest.simulate', params: IngestSimulateParams, options?: CallClusterOptions): ReturnType<ESClient['ingest']['simulate']>;
// nodes namespace
(endpoint: 'nodes.hotThreads', params: NodesHotThreadsParams, options?: CallClusterOptions): ReturnType<ESClient['nodes']['hotThreads']>;
(endpoint: 'nodes.info', params: NodesInfoParams, options?: CallClusterOptions): ReturnType<ESClient['nodes']['info']>;
(endpoint: 'nodes.stats', params: NodesStatsParams, options?: CallClusterOptions): ReturnType<ESClient['nodes']['stats']>;
// snapshot namespace
(endpoint: 'snapshot.create', params: SnapshotCreateParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['create']>;
(endpoint: 'snapshot.createRepository', params: SnapshotCreateRepositoryParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['createRepository']>;
(endpoint: 'snapshot.delete', params: SnapshotDeleteParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['delete']>;
(endpoint: 'snapshot.deleteRepository', params: SnapshotDeleteRepositoryParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['deleteRepository']>;
(endpoint: 'snapshot.get', params: SnapshotGetParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['get']>;
(endpoint: 'snapshot.getRepository', params: SnapshotGetRepositoryParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['getRepository']>;
(endpoint: 'snapshot.restore', params: SnapshotRestoreParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['restore']>;
(endpoint: 'snapshot.status', params: SnapshotStatusParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['status']>;
(endpoint: 'snapshot.verifyRepository', params: SnapshotVerifyRepositoryParams, options?: CallClusterOptions): ReturnType<ESClient['snapshot']['verifyRepository']>;
// tasks namespace
(endpoint: 'tasks.cancel', params: TasksCancelParams, options?: CallClusterOptions): ReturnType<ESClient['tasks']['cancel']>;
(endpoint: 'tasks.get', params: TasksGetParams, options?: CallClusterOptions): ReturnType<ESClient['tasks']['get']>;
(endpoint: 'tasks.list', params: TasksListParams, options?: CallClusterOptions): ReturnType<ESClient['tasks']['list']>;
/* tslint:enable */
// other APIs accessed via transport.request
(endpoint: 'transport.request', clientParams: AssistantAPIClientParams, options?: {}): Promise<
AssistanceAPIResponse
>;
(endpoint: 'transport.request', clientParams: DeprecationAPIClientParams, options?: {}): Promise<
DeprecationAPIResponse
>;
// Catch-all definition
<T = any>(endpoint: string, clientParams: any, options?: CallClusterOptions): Promise<T>;
}
export interface ElasticsearchPlugin {
ElasticsearchClientLogging: ElasticsearchClientLogging;

12
src/server/kbn_server.d.ts vendored
View file

@ -18,9 +18,10 @@
*/
import { Server } from 'hapi';
import { CallClusterWithRequest, ElasticsearchPlugin } from '../legacy/core_plugins/elasticsearch';
import { IndexPatternsServiceFactory } from './index_patterns';
import { SavedObjectsService } from './saved_objects';
import { SavedObjectsClient, SavedObjectsService } from './saved_objects';
export interface KibanaConfig {
get<T>(key: string): T;
@ -31,6 +32,7 @@ declare module 'hapi' {
interface PluginProperties {
elasticsearch: ElasticsearchPlugin;
kibana: any;
spaces: any;
// add new plugin types here
}
@ -41,11 +43,9 @@ declare module 'hapi' {
}
interface Request {
getBasePath: () => string;
}
interface Request {
getUiSettingsService: () => any;
getSavedObjectsClient(): SavedObjectsClient;
getBasePath(): string;
getUiSettingsService(): any;
}
}

7
src/server/saved_objects/service/create_saved_objects_service.d.ts vendored
View file

@ -17,19 +17,16 @@
* under the License.
*/
import { SavedObjectsRepository, ScopedSavedObjectsClientProvider } from './lib';
import { ScopedSavedObjectsClientProvider } from './lib';
import { SavedObjectsClient } from './saved_objects_client';
export interface SavedObjectsService<Request = any> {
// ATTENTION: these types are incomplete
addScopedSavedObjectsClientWrapperFactory: ScopedSavedObjectsClientProvider<
Request
>['addClientWrapperFactory'];
getSavedObjectsRepository: (
callCluster: (endpoint: string, clientParams: any, options: any) => Promise<any>
) => SavedObjectsRepository;
getScopedSavedObjectsClient: ScopedSavedObjectsClientProvider<Request>['getClient'];
SavedObjectsClient: typeof SavedObjectsClient;
types: string[];
getSavedObjectsRepository(...rest: any[]): any;
}

2
src/server/saved_objects/service/lib/repository.js
View file

@ -313,7 +313,7 @@ export class SavedObjectsRepository {
}
if (fields && !Array.isArray(fields)) {
throw new TypeError('options.searchFields must be an array');
throw new TypeError('options.fields must be an array');
}
const esOptions = {

65
src/server/saved_objects/service/saved_objects_client.d.ts vendored
View file

@ -28,30 +28,32 @@ export interface CreateOptions extends BaseOptions {
override?: boolean;
}
export interface BulkCreateObject {
export interface BulkCreateObject<T extends SavedObjectAttributes = any> {
id?: string;
type: string;
attributes: SavedObjectAttributes;
attributes: T;
extraDocumentProperties?: string[];
}
export interface BulkCreateResponse {
savedObjects: SavedObject[];
export interface BulkCreateResponse<T extends SavedObjectAttributes = any> {
savedObjects: Array<SavedObject<T>>;
}
export interface FindOptions extends BaseOptions {
type?: string | string[];
page?: number;
perPage?: number;
sortField?: string;
sortOrder?: string;
fields?: string[];
type?: string | string[];
search?: string;
searchFields?: string[];
}
export interface FindResponse {
saved_objects: SavedObject[];
export interface FindResponse<T extends SavedObjectAttributes = any> {
saved_objects: Array<SavedObject<T>>;
total: number;
perPage: number;
per_page: number;
page: number;
}
@ -65,15 +67,15 @@ export interface BulkGetObject {
}
export type BulkGetObjects = BulkGetObject[];
export interface BulkGetResponse {
savedObjects: SavedObject[];
export interface BulkGetResponse<T extends SavedObjectAttributes = any> {
savedObjects: Array<SavedObject<T>>;
}
export interface SavedObjectAttributes {
[key: string]: SavedObjectAttributes | string | number | boolean | null;
}
export interface SavedObject {
export interface SavedObject<T extends SavedObjectAttributes = any> {
id: string;
type: string;
version?: number;
@ -81,30 +83,41 @@ export interface SavedObject {
error?: {
message: string;
};
attributes: SavedObjectAttributes;
attributes: T;
}
export declare class SavedObjectsClient {
public static errors: typeof errors;
public errors: typeof errors;
public create: (
constructor(repository: SavedObjectsRepository);
public create<T extends SavedObjectAttributes = any>(
type: string,
attributes: SavedObjectAttributes,
attributes: T,
options?: CreateOptions
) => Promise<SavedObject>;
public bulkCreate: (
objects: BulkCreateObject[],
): Promise<SavedObject<T>>;
public bulkCreate<T extends SavedObjectAttributes = any>(
objects: Array<BulkCreateObject<T>>,
options?: CreateOptions
) => Promise<BulkCreateResponse>;
public delete: (type: string, id: string, options?: BaseOptions) => Promise<{}>;
public find: (options: FindOptions) => Promise<FindResponse>;
public bulkGet: (objects: BulkGetObjects, options?: BaseOptions) => Promise<BulkGetResponse>;
public get: (type: string, id: string, options?: BaseOptions) => Promise<SavedObject>;
public update: (
): Promise<BulkCreateResponse<T>>;
public delete(type: string, id: string, options?: BaseOptions): Promise<{}>;
public find<T extends SavedObjectAttributes = any>(
options: FindOptions
): Promise<FindResponse<T>>;
public bulkGet<T extends SavedObjectAttributes = any>(
objects: BulkGetObjects,
options?: BaseOptions
): Promise<BulkGetResponse<T>>;
public get<T extends SavedObjectAttributes = any>(
type: string,
id: string,
attributes: SavedObjectAttributes,
options?: BaseOptions
): Promise<SavedObject<T>>;
public update<T extends SavedObjectAttributes = any>(
type: string,
id: string,
attributes: Partial<T>,
options?: UpdateOptions
) => Promise<SavedObject>;
constructor(repository: SavedObjectsRepository);
): Promise<SavedObject<T>>;
}

2
x-pack/package.json
View file

@ -133,6 +133,7 @@
"@scant/router": "^0.1.0",
"@slack/client": "^4.8.0",
"@turf/boolean-contains": "6.0.1",
"@types/json-stable-stringify": "^1.0.32",
"angular-resource": "1.4.9",
"angular-sanitize": "1.6.5",
"angular-ui-ace": "0.2.3",
@ -189,6 +190,7 @@
"io-ts": "^1.4.2",
"joi": "^13.5.2",
"jquery": "^3.3.1",
"json-stable-stringify": "^1.0.1",
"jsonwebtoken": "^8.3.0",
"jstimezonedetect": "1.0.5",
"lodash": "npm:@elastic/lodash@3.10.1-kibana1",

7
x-pack/plugins/__mocks__/ui/chrome.js
View file

@ -36,8 +36,13 @@ function getInjected(key) {
}
}
function getXsrfToken() {
return 'kbn';
}
export default {
getInjected,
addBasePath,
getUiSettingsClient
getUiSettingsClient,
getXsrfToken
};

10
x-pack/plugins/spaces/server/lib/saved_objects_client/spaces_saved_objects_client.ts
View file

@ -84,7 +84,11 @@ export class SpacesSavedObjectsClient implements SavedObjectsClient {
* @property {string} [options.namespace]
* @returns {promise} - { id, type, version, attributes }
*/
public async create(type: string, attributes = {}, options: CreateOptions = {}) {
public async create<T extends SavedObjectAttributes>(
type: string,
attributes: T = {} as T,
options: CreateOptions = {}
) {
throwErrorIfTypeIsSpace(type);
throwErrorIfNamespaceSpecified(options);
@ -215,10 +219,10 @@ export class SpacesSavedObjectsClient implements SavedObjectsClient {
* @property {string} [options.namespace]
* @returns {promise}
*/
public async update(
public async update<T extends SavedObjectAttributes>(
type: string,
id: string,
attributes: SavedObjectAttributes,
attributes: Partial<T>,
options: UpdateOptions = {}
) {
throwErrorIfTypeIsSpace(type);

12
x-pack/plugins/spaces/server/routes/api/__fixtures__/create_test_handler.ts
View file

@ -5,11 +5,12 @@
*/
// @ts-ignore
import { PluginProperties, Server } from 'hapi';
import { Server } from 'hapi';
import { Legacy } from 'kibana';
import { SpacesClient } from '../../../lib/spaces_client';
import { createSpaces } from './create_spaces';
interface KibanaServer extends Server {
interface KibanaServer extends Legacy.Server {
savedObjects: any;
}
@ -46,13 +47,6 @@ const baseConfig: TestConfig = {
'xpack.spaces.maxSpaces': 1000,
};
// Merge / extend default interfaces for hapi. This is all faked out below.
declare module 'hapi' {
interface PluginProperties {
spaces: any;
}
}
export function createTestHandler(initApiFn: (server: any, preCheckLicenseImpl: any) => void) {
const teardowns: TeardownFn[] = [];

49
x-pack/plugins/upgrade_assistant/common/types.ts Normal file
View file

@ -0,0 +1,49 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
SavedObject,
SavedObjectAttributes,
} from 'src/server/saved_objects/service/saved_objects_client';
export enum ReindexStep {
// Enum values are spaced out by 10 to give us room to insert steps in between.
created = 0,
indexConsumersStopped = 10,
readonly = 20,
newIndexCreated = 30,
reindexStarted = 40,
reindexCompleted = 50,
aliasCreated = 60,
indexConsumersStarted = 70,
}
export enum ReindexStatus {
inProgress,
completed,
failed,
paused,
}
export const REINDEX_OP_TYPE = 'upgrade-assistant-reindex-operation';
export interface ReindexOperation extends SavedObjectAttributes {
indexName: string;
newIndexName: string;
status: ReindexStatus;
lastCompletedStep: ReindexStep;
locked: string | null;
reindexTaskId: string | null;
reindexTaskPercComplete: number | null;
errorMessage: string | null;
mlReindexCount: number | null;
}
export type ReindexSavedObject = SavedObject<ReindexOperation>;
export enum ReindexWarning {
allField,
booleanFields,
}

1
x-pack/plugins/upgrade_assistant/common/version.ts
View file

@ -12,3 +12,4 @@ const matches = currentVersionNum.match(/^([1-9]+)\.([0-9]+)\.([0-9]+)$/)!;
export const CURRENT_MAJOR_VERSION = matches[1];
export const NEXT_MAJOR_VERSION = (parseInt(CURRENT_MAJOR_VERSION, 10) + 1).toString();
export const PREV_MAJOR_VERSION = (parseInt(CURRENT_MAJOR_VERSION, 10) - 1).toString();

6
x-pack/plugins/upgrade_assistant/index.ts
View file

@ -15,6 +15,12 @@ export function upgradeAssistant(kibana: any) {
uiExports: {
managementSections: ['plugins/upgrade_assistant'],
styleSheetPaths: resolve(__dirname, 'public/index.scss'),
mappings: require('./mappings.json'),
savedObjectSchemas: {
'upgrade-assistant-reindex-operation': {
isNamespaceAgnostic: true,
},
},
},
publicDir: resolve(__dirname, 'public'),

13
x-pack/plugins/upgrade_assistant/mappings.json Normal file
View file

@ -0,0 +1,13 @@
{
"upgrade-assistant-reindex-operation": {
"dynamic": true,
"properties": {
"indexName": {
"type": "keyword"
},
"status": {
"type": "integer"
}
}
}
}

1
x-pack/plugins/upgrade_assistant/public/components/tabs.test.tsx
View file

@ -9,6 +9,7 @@ import { mountWithIntl } from 'test_utils/enzyme_helpers';
jest.mock('axios', () => ({
get: jest.fn(),
create: jest.fn(),
}));
import { UpgradeAssistantTabs } from './tabs';

1
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/_index.scss
View file

@ -1,4 +1,5 @@
@import './cell';
@import './reindex/index';
.upgDeprecations {
// Pull the container through the padding of EuiPageContent

22
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/cell.tsx
View file

@ -7,7 +7,6 @@
import React, { ReactNode, StatelessComponent } from 'react';
import {
EuiButton,
EuiFlexGroup,
EuiFlexItem,
EuiIcon,
@ -17,16 +16,14 @@ import {
EuiTitle,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { ReindexButton } from './reindex';
interface DeprecationCellProps {
items?: Array<{ title?: string; body: string }>;
reindexIndexName?: string;
docUrl?: string;
headline?: string;
healthColor?: string;
actions?: Array<{
label: string;
url: string;
}>;
children?: ReactNode;
}
@ -36,7 +33,7 @@ interface DeprecationCellProps {
export const DeprecationCell: StatelessComponent<DeprecationCellProps> = ({
headline,
healthColor,
actions,
reindexIndexName,
docUrl,
items = [],
children,
@ -78,14 +75,11 @@ export const DeprecationCell: StatelessComponent<DeprecationCellProps> = ({
))}
</EuiFlexItem>
{actions &&
actions.map(button => (
<EuiFlexItem key={button.url} grow={false}>
<EuiButton size="s" href={button.url} target="_blank">
{button.label}
</EuiButton>
</EuiFlexItem>
))}
{reindexIndexName && (
<EuiFlexItem grow={false}>
<ReindexButton indexName={reindexIndexName} />
</EuiFlexItem>
)}
</EuiFlexGroup>
<EuiSpacer size="s" />

28
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.test.tsx
View file

@ -10,12 +10,11 @@ import { shallowWithIntl } from 'test_utils/enzyme_helpers';
import { IndexDeprecationTableProps, IndexDeprecationTableUI } from './index_table';
describe('IndexDeprecationTable', () => {
const actions = [{ label: 'Do it', url: 'http://justdoit.com' }];
const defaultProps = {
indices: [
{ index: 'index1', details: 'Index 1 deets', actions },
{ index: 'index2', details: 'Index 2 deets', actions },
{ index: 'index3', details: 'Index 3 deets', actions },
{ index: 'index1', details: 'Index 1 deets', reindex: true },
{ index: 'index2', details: 'Index 2 deets', reindex: true },
{ index: 'index3', details: 'Index 3 deets', reindex: true },
],
} as IndexDeprecationTableProps;
@ -49,34 +48,19 @@ describe('IndexDeprecationTable', () => {
items={
Array [
Object {
"actions": Array [
Object {
"label": "Do it",
"url": "http://justdoit.com",
},
],
"details": "Index 1 deets",
"index": "index1",
"reindex": true,
},
Object {
"actions": Array [
Object {
"label": "Do it",
"url": "http://justdoit.com",
},
],
"details": "Index 2 deets",
"index": "index2",
"reindex": true,
},
Object {
"actions": Array [
Object {
"label": "Do it",
"url": "http://justdoit.com",
},
],
"details": "Index 3 deets",
"index": "index3",
"reindex": true,
},
]
}

31
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx
View file

@ -7,18 +7,16 @@
import { sortBy } from 'lodash';
import React from 'react';
import { EuiBasicTable, EuiButton } from '@elastic/eui';
import { EuiBasicTable } from '@elastic/eui';
import { injectI18n } from '@kbn/i18n/react';
import { ReindexButton } from './reindex';
const PAGE_SIZES = [10, 25, 50, 100, 250, 500, 1000];
export interface IndexDeprecationDetails {
index: string;
reindex: boolean;
details?: string;
actions?: Array<{
label: string;
url: string;
}>;
}
export interface IndexDeprecationTableProps extends ReactIntl.InjectedIntlProps {
@ -134,26 +132,21 @@ export class IndexDeprecationTableUI extends React.Component<
}
private get actionsColumn() {
// NOTE: this naive implementation assumes all indices in the table have
// the same actions (can still have different URLs). This should work for known usecases.
// NOTE: this naive implementation assumes all indices in the table are
// should show the reindex button. This should work for known usecases.
const { indices } = this.props;
if (!indices.find(i => i.actions !== undefined)) {
if (!indices.find(i => i.reindex)) {
return null;
}
const actions = indices[0].actions!;
return {
actions: actions.map((action, idx) => ({
render(index: IndexDeprecationDetails) {
const { url, label } = index.actions![idx];
return (
<EuiButton size="s" href={url} target="_blank">
{label}
</EuiButton>
);
actions: [
{
render(indexDep: IndexDeprecationDetails) {
return <ReindexButton indexName={indexDep.index} />;
},
},
})),
],
};
}
}

4
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.test.tsx
View file

@ -71,14 +71,14 @@ describe('DeprecationList', () => {
indices={
Array [
Object {
"actions": undefined,
"details": undefined,
"index": "0",
"reindex": false,
},
Object {
"actions": undefined,
"details": undefined,
"index": "1",
"reindex": false,
},
]
}

12
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx
View file

@ -10,10 +10,13 @@ import { DeprecationInfo } from 'src/legacy/core_plugins/elasticsearch';
import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis';
import { GroupByOption } from '../../../types';
import { CURRENT_MAJOR_VERSION } from 'x-pack/plugins/upgrade_assistant/common/version';
import { COLOR_MAP, LEVEL_MAP } from '../constants';
import { DeprecationCell } from './cell';
import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table';
const OLD_INDEX_MESSAGE = `Index created before ${CURRENT_MAJOR_VERSION}.0`;
const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => {
return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]);
};
@ -34,7 +37,7 @@ const MessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprecationI
<DeprecationCell
headline={deprecation.message}
healthColor={COLOR_MAP[deprecation.level]}
actions={deprecation.actions}
reindexIndexName={deprecation.message === OLD_INDEX_MESSAGE ? deprecation.index! : undefined}
docUrl={deprecation.url}
items={items}
/>
@ -83,17 +86,16 @@ export const DeprecationList: StatelessComponent<{
// If we're grouping by message and the first deprecation has an index field, show an index
// group deprecation. Otherwise, show each message.
if (currentGroupBy === GroupByOption.message && deprecations[0].index !== undefined) {
// If we're grouping by index we assume that every deprecation message is the same
// issue and that each deprecation will have an index associated with it.
// We assume that every deprecation message is the same issue (since they have the same
// message) and that each deprecation will have an index associated with it.
const indices = deprecations.map(dep => ({
index: dep.index!,
details: dep.details,
actions: dep.actions,
reindex: dep.message === OLD_INDEX_MESSAGE,
}));
return <IndexDeprecation indices={indices} deprecation={deprecations[0]} />;
} else if (currentGroupBy === GroupByOption.index) {
// If we're grouping by index show all info for each message
return (
<div>
{deprecations.sort(sortByLevelDesc).map(dep => (

5
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/_button.scss Normal file
View file

@ -0,0 +1,5 @@
.upgReindexButton__spinner {
position: relative;
top: $euiSizeXS / 2;
margin-right: $euiSizeXS;
}

2
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/_index.scss Normal file
View file

@ -0,0 +1,2 @@
@import './button';
@import './flyout/index';

143
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/button.tsx Normal file
View file

@ -0,0 +1,143 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment, ReactNode } from 'react';
import { Subscription } from 'rxjs';
import { EuiButton, EuiLoadingSpinner } from '@elastic/eui';
import { ReindexStatus } from '../../../../../../common/types';
import { LoadingState } from '../../../../types';
import { ReindexFlyout } from './flyout';
import { ReindexPollingService, ReindexState } from './polling_service';
interface ReindexButtonProps {
indexName: string;
}
interface ReindexButtonState {
flyoutVisible: boolean;
reindexState: ReindexState;
}
/**
* Displays a button that will display a flyout when clicked with the reindexing status for
* the given `indexName`.
*/
export class ReindexButton extends React.Component<ReindexButtonProps, ReindexButtonState> {
private service: ReindexPollingService;
private subscription?: Subscription;
constructor(props: ReindexButtonProps) {
super(props);
this.service = this.newService();
this.state = {
flyoutVisible: false,
reindexState: this.service.status$.value,
};
}
public async componentDidMount() {
this.subscribeToUpdates();
}
public async componentWillUnmount() {
this.unsubscribeToUpdates();
}
public componentDidUpdate(prevProps: ReindexButtonProps) {
if (prevProps.indexName !== this.props.indexName) {
this.unsubscribeToUpdates();
this.service = this.newService();
this.subscribeToUpdates();
}
}
public render() {
const { indexName } = this.props;
const { flyoutVisible, reindexState } = this.state;
const buttonProps: any = { size: 's', onClick: this.showFlyout };
let buttonContent: ReactNode = 'Reindex';
if (reindexState.loadingState === LoadingState.Loading) {
buttonProps.disabled = true;
buttonContent = 'Loading…';
} else {
switch (reindexState.status) {
case ReindexStatus.inProgress:
buttonContent = (
<span>
<EuiLoadingSpinner className="upgReindexButton__spinner" size="m" /> Reindexing…
</span>
);
break;
case ReindexStatus.completed:
buttonProps.color = 'secondary';
buttonProps.iconSide = 'left';
buttonProps.iconType = 'check';
buttonContent = 'Done';
break;
case ReindexStatus.failed:
buttonProps.color = 'danger';
buttonProps.iconSide = 'left';
buttonProps.iconType = 'cross';
buttonContent = 'Failed';
break;
case ReindexStatus.paused:
buttonProps.color = 'warning';
buttonProps.iconSide = 'left';
buttonProps.iconType = 'pause';
buttonContent = 'Paused';
}
}
return (
<Fragment>
<EuiButton {...buttonProps}>{buttonContent}</EuiButton>
{flyoutVisible && (
<ReindexFlyout
indexName={indexName}
closeFlyout={this.closeFlyout}
reindexState={reindexState}
startReindex={this.service.startReindex}
/>
)}
</Fragment>
);
}
private newService() {
return new ReindexPollingService(this.props.indexName);
}
private subscribeToUpdates() {
this.service.updateStatus();
this.subscription = this.service!.status$.subscribe(reindexState =>
this.setState({ reindexState })
);
}
private unsubscribeToUpdates() {
if (this.subscription) {
this.subscription.unsubscribe();
delete this.subscription;
}
if (this.service) {
this.service.stopPolling();
}
}
private showFlyout = () => {
this.setState({ flyoutVisible: true });
};
private closeFlyout = () => {
this.setState({ flyoutVisible: false });
};
}

81
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/checklist_step.test.tsx.snap Normal file
View file

@ -0,0 +1,81 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`ChecklistFlyout renders 1`] = `
<Fragment>
<EuiFlyoutBody>
<EuiCallOut
color="warning"
iconType="alert"
size="m"
title="Index is unable to ingest, update, or delete documents while reindexing"
>
<p>
If you can’t stop document updates or need to reindex into a new cluster, consider using a different upgrade strategy.
</p>
<p>
Reindexing will continue in the background, but if Kibana shuts down or restarts you will need to return to this page to resume reindexing.
</p>
</EuiCallOut>
<EuiSpacer
size="l"
/>
<EuiTitle
size="xs"
textTransform="none"
>
<h3>
Reindexing process
</h3>
</EuiTitle>
<Component
errorMessage={null}
lastCompletedStep={20}
reindexStatus={0}
reindexTaskPercComplete={null}
/>
</EuiFlyoutBody>
<EuiFlyoutFooter>
<EuiFlexGroup
alignItems="stretch"
component="div"
direction="row"
gutterSize="l"
justifyContent="spaceBetween"
responsive={true}
wrap={false}
>
<EuiFlexItem
component="div"
grow={false}
>
<EuiButtonEmpty
color="primary"
flush="left"
iconSide="left"
iconType="cross"
onClick={[MockFunction]}
type="button"
>
Close
</EuiButtonEmpty>
</EuiFlexItem>
<EuiFlexItem
component="div"
grow={false}
>
<EuiButton
color="primary"
disabled={true}
fill={true}
iconSide="left"
isLoading={true}
onClick={[MockFunction]}
type="button"
>
Reindexing…
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlyoutFooter>
</Fragment>
`;

176
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap Normal file
View file

@ -0,0 +1,176 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`WarningsFlyoutStep renders 1`] = `
<Fragment>
<EuiFlyoutBody>
<EuiCallOut
color="danger"
iconType="alert"
size="m"
title="This index requires destructive changes that can't be undone"
>
<p>
Back up your index, then proceed with the reindex by accepting each breaking change.
</p>
</EuiCallOut>
<EuiSpacer
size="l"
/>
<EuiText
grow={true}
size="m"
>
<EuiCheckbox
checked={false}
compressed={false}
disabled={false}
id="reindexWarning-0"
indeterminate={false}
label={
<strong>
<EuiCode>
_all
</EuiCode>
field will be removed
</strong>
}
onChange={[Function]}
/>
<p
className="upgWarningsStep__warningDescription"
>
The
<EuiCode>
_all
</EuiCode>
meta field is no longer supported in 7.0. Reindexing removes the
<EuiCode>
_all
</EuiCode>
field in the new index. Ensure that no application code or scripts reply on this field.
<br />
<EuiLink
color="primary"
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
target="_blank"
type="button"
>
Documentation
</EuiLink>
</p>
</EuiText>
<EuiSpacer
size="l"
/>
<EuiText
grow={true}
size="m"
>
<EuiCheckbox
checked={false}
compressed={false}
disabled={false}
id="reindexWarning-1"
indeterminate={false}
label={
<strong>
Boolean data in
<EuiCode>
_source
</EuiCode>
might change
</strong>
}
onChange={[Function]}
/>
<p
className="upgWarningsStep__warningDescription"
>
If a documents contain a boolean field that is neither
<EuiCode>
true
</EuiCode>
or
<EuiCode>
false
</EuiCode>
(for example,
<EuiCode>
"yes"
</EuiCode>
,
<EuiCode>
"on"
</EuiCode>
,
<EuiCode>
1
</EuiCode>
), reindexing converts these fields to
<EuiCode>
true
</EuiCode>
or
<EuiCode>
false
</EuiCode>
. Ensure that no application code or scripts rely on boolean fields in the deprecated format.
<br />
<EuiLink
color="primary"
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields"
target="_blank"
type="button"
>
Documentation
</EuiLink>
</p>
</EuiText>
</EuiFlyoutBody>
<EuiFlyoutFooter>
<EuiFlexGroup
alignItems="stretch"
component="div"
direction="row"
gutterSize="l"
justifyContent="spaceBetween"
responsive={true}
wrap={false}
>
<EuiFlexItem
component="div"
grow={false}
>
<EuiButtonEmpty
color="primary"
flush="left"
iconSide="left"
iconType="cross"
onClick={[MockFunction]}
type="button"
>
Cancel
</EuiButtonEmpty>
</EuiFlexItem>
<EuiFlexItem
component="div"
grow={false}
>
<EuiButton
color="danger"
disabled={true}
fill={true}
iconSide="left"
onClick={[MockFunction]}
type="button"
>
Continue with reindex
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlyoutFooter>
</Fragment>
`;

2
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/_index.scss Normal file
View file

@ -0,0 +1,2 @@
@import './step_progress';
@import './warnings_step';

45
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/_step_progress.scss Normal file
View file

@ -0,0 +1,45 @@
@import '@elastic/eui/src/components/call_out/variables';
@import '@elastic/eui/src/components/call_out/mixins';
.upgStepProgress__step {
display: flex;
align-items: center;
margin-top: $euiSize;
margin-bottom: $euiSizeS;
line-height: $euiSize;
}
.upgStepProgress__status {
@include size($euiSize);
margin-right: $euiSizeM;
}
$stepStatusToCallOutColor: (
failed: 'danger',
complete: 'success',
paused: 'warning',
);
.upgStepProgress__status--circle {
text-align: center;
border-radius: $euiSizeM;
line-height: $euiSize - 2px;
@each $status, $callOutColor in $stepStatusToCallOutColor {
&-#{$status} {
color: euiCallOutColor($callOutColor, 'foreground');
background-color: euiCallOutColor($callOutColor, 'background');
}
}
}
.upgStepProgress__title {
&--currentStep {
font-weight: $euiFontWeightBold;
}
}
.upgStepProgress__content {
display: block;
margin-left: $euiSize + $euiSizeM;
}

4
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/_warnings_step.scss Normal file
View file

@ -0,0 +1,4 @@
.upgWarningsStep__warningDescription {
margin-left: $euiSizeL;
margin-top: $euiSizeXS;
}

54
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/checklist_step.test.tsx Normal file
View file

@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { shallow } from 'enzyme';
import React from 'react';
import { ReindexStatus, ReindexStep, ReindexWarning } from '../../../../../../../common/types';
import { LoadingState } from '../../../../../types';
import { ChecklistFlyoutStep } from './checklist_step';
describe('ChecklistFlyout', () => {
const defaultProps = {
indexName: 'myIndex',
closeFlyout: jest.fn(),
confirmInputValue: 'CONFIRM',
onConfirmInputChange: jest.fn(),
startReindex: jest.fn(),
reindexState: {
loadingState: LoadingState.Success,
lastCompletedStep: ReindexStep.readonly,
status: ReindexStatus.inProgress,
reindexTaskPercComplete: null,
errorMessage: null,
reindexWarnings: [ReindexWarning.allField],
},
};
it('renders', () => {
expect(shallow(<ChecklistFlyoutStep {...defaultProps} />)).toMatchSnapshot();
});
it('disables button while reindexing', () => {
const wrapper = shallow(<ChecklistFlyoutStep {...defaultProps} />);
expect(wrapper.find('EuiButton').props().disabled).toBe(true);
});
it('calls startReindex when button is clicked', () => {
const props = {
...defaultProps,
reindexState: {
...defaultProps.reindexState,
lastCompletedStep: undefined,
status: undefined,
},
};
const wrapper = shallow(<ChecklistFlyoutStep {...props} />);
wrapper.find('EuiButton').simulate('click');
expect(props.startReindex).toHaveBeenCalled();
});
});

108
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/checklist_step.tsx Normal file
View file

@ -0,0 +1,108 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment } from 'react';
import {
EuiButton,
EuiButtonEmpty,
EuiCallOut,
EuiFlexGroup,
EuiFlexItem,
EuiFlyoutBody,
EuiFlyoutFooter,
EuiSpacer,
EuiTitle,
} from '@elastic/eui';
import { ReindexStatus } from '../../../../../../../common/types';
import { LoadingState } from '../../../../../types';
import { ReindexState } from '../polling_service';
import { ReindexProgress } from './progress';
const buttonLabel = (status?: ReindexStatus) => {
switch (status) {
case ReindexStatus.failed:
return 'Try again';
case ReindexStatus.inProgress:
return 'Reindexing…';
case ReindexStatus.completed:
return 'Done!';
case ReindexStatus.paused:
return 'Resume';
default:
return 'Run reindex';
}
};
/**
* Displays a flyout that shows the current reindexing status for a given index.
*/
export const ChecklistFlyoutStep: React.StatelessComponent<{
closeFlyout: () => void;
reindexState: ReindexState;
startReindex: () => void;
}> = ({ closeFlyout, reindexState, startReindex }) => {
const {
loadingState,
status,
reindexTaskPercComplete,
lastCompletedStep,
errorMessage,
} = reindexState;
const loading = loadingState === LoadingState.Loading || status === ReindexStatus.inProgress;
return (
<Fragment>
<EuiFlyoutBody>
<EuiCallOut
title="Index is unable to ingest, update, or delete documents while reindexing"
color="warning"
iconType="alert"
>
<p>
If you can’t stop document updates or need to reindex into a new cluster, consider using
a different upgrade strategy.
</p>
<p>
Reindexing will continue in the background, but if Kibana shuts down or restarts you
will need to return to this page to resume reindexing.
</p>
</EuiCallOut>
<EuiSpacer />
<EuiTitle size="xs">
<h3>Reindexing process</h3>
</EuiTitle>
<ReindexProgress
lastCompletedStep={lastCompletedStep}
reindexStatus={status}
reindexTaskPercComplete={reindexTaskPercComplete}
errorMessage={errorMessage}
/>
</EuiFlyoutBody>
<EuiFlyoutFooter>
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
<EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left">
Close
</EuiButtonEmpty>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiButton
fill
color={status === ReindexStatus.paused ? 'warning' : 'primary'}
iconType={status === ReindexStatus.paused ? 'play' : undefined}
onClick={startReindex}
isLoading={loading}
disabled={loading || status === ReindexStatus.completed}
>
{buttonLabel(status)}
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlyoutFooter>
</Fragment>
);
};

93
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/container.tsx Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React from 'react';
import { EuiFlyout, EuiFlyoutHeader, EuiPortal, EuiTitle } from '@elastic/eui';
import { ReindexState } from '../polling_service';
import { ChecklistFlyoutStep } from './checklist_step';
import { WarningsFlyoutStep } from './warnings_step';
enum ReindexFlyoutStep {
reindexWarnings,
checklist,
}
interface ReindexFlyoutProps {
indexName: string;
closeFlyout: () => void;
reindexState: ReindexState;
startReindex: () => void;
}
interface ReindexFlyoutState {
currentFlyoutStep: ReindexFlyoutStep;
}
/**
* Wrapper for the contents of the flyout that manages which step of the flyout to show.
*/
export class ReindexFlyout extends React.Component<ReindexFlyoutProps, ReindexFlyoutState> {
constructor(props: ReindexFlyoutProps) {
super(props);
const { status, reindexWarnings } = props.reindexState;
this.state = {
// If there are any warnings and we haven't started reindexing, show the warnings step first.
currentFlyoutStep:
reindexWarnings && reindexWarnings.length > 0 && status === undefined
? ReindexFlyoutStep.reindexWarnings
: ReindexFlyoutStep.checklist,
};
}
public render() {
const { closeFlyout, indexName, reindexState, startReindex } = this.props;
const { currentFlyoutStep } = this.state;
let flyoutContents: React.ReactNode;
switch (currentFlyoutStep) {
case ReindexFlyoutStep.reindexWarnings:
flyoutContents = (
<WarningsFlyoutStep
closeFlyout={closeFlyout}
warnings={reindexState.reindexWarnings!}
advanceNextStep={this.advanceNextStep}
/>
);
break;
case ReindexFlyoutStep.checklist:
flyoutContents = (
<ChecklistFlyoutStep
closeFlyout={closeFlyout}
reindexState={reindexState}
startReindex={startReindex}
/>
);
break;
default:
throw new Error(`Invalid flyout step: ${currentFlyoutStep}`);
}
return (
<EuiPortal>
<EuiFlyout onClose={closeFlyout} aria-labelledby="Reindex" ownFocus size="m" maxWidth>
<EuiFlyoutHeader hasBorder>
<EuiTitle size="s">
<h2>Reindex {indexName}</h2>
</EuiTitle>
</EuiFlyoutHeader>
{flyoutContents}
</EuiFlyout>
</EuiPortal>
);
}
public advanceNextStep = () => {
this.setState({ currentFlyoutStep: ReindexFlyoutStep.checklist });
};
}

7
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/index.tsx Normal file
View file

@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { ReindexFlyout } from './container';

80
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/progress.test.tsx Normal file
View file

@ -0,0 +1,80 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { shallow } from 'enzyme';
import React from 'react';
import { ReindexStatus, ReindexStep } from '../../../../../../../common/types';
import { ReindexProgress } from './progress';
describe('ReindexProgress', () => {
it('renders', () => {
const wrapper = shallow(
<ReindexProgress
lastCompletedStep={ReindexStep.created}
reindexStatus={ReindexStatus.inProgress}
reindexTaskPercComplete={null}
errorMessage={null}
/>
);
expect(wrapper).toMatchInlineSnapshot(`
<Component
steps={
Array [
Object {
"status": "incomplete",
"title": "Setting old index to read-only",
},
Object {
"status": "incomplete",
"title": "Creating new index",
},
Object {
"status": "incomplete",
"title": "Reindexing documents",
},
Object {
"status": "incomplete",
"title": "Swapping original index with alias",
},
]
}
/>
`);
});
it('displays errors in the step that failed', () => {
const wrapper = shallow(
<ReindexProgress
lastCompletedStep={ReindexStep.reindexCompleted}
reindexStatus={ReindexStatus.failed}
reindexTaskPercComplete={1}
errorMessage={`This is an error that happened on alias switch`}
/>
);
const aliasStep = wrapper.props().steps[3];
expect(aliasStep.children.props.errorMessage).toEqual(
`This is an error that happened on alias switch`
);
});
it('shows reindexing document progress bar', () => {
const wrapper = shallow(
<ReindexProgress
lastCompletedStep={ReindexStep.reindexStarted}
reindexStatus={ReindexStatus.inProgress}
reindexTaskPercComplete={0.25}
errorMessage={null}
/>
);
const reindexStep = wrapper.props().steps[2];
expect(reindexStep.children.type.name).toEqual('EuiProgress');
expect(reindexStep.children.props.value).toEqual(0.25);
});
});

123
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/progress.tsx Normal file
View file

@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React from 'react';
import { EuiCallOut, EuiProgress, EuiText } from '@elastic/eui';
import { ReindexStatus, ReindexStep } from '../../../../../../../common/types';
import { StepProgress, StepProgressStep } from './step_progress';
const ErrorCallout: React.StatelessComponent<{ errorMessage: string | null }> = ({
errorMessage,
}) => (
<EuiCallOut color="danger" title="There was an error">
<EuiText>
<p>{errorMessage}</p>
</EuiText>
</EuiCallOut>
);
const PausedCallout = () => (
<EuiCallOut
color="warning"
title="This step was paused due to a Kibana restart. Click 'Resume' below to continue."
/>
);
const orderedSteps = Object.values(ReindexStep).sort() as number[];
/**
* Displays a list of steps in the reindex operation, the current status, a progress bar,
* and any error messages that are encountered.
*/
export const ReindexProgress: React.StatelessComponent<{
lastCompletedStep?: ReindexStep;
reindexStatus?: ReindexStatus;
reindexTaskPercComplete: number | null;
errorMessage: string | null;
}> = ({ lastCompletedStep = -1, reindexStatus, reindexTaskPercComplete, errorMessage }) => {
const stepDetails = (thisStep: ReindexStep): Pick<StepProgressStep, 'status' | 'children'> => {
const previousStep = orderedSteps[orderedSteps.indexOf(thisStep) - 1];
if (reindexStatus === ReindexStatus.failed && lastCompletedStep === previousStep) {
return {
status: 'failed',
children: <ErrorCallout {...{ errorMessage }} />,
};
} else if (reindexStatus === ReindexStatus.paused && lastCompletedStep === previousStep) {
return {
status: 'paused',
children: <PausedCallout />,
};
} else if (reindexStatus === undefined || lastCompletedStep < previousStep) {
return {
status: 'incomplete',
};
} else if (lastCompletedStep === previousStep) {
return {
status: 'inProgress',
};
} else {
return {
status: 'complete',
};
}
};
// The reindexing step is special because it combines the starting and complete statuses into a single UI
// with a progress bar.
const reindexingDocsStep = { title: 'Reindexing documents' } as StepProgressStep;
if (
reindexStatus === ReindexStatus.failed &&
(lastCompletedStep === ReindexStep.newIndexCreated ||
lastCompletedStep === ReindexStep.reindexStarted)
) {
reindexingDocsStep.status = 'failed';
reindexingDocsStep.children = <ErrorCallout {...{ errorMessage }} />;
} else if (
reindexStatus === ReindexStatus.paused &&
(lastCompletedStep === ReindexStep.newIndexCreated ||
lastCompletedStep === ReindexStep.reindexStarted)
) {
reindexingDocsStep.status = 'paused';
reindexingDocsStep.children = <PausedCallout />;
} else if (reindexStatus === undefined || lastCompletedStep < ReindexStep.newIndexCreated) {
reindexingDocsStep.status = 'incomplete';
} else {
reindexingDocsStep.status =
lastCompletedStep === ReindexStep.newIndexCreated ||
lastCompletedStep === ReindexStep.reindexStarted
? 'inProgress'
: 'complete';
reindexingDocsStep.children = reindexTaskPercComplete ? (
<EuiProgress size="s" value={reindexTaskPercComplete} max={1} />
) : (
<EuiProgress size="s" />
);
}
return (
<StepProgress
steps={[
{
title: 'Setting old index to read-only',
...stepDetails(ReindexStep.readonly),
},
{
title: 'Creating new index',
...stepDetails(ReindexStep.newIndexCreated),
},
reindexingDocsStep,
{
title: 'Swapping original index with alias',
...stepDetails(ReindexStep.aliasCreated),
},
]}
/>
);
};

83
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/step_progress.tsx Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import classNames from 'classnames';
import React, { Fragment, ReactNode } from 'react';
import { EuiIcon, EuiLoadingSpinner } from '@elastic/eui';
type STATUS = 'incomplete' | 'inProgress' | 'complete' | 'failed' | 'paused';
const StepStatus: React.StatelessComponent<{ status: STATUS; idx: number }> = ({ status, idx }) => {
if (status === 'incomplete') {
return <span className="upgStepProgress__status">{idx + 1}.</span>;
} else if (status === 'inProgress') {
return <EuiLoadingSpinner size="m" className="upgStepProgress__status" />;
} else if (status === 'complete') {
return (
<span className="upgStepProgress__status upgStepProgress__status--circle upgStepProgress__status--circle-complete">
<EuiIcon type="check" size="s" />
</span>
);
} else if (status === 'paused') {
return (
<span className="upgStepProgress__status upgStepProgress__status--circle upgStepProgress__status--circle-paused">
<EuiIcon type="pause" size="s" />
</span>
);
} else if (status === 'failed') {
return (
<span className="upgStepProgress__status upgStepProgress__status--circle upgStepProgress__status--circle-failed">
<EuiIcon type="cross" size="s" />
</span>
);
}
throw new Error(`Unsupported status: ${status}`);
};
const Step: React.StatelessComponent<StepProgressStep & { idx: number }> = ({
title,
status,
children,
idx,
}) => {
const titleClassName = classNames('upgStepProgress__title', {
'upgStepProgress__title--currentStep':
status === 'inProgress' || status === 'paused' || status === 'failed',
});
return (
<Fragment>
<div className="upgStepProgress__step">
<StepStatus status={status} idx={idx} />
<p className={titleClassName}>{title}</p>
</div>
{children && <div className="upgStepProgress__content">{children}</div>}
</Fragment>
);
};
export interface StepProgressStep {
title: string;
status: STATUS;
children?: ReactNode;
}
/**
* A generic component that displays a series of automated steps and the system's progress.
*/
export const StepProgress: React.StatelessComponent<{
steps: StepProgressStep[];
}> = ({ steps }) => {
return (
<div className="upgStepProgress__container">
{steps.map((step, idx) => (
<Step key={step.title} {...step} idx={idx} />
))}
</div>
);
};

39
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx Normal file
View file

@ -0,0 +1,39 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { mount, shallow } from 'enzyme';
import React from 'react';
import { ReindexWarning } from '../../../../../../../common/types';
import { idForWarning, WarningsFlyoutStep } from './warnings_step';
describe('WarningsFlyoutStep', () => {
const defaultProps = {
advanceNextStep: jest.fn(),
warnings: [ReindexWarning.allField, ReindexWarning.booleanFields],
closeFlyout: jest.fn(),
};
it('renders', () => {
expect(shallow(<WarningsFlyoutStep {...defaultProps} />)).toMatchSnapshot();
});
it('does not allow proceeding until all are checked', () => {
const wrapper = mount(<WarningsFlyoutStep {...defaultProps} />);
const button = wrapper.find('EuiButton');
button.simulate('click');
expect(defaultProps.advanceNextStep).not.toHaveBeenCalled();
wrapper.find(`input#${idForWarning(ReindexWarning.allField)}`).simulate('change');
button.simulate('click');
expect(defaultProps.advanceNextStep).not.toHaveBeenCalled();
wrapper.find(`input#${idForWarning(ReindexWarning.booleanFields)}`).simulate('change');
button.simulate('click');
expect(defaultProps.advanceNextStep).toHaveBeenCalled();
});
});

168
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx Normal file
View file

@ -0,0 +1,168 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment } from 'react';
import {
EuiButton,
EuiButtonEmpty,
EuiCallOut,
EuiCheckbox,
EuiCode,
EuiFlexGroup,
EuiFlexItem,
EuiFlyoutBody,
EuiFlyoutFooter,
EuiLink,
EuiSpacer,
EuiText,
} from '@elastic/eui';
import { ReindexWarning } from '../../../../../../../common/types';
export const idForWarning = (warning: ReindexWarning) => `reindexWarning-${warning}`;
interface WarningsConfirmationFlyoutProps {
closeFlyout: () => void;
warnings: ReindexWarning[];
advanceNextStep: () => void;
}
interface WarningsConfirmationFlyoutState {
checkedIds: { [id: string]: boolean };
}
/**
* Displays warning text about destructive changes required to reindex this index. The user
* must acknowledge each change before being allowed to proceed.
*/
export class WarningsFlyoutStep extends React.Component<
WarningsConfirmationFlyoutProps,
WarningsConfirmationFlyoutState
> {
constructor(props: WarningsConfirmationFlyoutProps) {
super(props);
this.state = {
checkedIds: props.warnings.reduce(
(checkedIds, warning) => {
checkedIds[idForWarning(warning)] = false;
return checkedIds;
},
{} as { [id: string]: boolean }
),
};
}
public render() {
const { warnings, closeFlyout, advanceNextStep } = this.props;
const { checkedIds } = this.state;
// Do not allow to proceed until all checkboxes are checked.
const blockAdvance = Object.values(checkedIds).filter(v => v).length < warnings.length;
return (
<Fragment>
<EuiFlyoutBody>
<EuiCallOut
title="This index requires destructive changes that can't be undone"
color="danger"
iconType="alert"
>
<p>
Back up your index, then proceed with the reindex by accepting each breaking change.
</p>
</EuiCallOut>
<EuiSpacer />
{warnings.includes(ReindexWarning.allField) && (
<EuiText>
<EuiCheckbox
id={idForWarning(ReindexWarning.allField)}
label={
<strong>
<EuiCode>_all</EuiCode> field will be removed
</strong>
}
checked={checkedIds[idForWarning(ReindexWarning.allField)]}
onChange={this.onChange}
/>
<p className="upgWarningsStep__warningDescription">
The <EuiCode>_all</EuiCode> meta field is no longer supported in 7.0. Reindexing
removes the <EuiCode>_all</EuiCode> field in the new index. Ensure that no
application code or scripts reply on this field.
<br />
<EuiLink
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
target="_blank"
>
Documentation
</EuiLink>
</p>
</EuiText>
)}
<EuiSpacer />
{warnings.includes(ReindexWarning.booleanFields) && (
<EuiText>
<EuiCheckbox
id={idForWarning(ReindexWarning.booleanFields)}
label={
<strong>
Boolean data in <EuiCode>_source</EuiCode> might change
</strong>
}
checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]}
onChange={this.onChange}
/>
<p className="upgWarningsStep__warningDescription">
If a documents contain a boolean field that is neither <EuiCode>true</EuiCode> or{' '}
<EuiCode>false</EuiCode> (for example, <EuiCode>"yes"</EuiCode>,{' '}
<EuiCode>"on"</EuiCode>, <EuiCode>1</EuiCode>), reindexing converts these fields to{' '}
<EuiCode>true</EuiCode> or <EuiCode>false</EuiCode>. Ensure that no application code
or scripts rely on boolean fields in the deprecated format.
<br />
<EuiLink
href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields"
target="_blank"
>
Documentation
</EuiLink>
</p>
</EuiText>
)}
</EuiFlyoutBody>
<EuiFlyoutFooter>
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
<EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left">
Cancel
</EuiButtonEmpty>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiButton fill color="danger" onClick={advanceNextStep} disabled={blockAdvance}>
Continue with reindex
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlyoutFooter>
</Fragment>
);
}
private onChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const optionId = e.target.id;
const nextCheckedIds = {
...this.state.checkedIds,
...{
[optionId]: !this.state.checkedIds[optionId],
},
};
this.setState({ checkedIds: nextCheckedIds });
};
}

7
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/index.tsx Normal file
View file

@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { ReindexButton } from './button';

101
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/polling_service.test.ts Normal file
View file

@ -0,0 +1,101 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ReindexStatus, ReindexStep } from '../../../../../../common/types';
const mockClient = {
post: jest.fn().mockResolvedValue({
lastCompletedStep: ReindexStep.created,
status: ReindexStatus.inProgress,
}),
get: jest.fn().mockResolvedValue({
status: 200,
data: {
warnings: [],
reindexOp: null,
},
}),
};
jest.mock('axios', () => ({
create: jest.fn().mockReturnValue(mockClient),
}));
import { ReindexPollingService } from './polling_service';
describe('ReindexPollingService', () => {
beforeEach(() => {
mockClient.post.mockReset();
mockClient.get.mockReset();
});
it('does not poll when reindexOp is null', async () => {
mockClient.get.mockResolvedValueOnce({
status: 200,
data: {
warnings: [],
reindexOp: null,
},
});
const service = new ReindexPollingService('myIndex');
service.updateStatus();
await new Promise(resolve => setTimeout(resolve, 1200)); // wait for poll interval
expect(mockClient.get).toHaveBeenCalledTimes(1);
service.stopPolling();
});
it('does not poll when first check is a 200 and status is failed', async () => {
mockClient.get.mockResolvedValue({
status: 200,
data: {
warnings: [],
reindexOp: {
lastCompletedStep: ReindexStep.created,
status: ReindexStatus.failed,
errorMessage: `Oh no!`,
},
},
});
const service = new ReindexPollingService('myIndex');
service.updateStatus();
await new Promise(resolve => setTimeout(resolve, 1200)); // wait for poll interval
expect(mockClient.get).toHaveBeenCalledTimes(1);
expect(service.status$.value.errorMessage).toEqual(`Oh no!`);
service.stopPolling();
});
it('begins to poll when first check is a 200 and status is inProgress', async () => {
mockClient.get.mockResolvedValue({
status: 200,
data: {
warnings: [],
reindexOp: {
lastCompletedStep: ReindexStep.created,
status: ReindexStatus.inProgress,
},
},
});
const service = new ReindexPollingService('myIndex');
service.updateStatus();
await new Promise(resolve => setTimeout(resolve, 1200)); // wait for poll interval
expect(mockClient.get).toHaveBeenCalledTimes(2);
service.stopPolling();
});
describe('startReindex', () => {
it('posts to endpoint', async () => {
const service = new ReindexPollingService('myIndex');
await service.startReindex();
expect(mockClient.post).toHaveBeenCalledWith('/api/upgrade_assistant/reindex/myIndex');
});
});
});

135
x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/polling_service.ts Normal file
View file

@ -0,0 +1,135 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import axios from 'axios';
import chrome from 'ui/chrome';
import { BehaviorSubject } from 'rxjs';
import {
ReindexOperation,
ReindexStatus,
ReindexStep,
ReindexWarning,
} from '../../../../../../common/types';
import { LoadingState } from '../../../../types';
const POLL_INTERVAL = 1000;
const XSRF = chrome.getXsrfToken();
export const APIClient = axios.create({
headers: {
Accept: 'application/json',
credentials: 'same-origin',
'Content-Type': 'application/json',
'kbn-version': XSRF,
'kbn-xsrf': XSRF,
},
});
export interface ReindexState {
loadingState: LoadingState;
lastCompletedStep?: ReindexStep;
status?: ReindexStatus;
reindexTaskPercComplete: number | null;
errorMessage: string | null;
reindexWarnings?: ReindexWarning[];
}
interface StatusResponse {
warnings?: ReindexWarning[];
reindexOp?: ReindexOperation;
}
/**
* Service used by the frontend to start reindexing and get updates on the state of a reindex
* operation. Exposes an Observable that can be used to subscribe to state updates.
*/
export class ReindexPollingService {
public status$: BehaviorSubject<ReindexState>;
private pollTimeout?: NodeJS.Timeout;
constructor(private indexName: string) {
this.status$ = new BehaviorSubject<ReindexState>({
loadingState: LoadingState.Loading,
errorMessage: null,
reindexTaskPercComplete: null,
});
}
public updateStatus = async () => {
// Prevent two loops from being started.
this.stopPolling();
try {
const { data } = await APIClient.get<StatusResponse>(
chrome.addBasePath(`/api/upgrade_assistant/reindex/${this.indexName}`)
);
this.updateWithResponse(data);
// Only keep polling if it exists and is in progress.
if (data.reindexOp && data.reindexOp.status === ReindexStatus.inProgress) {
this.pollTimeout = setTimeout(this.updateStatus, POLL_INTERVAL);
}
} catch (e) {
this.status$.next({
...this.status$.value,
status: ReindexStatus.failed,
});
}
};
public stopPolling = () => {
if (this.pollTimeout) {
clearTimeout(this.pollTimeout);
}
};
public startReindex = async () => {
try {
// Optimistically assume it will start, reset other state.
const currentValue = this.status$.value;
this.status$.next({
...currentValue,
// Only reset last completed step if we aren't currently paused
lastCompletedStep:
currentValue.status === ReindexStatus.paused ? currentValue.lastCompletedStep : undefined,
status: ReindexStatus.inProgress,
reindexTaskPercComplete: null,
errorMessage: null,
});
const { data } = await APIClient.post<ReindexOperation>(
chrome.addBasePath(`/api/upgrade_assistant/reindex/${this.indexName}`)
);
this.updateWithResponse({ reindexOp: data });
this.updateStatus();
} catch (e) {
this.status$.next({ ...this.status$.value, status: ReindexStatus.failed });
}
};
private updateWithResponse = ({ reindexOp, warnings }: StatusResponse) => {
// Next value should always include the entire state, not just what changes.
// We make a shallow copy as a starting new state.
const nextValue = {
...this.status$.value,
// If we're getting any updates, set to success.
loadingState: LoadingState.Success,
};
if (warnings) {
nextValue.reindexWarnings = warnings;
}
if (reindexOp) {
nextValue.lastCompletedStep = reindexOp.lastCompletedStep;
nextValue.status = reindexOp.status;
nextValue.reindexTaskPercComplete = reindexOp.reindexTaskPercComplete;
nextValue.errorMessage = reindexOp.errorMessage;
}
this.status$.next(nextValue);
};
}

13
x-pack/plugins/upgrade_assistant/server/index.ts
View file

@ -5,10 +5,23 @@
*/
import { Legacy } from 'kibana';
import { credentialStoreFactory } from './lib/reindexing/credential_store';
import { registerClusterCheckupRoutes } from './routes/cluster_checkup';
import { registerDeprecationLoggingRoutes } from './routes/deprecation_logging';
import { registerReindexIndicesRoutes, registerReindexWorker } from './routes/reindex_indices';
export function initServer(server: Legacy.Server) {
registerClusterCheckupRoutes(server);
registerDeprecationLoggingRoutes(server);
// The ReindexWorker uses a map of request headers that contain the authentication credentials
// for a given reindex. We cannot currently store these in an the .kibana index b/c we do not
// want to expose these credentials to any unauthenticated users. We also want to avoid any need
// to add a user for a special index just for upgrading. This in-memory cache allows us to
// process jobs without the browser staying on the page, but will require that jobs go into
// a paused state if no Kibana nodes have the required credentials.
const credentialStore = credentialStoreFactory();
const worker = registerReindexWorker(server, credentialStore);
registerReindexIndicesRoutes(server, worker, credentialStore);
}

4
x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts
View file

@ -12,10 +12,6 @@ import { DeprecationAPIResponse, DeprecationInfo } from 'src/legacy/core_plugins
export interface EnrichedDeprecationInfo extends DeprecationInfo {
index?: string;
node?: string;
actions?: Array<{
label: string;
url: string;
}>;
}
export interface UpgradeAssistantStatus {

36
x-pack/plugins/upgrade_assistant/server/lib/reindexing/credential_store.test.ts Normal file
View file

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ReindexSavedObject } from '../../../common/types';
import { Credential, credentialStoreFactory } from './credential_store';
describe('credentialStore', () => {
it('retrieves the same credentials for the same state', () => {
const creds = { key: '1' } as Credential;
const reindexOp = {
id: 'asdf',
attributes: { indexName: 'test', lastCompletedStep: 1, locked: null },
} as ReindexSavedObject;
const credStore = credentialStoreFactory();
credStore.set(reindexOp, creds);
expect(credStore.get(reindexOp)).toEqual(creds);
});
it('does retrieve credentials if the state is changed', () => {
const creds = { key: '1' } as Credential;
const reindexOp = {
id: 'asdf',
attributes: { indexName: 'test', lastCompletedStep: 1, locked: null },
} as ReindexSavedObject;
const credStore = credentialStoreFactory();
credStore.set(reindexOp, creds);
reindexOp.attributes.lastCompletedStep = 0;
expect(credStore.get(reindexOp)).not.toBeDefined();
});
});

51
x-pack/plugins/upgrade_assistant/server/lib/reindexing/credential_store.ts Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { createHash } from 'crypto';
import { Request } from 'hapi';
import stringify from 'json-stable-stringify';
import { ReindexSavedObject } from '../../../common/types';
export type Credential = Request['headers'];
/**
* An in-memory cache for user credentials to be used for reindexing operations. When looking up
* credentials, the reindex operation must be in the same state it was in when the credentials
* were stored. This prevents any tampering of the .kibana index by an unpriviledged user from
* affecting the reindex process.
*/
export interface CredentialStore {
get(reindexOp: ReindexSavedObject): Credential | undefined;
set(reindexOp: ReindexSavedObject, credential: Credential): void;
clear(): void;
}
export const credentialStoreFactory = (): CredentialStore => {
const credMap = new Map<string, Credential>();
// Generates a stable hash for the reindex operation's current state.
const getHash = (reindexOp: ReindexSavedObject) =>
createHash('sha256')
.update(stringify({ id: reindexOp.id, ...reindexOp.attributes }))
.digest('base64');
return {
get(reindexOp: ReindexSavedObject) {
return credMap.get(getHash(reindexOp));
},
set(reindexOp: ReindexSavedObject, credential: Credential) {
credMap.set(getHash(reindexOp), credential);
},
clear() {
for (const k of credMap.keys()) {
credMap.delete(k);
}
},
};
};

8
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { reindexServiceFactory } from './reindex_service';
export { ReindexWorker } from './worker';

244
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts Normal file
View file

@ -0,0 +1,244 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ReindexWarning } from '../../../common/types';
import { findBooleanFields, getReindexWarnings, transformFlatSettings } from './index_settings';
describe('transformFlatSettings', () => {
it('does not blow up for empty mappings', () => {
expect(
transformFlatSettings({
settings: {},
mappings: {},
})
).toEqual({
settings: {},
mappings: {},
});
});
it('removes settings that cannot be set on a new index', () => {
expect(
transformFlatSettings({
settings: {
// Settings that should get preserved
'index.number_of_replicas': '1',
'index.number_of_shards': '5',
// Blacklisted settings
'index.uuid': 'i66b9149a-00ee-42d9-8ca1-85ae927924bf',
'index.blocks.write': 'true',
'index.creation_date': '1547052614626',
'index.legacy': '6',
'index.mapping.single_type': 'true',
'index.provided_name': 'test1',
'index.routing.allocation.initial_recovery._id': '1',
'index.version.created': '123123',
'index.version.upgraded': '123123',
},
mappings: {},
})
).toEqual({
settings: {
'index.number_of_replicas': '1',
'index.number_of_shards': '5',
},
mappings: {},
});
});
it('fixes negative values of delayed_timeout', () => {
expect(
transformFlatSettings({
settings: {
'index.unassigned.node_left.delayed_timeout': '-10',
},
mappings: {},
})
).toEqual({
settings: {
'index.unassigned.node_left.delayed_timeout': '0',
},
mappings: {},
});
});
it('does not allow index.shard.check_on_startup to be set to "fix"', () => {
expect(() =>
transformFlatSettings({
settings: {
'index.shard.check_on_startup': 'fix',
},
mappings: {},
})
).toThrowError(`index.shard.check_on_startup cannot be set to 'fix'`);
});
it('does not allow index.percolator.map_unmapped_fields_as_string to be set', () => {
expect(() =>
transformFlatSettings({
settings: {
'index.percolator.map_unmapped_fields_as_string': 'blah',
},
mappings: {},
})
).toThrowError(`index.percolator.map_unmapped_fields_as_string is no longer supported.`);
});
it('removes _default_ mapping types', () => {
expect(
transformFlatSettings({
settings: {},
mappings: {
_default_: {},
myType: {},
},
})
).toEqual({
settings: {},
mappings: {
myType: {},
},
});
});
it('does not allow multiple mapping types', () => {
expect(() =>
transformFlatSettings({
settings: {},
mappings: {
myType1: {},
myType2: {},
},
})
).toThrowError(`Indices with more than one mapping type are not supported in 7.0.`);
});
it('removes _all.enablead = false', () => {
expect(
transformFlatSettings({
settings: {},
mappings: {
myType: {
_all: { enabled: false },
},
},
})
).toEqual({
settings: {},
mappings: {
myType: {},
},
});
});
it('removes _all.enablead = true', () => {
expect(
transformFlatSettings({
settings: {},
mappings: {
myType: {
_all: { enabled: true },
},
},
})
).toEqual({
settings: {},
mappings: {
myType: {},
},
});
});
});
describe('getReindexWarnings', () => {
it('fails if there are multiple mapping types', () => {
expect(() =>
getReindexWarnings({
settings: {},
mappings: {
myType1: {},
myType2: {},
},
})
).toThrowError();
});
it('does not fail if there is a _default_ mapping', () => {
expect(
getReindexWarnings({
settings: {},
mappings: {
_default_: {},
myType1: {},
},
})
).toEqual([]);
});
it('does not blow up for empty mappings', () => {
expect(
getReindexWarnings({
settings: {},
mappings: {},
})
).toEqual([]);
});
it('returns allField if has _all in mapping type', () => {
expect(
getReindexWarnings({
settings: {},
mappings: {
myType: {
_all: { enabled: true },
},
},
})
).toEqual([ReindexWarning.allField]);
});
it('returns booleanFields if mapping type has any boolean fields', () => {
expect(
getReindexWarnings({
settings: {},
mappings: {
myType: {
properties: {
field1: { type: 'boolean' },
},
},
},
})
).toEqual([ReindexWarning.booleanFields]);
});
});
describe('findBooleanFields', () => {
it('returns nested fields', () => {
const mappingProperties = {
region: {
type: 'boolean',
},
manager: {
properties: {
age: { type: 'boolean' },
name: {
properties: {
first: { type: 'text' },
last: { type: 'boolean' },
},
},
},
},
};
expect(findBooleanFields(mappingProperties)).toEqual([
['region'],
['manager', 'age'],
['manager', 'name', 'last'],
]);
});
});

140
x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts Normal file
View file

@ -0,0 +1,140 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { flow, omit } from 'lodash';
import { ReindexWarning } from '../../../common/types';
import { FlatSettings, MappingProperties, TypeMapping } from './types';
/**
* Validates, and updates deprecated settings and mappings to be applied to the
* new updated index.
*/
export const transformFlatSettings = (flatSettings: FlatSettings) => {
const settings = transformSettings(flatSettings.settings);
const mappings = transformMappings(flatSettings.mappings);
return { settings, mappings };
};
/**
* Returns an array of warnings that should be displayed to user before reindexing begins.
* @param flatSettings
*/
export const getReindexWarnings = (flatSettings: FlatSettings): ReindexWarning[] => {
const mapping = getSingleMappingType(flatSettings.mappings);
const warnings = [
[ReindexWarning.allField, Boolean(mapping && mapping._all && mapping._all.enabled)],
[
ReindexWarning.booleanFields,
Boolean(mapping && mapping.properties && findBooleanFields(mapping.properties).length > 0),
],
] as Array<[ReindexWarning, boolean]>;
return warnings.filter(([_, applies]) => applies).map(([warning, _]) => warning);
};
/**
* Returns an array of field paths for all boolean fields, where each field path is an array of strings.
* Example:
* For the mapping type:
* ```
* {
* "field1": { "type": "boolean" },
* "nested": {
* "field2": { "type": "boolean" }
* }
* }
* ```
* The fieldPaths would be: `[['field1'], ['nested', 'field2']]`
* @param properties
*/
export const findBooleanFields = (properties: MappingProperties): string[][] =>
Object.keys(properties).reduce(
(res, propertyName) => {
if (properties[propertyName].type === 'boolean') {
// If this field is a boolean, add it
res.push([propertyName]);
} else if (properties[propertyName].properties) {
// If this is a nested object/array get the nested fields and prepend the field path with the current field.
const nested = findBooleanFields(properties[propertyName].properties!);
res = [...res, ...nested.map(n => [propertyName, ...n])];
}
return res;
},
[] as string[][]
);
const removeUnsettableSettings = (settings: FlatSettings['settings']) =>
omit(settings, [
'index.uuid',
'index.blocks.write',
'index.creation_date',
'index.legacy',
'index.mapping.single_type',
'index.provided_name',
'index.routing.allocation.initial_recovery._id',
'index.version.created',
'index.version.upgraded',
]);
const updateFixableSettings = (settings: FlatSettings['settings']) => {
const delayedTimeout = settings['index.unassigned.node_left.delayed_timeout'];
if (delayedTimeout && parseInt(delayedTimeout, 10) < 0) {
settings['index.unassigned.node_left.delayed_timeout'] = '0';
}
return settings;
};
const validateSettings = (settings: FlatSettings['settings']) => {
if (settings['index.shard.check_on_startup'] === 'fix') {
throw new Error(`index.shard.check_on_startup cannot be set to 'fix'`);
}
if (settings['index.percolator.map_unmapped_fields_as_string']) {
throw new Error(`index.percolator.map_unmapped_fields_as_string is no longer supported.`);
}
return settings;
};
// Use `flow` to pipe the settings through each function.
const transformSettings = flow(
removeUnsettableSettings,
updateFixableSettings,
validateSettings
);
const updateFixableMappings = (mappings: FlatSettings['mappings']) => {
if (mappings._default_) {
delete mappings._default_;
}
const mapping = getSingleMappingType(mappings);
if (mapping && mapping._all) {
delete mapping._all;
}
return mappings;
};
const transformMappings = flow(updateFixableMappings);
export const getSingleMappingType = (
mappings: FlatSettings['mappings']
): TypeMapping | undefined => {
const mappingTypes = Object.keys(mappings)
// Ignore _default_ mapping types.
.filter(t => t !== '_default_');
if (mappingTypes.length > 1) {
throw new Error(`Indices with more than one mapping type are not supported in 7.0.`);
}
return mappings[mappingTypes[0]];
};

349
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts Normal file
View file

@ -0,0 +1,349 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import moment from 'moment';
import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
import {
REINDEX_OP_TYPE,
ReindexSavedObject,
ReindexStatus,
ReindexStep,
} from 'x-pack/plugins/upgrade_assistant/common/types';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
} from 'x-pack/plugins/upgrade_assistant/common/version';
import {
LOCK_WINDOW,
ML_LOCK_DOC_ID,
ReindexActions,
reindexActionsFactory,
} from './reindex_actions';
describe('ReindexActions', () => {
let client: jest.Mocked<any>;
let callCluster: jest.Mock<CallCluster>;
let actions: ReindexActions;
const unimplemented = (name: string) => () =>
Promise.reject(`Mock function ${name} was not implemented!`);
beforeEach(() => {
client = {
errors: null,
create: jest.fn(unimplemented('create')),
bulkCreate: jest.fn(unimplemented('bulkCreate')),
delete: jest.fn(unimplemented('delete')),
find: jest.fn(unimplemented('find')),
bulkGet: jest.fn(unimplemented('bulkGet')),
get: jest.fn(unimplemented('get')),
// Fake update implementation that simply resolves to whatever the update says.
update: jest.fn((type: string, id: string, attributes: object) =>
Promise.resolve({ id, attributes } as ReindexSavedObject)
) as any,
};
callCluster = jest.fn();
actions = reindexActionsFactory(client, callCluster);
});
describe('createReindexOp', () => {
beforeEach(() => client.create.mockResolvedValue());
it(`appends -reindexed-v${CURRENT_MAJOR_VERSION} to new name`, async () => {
await actions.createReindexOp('myIndex');
expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, {
indexName: 'myIndex',
newIndexName: `myIndex-reindexed-v${CURRENT_MAJOR_VERSION}`,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,
reindexTaskId: null,
reindexTaskPercComplete: null,
errorMessage: null,
mlReindexCount: null,
});
});
it(`replaces -reindexed-v${PREV_MAJOR_VERSION} with -reindexed-v${CURRENT_MAJOR_VERSION}`, async () => {
await actions.createReindexOp(`myIndex-reindexed-v${PREV_MAJOR_VERSION}`);
expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, {
indexName: `myIndex-reindexed-v${PREV_MAJOR_VERSION}`,
newIndexName: `myIndex-reindexed-v${CURRENT_MAJOR_VERSION}`,
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,
reindexTaskId: null,
reindexTaskPercComplete: null,
errorMessage: null,
mlReindexCount: null,
});
});
});
describe('updateReindexOp', () => {
it('calls update with the combined attributes', async () => {
await actions.updateReindexOp(
{
type: REINDEX_OP_TYPE,
id: '9',
attributes: { indexName: 'hi', locked: moment().format() },
version: 1,
} as ReindexSavedObject,
{ newIndexName: 'test' }
);
expect(client.update).toHaveBeenCalled();
const args = client.update.mock.calls[0];
expect(args[0]).toEqual(REINDEX_OP_TYPE);
expect(args[1]).toEqual('9');
expect(args[2].indexName).toEqual('hi');
expect(args[2].newIndexName).toEqual('test');
expect(args[3]).toEqual({ version: 1 });
});
it('throws if the reindexOp is not locked', async () => {
await expect(
actions.updateReindexOp(
{
type: REINDEX_OP_TYPE,
id: '10',
attributes: { indexName: 'hi', locked: null },
version: 1,
} as ReindexSavedObject,
{ newIndexName: 'test' }
)
).rejects.toThrow();
expect(client.update).not.toHaveBeenCalled();
});
});
describe('runWhileLocked', () => {
it('locks and unlocks if object is unlocked', async () => {
const reindexOp = { id: '1', attributes: { locked: null } } as ReindexSavedObject;
await actions.runWhileLocked(reindexOp, op => Promise.resolve(op));
expect(client.update).toHaveBeenCalledTimes(2);
// Locking update call
const id1 = client.update.mock.calls[0][1];
const attr1 = client.update.mock.calls[0][2];
expect(id1).toEqual('1');
expect(attr1.locked).not.toBeNull();
// Unlocking update call
const id2 = client.update.mock.calls[1][1];
const attr2 = client.update.mock.calls[1][2];
expect(id2).toEqual('1');
expect(attr2.locked).toBeNull();
});
it("locks and unlocks if object's lock is expired", async () => {
const reindexOp = {
id: '1',
attributes: {
// Set locked timestamp to timeout + 10 seconds ago
locked: moment()
.subtract(LOCK_WINDOW)
.subtract(moment.duration(10, 'seconds'))
.format(),
},
} as ReindexSavedObject;
await actions.runWhileLocked(reindexOp, op => Promise.resolve(op));
expect(client.update).toHaveBeenCalledTimes(2);
// Locking update call
const id1 = client.update.mock.calls[0][1];
const attr1 = client.update.mock.calls[0][2];
expect(id1).toEqual('1');
expect(attr1.locked).not.toBeNull();
// Unlocking update call
const id2 = client.update.mock.calls[1][1];
const attr2 = client.update.mock.calls[1][2];
expect(id2).toEqual('1');
expect(attr2.locked).toBeNull();
});
it('still locks and unlocks if func throws', async () => {
const reindexOp = { id: '1', attributes: { locked: null } } as ReindexSavedObject;
await expect(
actions.runWhileLocked(reindexOp, op => Promise.reject(new Error('IT FAILED!')))
).rejects.toThrow('IT FAILED!');
expect(client.update).toHaveBeenCalledTimes(2);
// Locking update call
const id1 = client.update.mock.calls[0][1];
const attr1 = client.update.mock.calls[0][2];
expect(id1).toEqual('1');
expect(attr1.locked).not.toBeNull();
// Unlocking update call
const id2 = client.update.mock.calls[1][1];
const attr2 = client.update.mock.calls[1][2];
expect(id2).toEqual('1');
expect(attr2.locked).toBeNull();
});
it('throws if lock is not exprired', async () => {
const reindexOp = {
id: '1',
attributes: { locked: moment().format() },
} as ReindexSavedObject;
await expect(actions.runWhileLocked(reindexOp, op => Promise.resolve(op))).rejects.toThrow();
});
});
describe('findAllByStatus', () => {
it('returns saved_objects', async () => {
client.find.mockResolvedValue({ saved_objects: ['results!'] });
await expect(actions.findAllByStatus(ReindexStatus.inProgress)).resolves.toEqual([
'results!',
]);
expect(client.find).toHaveBeenCalledWith({
type: REINDEX_OP_TYPE,
search: '0',
searchFields: ['status'],
});
});
it('handles paging', async () => {
client.find
.mockResolvedValueOnce({
total: 20,
page: 0,
per_page: 10,
saved_objects: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
})
.mockResolvedValueOnce({
total: 20,
page: 1,
per_page: 10,
saved_objects: [11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
});
// Really prettier??
await expect(actions.findAllByStatus(ReindexStatus.completed)).resolves.toEqual([
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
]);
});
});
describe('getBooleanFieldPaths', () => {
it('returns array of array of boolean path strings', async () => {
callCluster.mockResolvedValueOnce({
myIndex: {
mappings: {
_doc: {
properties: {
field1: { type: 'boolean' },
nested: { properties: { field2: { type: 'boolean' } } },
},
},
},
},
});
await expect(actions.getBooleanFieldPaths('myIndex')).resolves.toEqual([
['field1'],
['nested', 'field2'],
]);
});
it('returns [] if there are no mapping types', async () => {
callCluster.mockResolvedValueOnce({ myIndex: { mappings: {} } });
await expect(actions.getBooleanFieldPaths('myIndex')).resolves.toEqual([]);
});
it('throws if there are multiple mapping types', async () => {
callCluster.mockResolvedValueOnce({ myIndex: { mappings: { type1: {}, type2: {} } } });
await expect(actions.getBooleanFieldPaths('myIndex')).rejects.toThrow();
});
});
describe('getFlatSettings', () => {
it('returns flat settings', async () => {
callCluster.mockResolvedValueOnce({
myIndex: {
settings: { 'index.mySetting': '1' },
mappings: {},
},
});
await expect(actions.getFlatSettings('myIndex')).resolves.toEqual({
settings: { 'index.mySetting': '1' },
mappings: {},
});
});
it('returns null if index does not exist', async () => {
callCluster.mockResolvedValueOnce({});
await expect(actions.getFlatSettings('myIndex')).resolves.toBeNull();
});
});
describe('runWhileMlLocked', () => {
it('creates the ML doc if it does not exist and executes callback', async () => {
expect.assertions(3);
client.get.mockRejectedValueOnce(Boom.notFound()); // mock no ML doc exists yet
client.create.mockImplementationOnce((type: any, attributes: any, { id }: any) =>
Promise.resolve({
type,
id,
attributes,
})
);
let flip = false;
await actions.runWhileMlLocked(async mlDoc => {
expect(mlDoc.id).toEqual(ML_LOCK_DOC_ID);
expect(mlDoc.attributes.mlReindexCount).toEqual(0);
flip = true;
return mlDoc;
});
expect(flip).toEqual(true);
});
it('fails after 10 attempts to lock', async () => {
jest.setTimeout(20000); // increase the timeout
client.get.mockResolvedValue({
type: REINDEX_OP_TYPE,
id: ML_LOCK_DOC_ID,
attributes: { mlReindexCount: 0 },
});
client.update.mockRejectedValue(new Error('NO LOCKING!'));
await expect(actions.runWhileMlLocked(async m => m)).rejects.toThrow(
'Could not acquire lock for ML jobs'
);
expect(client.update).toHaveBeenCalledTimes(10);
// Restore default timeout.
jest.setTimeout(5000);
});
});
});

352
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts Normal file
View file

@ -0,0 +1,352 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import moment from 'moment';
import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
import {
FindResponse,
SavedObjectsClient,
} from 'src/server/saved_objects/service/saved_objects_client';
import {
CURRENT_MAJOR_VERSION,
PREV_MAJOR_VERSION,
} from 'x-pack/plugins/upgrade_assistant/common/version';
import {
REINDEX_OP_TYPE,
ReindexOperation,
ReindexSavedObject,
ReindexStatus,
ReindexStep,
} from '../../../common/types';
import { findBooleanFields, getSingleMappingType } from './index_settings';
import { FlatSettings } from './types';
// TODO: base on elasticsearch.requestTimeout?
export const LOCK_WINDOW = moment.duration(90, 'seconds');
export const ML_LOCK_DOC_ID = '___ML_REINDEX_LOCK___';
/**
* A collection of utility functions pulled out out of the ReindexService to make testing simpler.
* This is NOT intended to be used by any other code.
*/
export interface ReindexActions {
/**
* Namespace for ML-specific actions.
*/
// ml: MlActions;
/**
* Creates a new reindexOp, does not perform any pre-flight checks.
* @param indexName
*/
createReindexOp(indexName: string): Promise<ReindexSavedObject>;
/**
* Deletes a reindexOp.
* @param reindexOp
*/
deleteReindexOp(reindexOp: ReindexSavedObject): void;
/**
* Updates a ReindexSavedObject.
* @param reindexOp
* @param attrs
*/
updateReindexOp(
reindexOp: ReindexSavedObject,
attrs?: Partial<ReindexOperation>
): Promise<ReindexSavedObject>;
/**
* Runs a callback function while locking the reindex operation. Guaranteed to unlock the reindex operation when complete.
* @param func A function to run with the locked ML lock document. Must return a promise that resolves
* to the updated ReindexSavedObject.
*/
runWhileLocked(
reindexOp: ReindexSavedObject,
func: (reindexOp: ReindexSavedObject) => Promise<ReindexSavedObject>
): Promise<ReindexSavedObject>;
/**
* Finds the reindex operation saved object for the given index.
* @param indexName
*/
findReindexOperations(indexName: string): Promise<FindResponse<ReindexOperation>>;
/**
* Returns an array of all reindex operations that have a status.
*/
findAllByStatus(status: ReindexStatus): Promise<ReindexSavedObject[]>;
/**
* Returns array of field paths to boolean fields in the index's mapping.
* @param indexName
*/
getBooleanFieldPaths(indexName: string): Promise<string[][]>;
/**
* Retrieve index settings (in flat, dot-notation style) and mappings.
* @param indexName
*/
getFlatSettings(indexName: string): Promise<FlatSettings | null>;
// ----- Below are only for ML indices
/**
* Atomically increments the number of reindex operations running for ML jobs.
*/
incrementMlReindexes(): Promise<void>;
/**
* Atomically decrements the number of reindex operations running for ML jobs.
*/
decrementMlReindexes(): Promise<void>;
/**
* Runs a callback function while locking the ML count.
* @param func A function to run with the locked ML lock document. Must return a promise that resolves
* to the updated ReindexSavedObject.
*/
runWhileMlLocked(
func: (mlLockDoc: ReindexSavedObject) => Promise<ReindexSavedObject>
): Promise<void>;
/**
* Exposed only for testing, DO NOT USE.
*/
_fetchAndLockMlDoc(): Promise<ReindexSavedObject>;
}
export const reindexActionsFactory = (
client: SavedObjectsClient,
callCluster: CallCluster
): ReindexActions => {
// ----- Internal functions
/**
* Generates a new index name for the new index. Iterates until it finds an index
* that doesn't already exist.
* @param indexName
*/
const getNewIndexName = (indexName: string) => {
const prevVersionSuffix = `-reindexed-v${PREV_MAJOR_VERSION}`;
const currentVersionSuffix = `-reindexed-v${CURRENT_MAJOR_VERSION}`;
if (indexName.endsWith(prevVersionSuffix)) {
return indexName.replace(new RegExp(`${prevVersionSuffix}$`), currentVersionSuffix);
} else {
return `${indexName}${currentVersionSuffix}`;
}
};
const isLocked = (reindexOp: ReindexSavedObject) => {
if (reindexOp.attributes.locked) {
const now = moment();
const lockedTime = moment(reindexOp.attributes.locked);
// If the object has been locked for more than the LOCK_WINDOW, assume the process that locked it died.
if (now.subtract(LOCK_WINDOW) < lockedTime) {
return true;
}
}
return false;
};
const acquireLock = async (reindexOp: ReindexSavedObject) => {
if (isLocked(reindexOp)) {
throw new Error(`Another Kibana process is currently modifying this reindex operation.`);
}
return client.update<ReindexOperation>(
REINDEX_OP_TYPE,
reindexOp.id,
{ ...reindexOp.attributes, locked: moment().format() },
{ version: reindexOp.version }
);
};
const releaseLock = (reindexOp: ReindexSavedObject) => {
return client.update<ReindexOperation>(
REINDEX_OP_TYPE,
reindexOp.id,
{ ...reindexOp.attributes, locked: null },
{ version: reindexOp.version }
);
};
// ----- Public interface
return {
async createReindexOp(indexName: string) {
return client.create<ReindexOperation>(REINDEX_OP_TYPE, {
indexName,
newIndexName: getNewIndexName(indexName),
status: ReindexStatus.inProgress,
lastCompletedStep: ReindexStep.created,
locked: null,
reindexTaskId: null,
reindexTaskPercComplete: null,
errorMessage: null,
mlReindexCount: null,
});
},
deleteReindexOp(reindexOp: ReindexSavedObject) {
return client.delete(REINDEX_OP_TYPE, reindexOp.id);
},
async updateReindexOp(reindexOp: ReindexSavedObject, attrs: Partial<ReindexOperation> = {}) {
if (!isLocked(reindexOp)) {
throw new Error(`ReindexOperation must be locked before updating.`);
}
const newAttrs = { ...reindexOp.attributes, locked: moment().format(), ...attrs };
return client.update<ReindexOperation>(REINDEX_OP_TYPE, reindexOp.id, newAttrs, {
version: reindexOp.version,
});
},
async runWhileLocked(reindexOp, func) {
reindexOp = await acquireLock(reindexOp);
try {
reindexOp = await func(reindexOp);
} finally {
reindexOp = await releaseLock(reindexOp);
}
return reindexOp;
},
findReindexOperations(indexName: string) {
return client.find<ReindexOperation>({
type: REINDEX_OP_TYPE,
search: `"${indexName}"`,
searchFields: ['indexName'],
});
},
async findAllByStatus(status: ReindexStatus) {
const firstPage = await client.find<ReindexOperation>({
type: REINDEX_OP_TYPE,
search: status.toString(),
searchFields: ['status'],
});
if (firstPage.total === firstPage.saved_objects.length) {
return firstPage.saved_objects;
}
let allOps = firstPage.saved_objects;
let page = firstPage.page + 1;
while (allOps.length < firstPage.total) {
const nextPage = await client.find<ReindexOperation>({
type: REINDEX_OP_TYPE,
search: status.toString(),
searchFields: ['status'],
page,
});
allOps = [...allOps, ...nextPage.saved_objects];
page++;
}
return allOps;
},
async getBooleanFieldPaths(indexName: string) {
const results = await callCluster('indices.getMapping', { index: indexName });
const mapping = getSingleMappingType(results[indexName].mappings);
// It's possible an index doesn't have a mapping.
return mapping && mapping.properties ? findBooleanFields(mapping.properties) : [];
},
async getFlatSettings(indexName: string) {
const flatSettings = (await callCluster('transport.request', {
// TODO: set `&include_type_name=true` to false in 7.0
path: `/${encodeURIComponent(indexName)}?flat_settings=true`,
})) as { [indexName: string]: FlatSettings };
if (!flatSettings[indexName]) {
return null;
}
return flatSettings[indexName];
},
async _fetchAndLockMlDoc() {
const fetchDoc = async () => {
try {
return await client.get<ReindexOperation>(REINDEX_OP_TYPE, ML_LOCK_DOC_ID);
} catch (e) {
if (e.isBoom && e.output.statusCode === 404) {
return await client.create<ReindexOperation>(
REINDEX_OP_TYPE,
{
indexName: null,
newIndexName: null,
locked: null,
status: null,
lastCompletedStep: null,
reindexTaskId: null,
reindexTaskPercComplete: null,
errorMessage: null,
mlReindexCount: 0,
} as any,
{ id: ML_LOCK_DOC_ID }
);
} else {
throw e;
}
}
};
const lockDoc = async (attempt = 1): Promise<ReindexSavedObject> => {
try {
// Refetch the document each time to avoid version conflicts.
return await acquireLock(await fetchDoc());
} catch (e) {
if (attempt >= 10) {
throw new Error(`Could not acquire lock for ML jobs`);
}
await new Promise(resolve => setTimeout(resolve, 1000));
return lockDoc(attempt + 1);
}
};
return lockDoc();
},
async incrementMlReindexes() {
this.runWhileMlLocked(mlDoc =>
this.updateReindexOp(mlDoc, {
mlReindexCount: mlDoc.attributes.mlReindexCount! + 1,
})
);
},
async decrementMlReindexes() {
this.runWhileMlLocked(mlDoc =>
this.updateReindexOp(mlDoc, {
mlReindexCount: mlDoc.attributes.mlReindexCount! - 1,
})
);
},
async runWhileMlLocked(func) {
let mlDoc = await this._fetchAndLockMlDoc();
try {
mlDoc = await func(mlDoc);
} finally {
await releaseLock(mlDoc);
}
},
};
};

818
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts Normal file
View file

@ -0,0 +1,818 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
import {
ReindexOperation,
ReindexSavedObject,
ReindexStatus,
ReindexStep,
ReindexWarning,
} from '../../../common/types';
import { ReindexService, reindexServiceFactory } from './reindex_service';
describe('reindexService', () => {
let actions: jest.Mocked<any>;
let callCluster: jest.Mock<CallCluster>;
let service: ReindexService;
const updateMockImpl = (reindexOp: ReindexSavedObject, attrs: Partial<ReindexOperation> = {}) =>
Promise.resolve({
...reindexOp,
attributes: { ...reindexOp.attributes, ...attrs },
} as ReindexSavedObject);
const unimplemented = (name: string) => () =>
Promise.reject(`Mock function ${name} was not implemented!`);
beforeEach(() => {
actions = {
createReindexOp: jest.fn(unimplemented('createReindexOp')),
deleteReindexOp: jest.fn(unimplemented('deleteReindexOp')),
updateReindexOp: jest.fn(updateMockImpl),
runWhileLocked: jest.fn((reindexOp: any, func: any) => func(reindexOp)),
findReindexOperations: jest.fn(unimplemented('findReindexOperations')),
findAllByStatus: jest.fn(unimplemented('findAllInProgressOperations')),
getBooleanFieldPaths: jest.fn(unimplemented('getBooleanFieldPaths')),
getFlatSettings: jest.fn(unimplemented('getFlatSettings')),
cleanupChanges: jest.fn(),
incrementMlReindexes: jest.fn(unimplemented('incrementMlReindexes')),
decrementMlReindexes: jest.fn(unimplemented('decrementMlReindexes')),
runWhileMlLocked: jest.fn(async (f: any) => f({ attributes: {} })),
};
callCluster = jest.fn();
service = reindexServiceFactory(callCluster, actions);
});
describe('detectReindexWarnings', () => {
it('fetches reindex warnings from flat settings', async () => {
actions.getFlatSettings.mockResolvedValueOnce({
settings: {},
mappings: {
_doc: {
properties: { https: { type: 'boolean' } },
_all: { enabled: true },
},
},
});
const reindexWarnings = await service.detectReindexWarnings('myIndex');
expect(reindexWarnings).toEqual([ReindexWarning.allField, ReindexWarning.booleanFields]);
});
it('returns null if index does not exist', async () => {
actions.getFlatSettings.mockResolvedValueOnce(null);
const reindexWarnings = await service.detectReindexWarnings('myIndex');
expect(reindexWarnings).toBeNull();
});
});
describe('createReindexOperation', () => {
it('creates new reindex operation', async () => {
callCluster.mockResolvedValueOnce(true); // indices.exist
actions.findReindexOperations.mockResolvedValueOnce({ total: 0 });
actions.createReindexOp.mockResolvedValueOnce();
await service.createReindexOperation('myIndex');
expect(actions.createReindexOp).toHaveBeenCalledWith('myIndex');
});
it('fails if index does not exist', async () => {
callCluster.mockResolvedValueOnce(false); // indices.exist
await expect(service.createReindexOperation('myIndex')).rejects.toThrow();
expect(actions.createReindexOp).not.toHaveBeenCalled();
});
it('deletes existing operation if it failed', async () => {
callCluster.mockResolvedValueOnce(true); // indices.exist
actions.findReindexOperations.mockResolvedValueOnce({
saved_objects: [{ id: 1, attributes: { status: ReindexStatus.failed } }],
total: 1,
});
actions.deleteReindexOp.mockResolvedValueOnce();
actions.createReindexOp.mockResolvedValueOnce();
await service.createReindexOperation('myIndex');
expect(actions.deleteReindexOp).toHaveBeenCalledWith({
id: 1,
attributes: { status: ReindexStatus.failed },
});
});
it('fails if existing operation did not fail', async () => {
callCluster.mockResolvedValueOnce(true); // indices.exist
actions.findReindexOperations.mockResolvedValueOnce({
saved_objects: [{ id: 1, attributes: { status: ReindexStatus.inProgress } }],
total: 1,
});
await expect(service.createReindexOperation('myIndex')).rejects.toThrow();
expect(actions.deleteReindexOp).not.toHaveBeenCalled();
expect(actions.createReindexOp).not.toHaveBeenCalled();
});
});
describe('findReindexOperation', () => {
it('returns the only result', async () => {
actions.findReindexOperations.mockResolvedValue({ total: 1, saved_objects: ['fake object'] });
await expect(service.findReindexOperation('myIndex')).resolves.toEqual('fake object');
});
it('returns null if there are no results', async () => {
actions.findReindexOperations.mockResolvedValue({ total: 0 });
await expect(service.findReindexOperation('myIndex')).resolves.toBeNull();
});
it('fails if there is more than 1 result', async () => {
actions.findReindexOperations.mockResolvedValue({ total: 2 });
await expect(service.findReindexOperation('myIndex')).rejects.toThrow();
});
});
describe('processNextStep', () => {
describe('locking', () => {
// These tests depend on an implementation detail that if no status is set, the state machine
// is not activated, just the locking mechanism.
it('runs with runWhileLocked', async () => {
const reindexOp = { id: '1', attributes: { locked: null } } as ReindexSavedObject;
await service.processNextStep(reindexOp);
expect(actions.runWhileLocked).toHaveBeenCalled();
});
});
});
describe('pauseReindexOperation', () => {
it('runs with runWhileLocked', async () => {
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce({
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.inProgress },
});
await service.pauseReindexOperation('myIndex');
expect(actions.runWhileLocked).toHaveBeenCalled();
findSpy.mockRestore();
});
it('sets the status to paused', async () => {
const reindexOp = {
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.inProgress },
} as ReindexSavedObject;
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce(reindexOp);
await expect(service.pauseReindexOperation('myIndex')).resolves.toEqual({
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.paused },
});
expect(findSpy).toHaveBeenCalledWith('myIndex');
expect(actions.updateReindexOp).toHaveBeenCalledWith(reindexOp, {
status: ReindexStatus.paused,
});
findSpy.mockRestore();
});
it('throws if reindexOp is not inProgress', async () => {
const reindexOp = {
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.failed },
} as ReindexSavedObject;
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce(reindexOp);
await expect(service.pauseReindexOperation('myIndex')).rejects.toThrow();
expect(actions.updateReindexOp).not.toHaveBeenCalled();
findSpy.mockRestore();
});
it('throws in reindex operation does not exist', async () => {
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce(null);
await expect(service.pauseReindexOperation('myIndex')).rejects.toThrow();
expect(actions.updateReindexOp).not.toHaveBeenCalled();
findSpy.mockRestore();
});
});
describe('resumeReindexOperation', () => {
it('runs with runWhileLocked', async () => {
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce({
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.paused },
});
await service.resumeReindexOperation('myIndex');
expect(actions.runWhileLocked).toHaveBeenCalled();
findSpy.mockRestore();
});
it('sets the status to inProgress', async () => {
const reindexOp = {
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.paused },
} as ReindexSavedObject;
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce(reindexOp);
await expect(service.resumeReindexOperation('myIndex')).resolves.toEqual({
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.inProgress },
});
expect(findSpy).toHaveBeenCalledWith('myIndex');
expect(actions.updateReindexOp).toHaveBeenCalledWith(reindexOp, {
status: ReindexStatus.inProgress,
});
findSpy.mockRestore();
});
it('throws if reindexOp is not inProgress', async () => {
const reindexOp = {
id: '2',
attributes: { indexName: 'myIndex', status: ReindexStatus.failed },
} as ReindexSavedObject;
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce(reindexOp);
await expect(service.resumeReindexOperation('myIndex')).rejects.toThrow();
expect(actions.updateReindexOp).not.toHaveBeenCalled();
findSpy.mockRestore();
});
it('throws in reindex operation does not exist', async () => {
const findSpy = jest.spyOn(service, 'findReindexOperation').mockResolvedValueOnce(null);
await expect(service.resumeReindexOperation('myIndex')).rejects.toThrow();
expect(actions.updateReindexOp).not.toHaveBeenCalled();
findSpy.mockRestore();
});
});
describe('state machine, lastCompletedStep ===', () => {
const defaultAttributes = {
indexName: 'myIndex',
newIndexName: 'myIndex-reindex-0',
status: ReindexStatus.inProgress,
};
const settingsMappings = {
settings: { 'index.number_of_replicas': 7, 'index.blocks.write': true },
mappings: { _doc: { properties: { timestampl: { type: 'date' } } } },
};
describe('created', () => {
const reindexOp = {
id: '1',
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.created },
} as ReindexSavedObject;
// ML
const mlReindexOp = {
id: '2',
attributes: { ...reindexOp.attributes, indexName: '.ml-anomalies' },
} as ReindexSavedObject;
it('does nothing if index is not an ML index', async () => {
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStopped);
expect(actions.incrementMlReindexes).not.toHaveBeenCalled();
expect(actions.runWhileMlLocked).not.toHaveBeenCalled();
expect(callCluster).not.toHaveBeenCalled();
});
it('increments ML reindexes and calls ML stop endpoint', async () => {
actions.incrementMlReindexes.mockResolvedValueOnce();
actions.runWhileMlLocked.mockImplementationOnce(async (f: any) => f());
callCluster
// Mock call to /_nodes for version check
.mockResolvedValueOnce({ nodes: { nodeX: { version: '6.7.0-alpha' } } })
// Mock call to /_ml/set_upgrade_mode?enabled=true
.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStopped);
expect(actions.incrementMlReindexes).toHaveBeenCalled();
expect(actions.runWhileMlLocked).toHaveBeenCalled();
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=true',
method: 'POST',
});
});
it('fails if ML reindexes cannot be incremented', async () => {
actions.incrementMlReindexes.mockRejectedValueOnce(new Error(`Can't lock!`));
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy();
expect(callCluster).not.toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=true',
method: 'POST',
});
});
it('fails if ML doc cannot be locked', async () => {
actions.incrementMlReindexes.mockResolvedValueOnce();
actions.runWhileMlLocked.mockRejectedValueOnce(new Error(`Can't lock!`));
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy();
expect(callCluster).not.toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=true',
method: 'POST',
});
});
it('fails if ML endpoint fails', async () => {
actions.incrementMlReindexes.mockResolvedValueOnce();
callCluster
// Mock call to /_nodes for version check
.mockResolvedValueOnce({ nodes: { nodeX: { version: '6.7.0' } } })
// Mock call to /_ml/set_upgrade_mode?enabled=true
.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage!.includes('Could not stop ML jobs')).toBeTruthy();
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=true',
method: 'POST',
});
});
it('fails if not all nodes have been upgraded to 6.7.0', async () => {
actions.incrementMlReindexes.mockResolvedValueOnce();
callCluster
// Mock call to /_nodes for version check
.mockResolvedValueOnce({ nodes: { nodeX: { version: '6.6.0' } } });
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(
updatedOp.attributes.errorMessage!.includes('Some nodes are not on minimum version')
).toBeTruthy();
// Should not have called ML endpoint at all
expect(callCluster).not.toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=true',
method: 'POST',
});
});
// Watcher
const watcherReindexOp = {
id: '2',
attributes: { ...reindexOp.attributes, indexName: '.watches' },
} as ReindexSavedObject;
it('does nothing if index is not a watcher index', async () => {
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStopped);
expect(callCluster).not.toHaveBeenCalled();
});
it('calls watcher start endpoint', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(watcherReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStopped);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_xpack/watcher/_stop',
method: 'POST',
});
});
it('fails if watcher start endpoint fails', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(watcherReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_xpack/watcher/_stop',
method: 'POST',
});
});
it('fails if watcher start endpoint throws', async () => {
callCluster.mockRejectedValueOnce(new Error('Whoops!'));
const updatedOp = await service.processNextStep(watcherReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.created);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_xpack/watcher/_stop',
method: 'POST',
});
});
});
describe('indexConsumersStopped', () => {
const reindexOp = {
id: '1',
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.indexConsumersStopped },
} as ReindexSavedObject;
it('blocks writes and updates lastCompletedStep', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.readonly);
expect(callCluster).toHaveBeenCalledWith('indices.putSettings', {
index: 'myIndex',
body: { 'index.blocks.write': true },
});
});
it('fails if setting updates are not acknowledged', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStopped);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
it('fails if setting updates fail', async () => {
callCluster.mockRejectedValueOnce(new Error('blah!'));
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStopped);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
});
describe('readonly', () => {
const reindexOp = {
id: '1',
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.readonly },
} as ReindexSavedObject;
// The more intricate details of how the settings are chosen are test separately.
it('creates new index with settings and mappings and updates lastCompletedStep', async () => {
actions.getFlatSettings.mockResolvedValueOnce(settingsMappings);
callCluster.mockResolvedValueOnce({ acknowledged: true }); // indices.create
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.newIndexCreated);
expect(callCluster).toHaveBeenCalledWith('indices.create', {
index: 'myIndex-reindex-0',
body: {
// index.blocks.write should be removed from the settings for the new index.
settings: { 'index.number_of_replicas': 7 },
mappings: settingsMappings.mappings,
},
});
});
it('fails if create index is not acknowledged', async () => {
callCluster
.mockResolvedValueOnce({ myIndex: settingsMappings })
.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.readonly);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
it('fails if create index fails', async () => {
callCluster
.mockResolvedValueOnce({ myIndex: settingsMappings })
.mockRejectedValueOnce(new Error(`blah!`))
.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.readonly);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
// Original index should have been set back to allow reads.
expect(callCluster).toHaveBeenCalledWith('indices.putSettings', {
index: 'myIndex',
body: { 'index.blocks.write': false },
});
});
});
describe('newIndexCreated', () => {
const reindexOp = {
id: '1',
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.newIndexCreated },
} as ReindexSavedObject;
it('starts reindex, saves taskId, and updates lastCompletedStep', async () => {
actions.getBooleanFieldPaths.mockResolvedValue([]);
callCluster.mockResolvedValueOnce({ task: 'xyz' }); // reindex
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.reindexStarted);
expect(updatedOp.attributes.reindexTaskId).toEqual('xyz');
expect(updatedOp.attributes.reindexTaskPercComplete).toEqual(0);
expect(callCluster).toHaveBeenLastCalledWith('reindex', {
refresh: true,
waitForCompletion: false,
body: {
source: { index: 'myIndex' },
dest: { index: 'myIndex-reindex-0' },
},
});
});
it('adds painless script if there are boolean fields', async () => {
actions.getBooleanFieldPaths.mockResolvedValue([['field1'], ['nested', 'field2']]);
callCluster.mockResolvedValueOnce({ task: 'xyz' }); // reindex
await service.processNextStep(reindexOp);
const reindexBody = callCluster.mock.calls[0][1].body;
expect(reindexBody.script.lang).toEqual('painless');
expect(typeof reindexBody.script.source).toEqual('string');
expect(reindexBody.script.params.booleanFieldPaths).toEqual([
['field1'],
['nested', 'field2'],
]);
});
it('fails if starting reindex fails', async () => {
actions.getBooleanFieldPaths.mockResolvedValue([['field1'], ['nested', 'field2']]);
callCluster.mockRejectedValueOnce(new Error('blah!')).mockResolvedValueOnce({});
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.newIndexCreated);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
});
describe('reindexStarted', () => {
const reindexOp = {
id: '1',
attributes: {
...defaultAttributes,
lastCompletedStep: ReindexStep.reindexStarted,
reindexTaskId: 'xyz',
},
} as ReindexSavedObject;
describe('reindex task is not complete', () => {
it('updates reindexTaskPercComplete', async () => {
callCluster.mockResolvedValueOnce({
completed: false,
task: { status: { created: 10, total: 100 } },
});
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.reindexStarted);
expect(updatedOp.attributes.reindexTaskPercComplete).toEqual(0.1); // 10 / 100 = 0.1
});
});
describe('reindex task is complete', () => {
it('deletes task, updates reindexTaskPercComplete, updates lastCompletedStep', async () => {
callCluster
.mockResolvedValueOnce({
completed: true,
task: { status: { created: 100, total: 100 } },
})
.mockResolvedValueOnce({ count: 100 })
.mockResolvedValueOnce({ result: 'deleted' });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.reindexCompleted);
expect(updatedOp.attributes.reindexTaskPercComplete).toEqual(1);
expect(callCluster).toHaveBeenCalledWith('delete', {
index: '.tasks',
type: 'task',
id: 'xyz',
});
});
it('fails if docs created is less than count in source index', async () => {
callCluster
.mockResolvedValueOnce({
completed: true,
task: { status: { created: 95, total: 95 } },
})
.mockReturnValueOnce({ count: 100 });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.reindexStarted);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
});
});
describe('reindexCompleted', () => {
const reindexOp = {
id: '1',
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.reindexCompleted },
} as ReindexSavedObject;
it('switches aliases, sets as complete, and updates lastCompletedStep', async () => {
callCluster
.mockResolvedValueOnce({ myIndex: { aliases: {} } })
.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(callCluster).toHaveBeenCalledWith('indices.updateAliases', {
body: {
actions: [
{ add: { index: 'myIndex-reindex-0', alias: 'myIndex' } },
{ remove_index: { index: 'myIndex' } },
],
},
});
});
it('moves existing aliases over to new index', async () => {
callCluster
.mockResolvedValueOnce({
myIndex: {
aliases: {
myAlias: {},
myFilteredAlias: { filter: { term: { https: true } } },
},
},
})
.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(callCluster).toHaveBeenCalledWith('indices.updateAliases', {
body: {
actions: [
{ add: { index: 'myIndex-reindex-0', alias: 'myIndex' } },
{ remove_index: { index: 'myIndex' } },
{ add: { index: 'myIndex-reindex-0', alias: 'myAlias' } },
{
add: {
index: 'myIndex-reindex-0',
alias: 'myFilteredAlias',
filter: { term: { https: true } },
},
},
],
},
});
});
it('fails if switching aliases is not acknowledged', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.reindexCompleted);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
it('fails if switching aliases fails', async () => {
callCluster.mockRejectedValueOnce(new Error('blah!'));
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.reindexCompleted);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage).not.toBeNull();
});
});
describe('aliasCreated', () => {
const reindexOp = {
id: '1',
attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.aliasCreated },
} as ReindexSavedObject;
// ML
const mlReindexOp = {
id: '2',
attributes: { ...reindexOp.attributes, indexName: '.ml-anomalies' },
} as ReindexSavedObject;
it('does nothing if index is not an ML index', async () => {
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStarted);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.completed);
expect(callCluster).not.toHaveBeenCalled();
});
it('decrements ML reindexes and calls ML start endpoint if no remaining ML jobs', async () => {
actions.decrementMlReindexes.mockResolvedValue();
actions.runWhileMlLocked.mockImplementationOnce(async (f: any) =>
f({ attributes: { mlReindexCount: 0 } })
);
// Mock call to /_ml/set_upgrade_mode?enabled=false
callCluster.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStarted);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=false',
method: 'POST',
});
});
it('does not call ML start endpoint if there are remaining ML jobs', async () => {
actions.decrementMlReindexes.mockResolvedValue();
actions.runWhileMlLocked.mockImplementationOnce(async (f: any) =>
f({ attributes: { mlReindexCount: 2 } })
);
// Mock call to /_ml/set_upgrade_mode?enabled=false
callCluster.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStarted);
expect(callCluster).not.toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=false',
method: 'POST',
});
});
it('fails if ML reindexes cannot be decremented', async () => {
// Mock unable to lock ml doc
actions.decrementMlReindexes.mockRejectedValue(new Error(`Can't lock!`));
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy();
expect(callCluster).not.toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=false',
method: 'POST',
});
});
it('fails if ML doc cannot be locked', async () => {
actions.decrementMlReindexes.mockResolvedValue();
// Mock unable to lock ml doc
actions.runWhileMlLocked.mockRejectedValueOnce(new Error(`Can't lock!`));
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(updatedOp.attributes.errorMessage!.includes(`Can't lock!`)).toBeTruthy();
expect(callCluster).not.toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=false',
method: 'POST',
});
});
it('fails if ML endpoint fails', async () => {
actions.decrementMlReindexes.mockResolvedValue();
actions.runWhileMlLocked.mockImplementationOnce(async (f: any) =>
f({ attributes: { mlReindexCount: 0 } })
);
// Mock call to /_ml/set_upgrade_mode?enabled=true
callCluster.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(mlReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(
updatedOp.attributes.errorMessage!.includes('Could not resume ML jobs')
).toBeTruthy();
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=false',
method: 'POST',
});
});
// Watcher
const watcherReindexOp = {
id: '2',
attributes: { ...reindexOp.attributes, indexName: '.watches' },
} as ReindexSavedObject;
it('does nothing if index is not a watcher index', async () => {
const updatedOp = await service.processNextStep(reindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStarted);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.completed);
expect(callCluster).not.toHaveBeenCalled();
});
it('calls watcher start endpoint', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: true });
const updatedOp = await service.processNextStep(watcherReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.indexConsumersStarted);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.completed);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_xpack/watcher/_start',
method: 'POST',
});
});
it('fails if watcher start endpoint fails', async () => {
callCluster.mockResolvedValueOnce({ acknowledged: false });
const updatedOp = await service.processNextStep(watcherReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_xpack/watcher/_start',
method: 'POST',
});
});
it('fails if watcher start endpoint throws', async () => {
callCluster.mockRejectedValueOnce(new Error('Whoops!'));
const updatedOp = await service.processNextStep(watcherReindexOp);
expect(updatedOp.attributes.lastCompletedStep).toEqual(ReindexStep.aliasCreated);
expect(updatedOp.attributes.status).toEqual(ReindexStatus.failed);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_xpack/watcher/_start',
method: 'POST',
});
});
});
});
});

557
x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts Normal file
View file

@ -0,0 +1,557 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
import {
ReindexSavedObject,
ReindexStatus,
ReindexStep,
ReindexWarning,
} from '../../../common/types';
import { getReindexWarnings, transformFlatSettings } from './index_settings';
import { ReindexActions } from './reindex_actions';
const VERSION_REGEX = new RegExp(/^([1-9]+)\.([0-9]+)\.([0-9]+)/);
export interface ReindexService {
/**
* Checks an index's settings and mappings to flag potential issues during reindex.
* Resolves to null if index does not exist.
* @param indexName
*/
detectReindexWarnings(indexName: string): Promise<ReindexWarning[] | null>;
/**
* Creates a new reindex operation for a given index.
* @param indexName
*/
createReindexOperation(indexName: string): Promise<ReindexSavedObject>;
/**
* Retrieves all reindex operations that have the given status.
* @param status
*/
findAllByStatus(status: ReindexStatus): Promise<ReindexSavedObject[]>;
/**
* Finds the reindex operation for the given index.
* Resolves to null if there is no existing reindex operation for this index.
* @param indexName
*/
findReindexOperation(indexName: string): Promise<ReindexSavedObject | null>;
/**
* Process the reindex operation through one step of the state machine and resolves
* to the updated reindex operation.
* @param reindexOp
*/
processNextStep(reindexOp: ReindexSavedObject): Promise<ReindexSavedObject>;
/**
* Pauses the in-progress reindex operation for a given index.
* @param indexName
*/
pauseReindexOperation(indexName: string): Promise<ReindexSavedObject>;
/**
* Resumes the paused reindex operation for a given index.
* @param indexName
*/
resumeReindexOperation(indexName: string): Promise<ReindexSavedObject>;
}
export const reindexServiceFactory = (
callCluster: CallCluster,
actions: ReindexActions
): ReindexService => {
// ------ Utility functions
/**
* If the index is a ML index that will cause jobs to fail when set to readonly,
* turn on 'upgrade mode' to pause all ML jobs.
* @param reindexOp
*/
const stopMlJobs = async () => {
await actions.incrementMlReindexes();
await actions.runWhileMlLocked(async mlDoc => {
await validateNodesMinimumVersion(6, 7);
const res = await callCluster('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=true',
method: 'POST',
});
if (!res.acknowledged) {
throw new Error(`Could not stop ML jobs`);
}
return mlDoc;
});
};
/**
* Resumes ML jobs if there are no more remaining reindex operations.
*/
const resumeMlJobs = async () => {
await actions.decrementMlReindexes();
await actions.runWhileMlLocked(async mlDoc => {
if (mlDoc.attributes.mlReindexCount === 0) {
const res = await callCluster('transport.request', {
path: '/_ml/set_upgrade_mode?enabled=false',
method: 'POST',
});
if (!res.acknowledged) {
throw new Error(`Could not resume ML jobs`);
}
}
return mlDoc;
});
};
/**
* Stops Watcher in Elasticsearch.
*/
const stopWatcher = async () => {
const { acknowledged } = await callCluster('transport.request', {
path: '/_xpack/watcher/_stop',
method: 'POST',
});
if (!acknowledged) {
throw new Error('Could not stop Watcher');
}
};
/**
* Starts Watcher in Elasticsearch.
*/
const startWatcher = async () => {
const { acknowledged } = await callCluster('transport.request', {
path: '/_xpack/watcher/_start',
method: 'POST',
});
if (!acknowledged) {
throw new Error('Could not start Watcher');
}
};
const cleanupChanges = async (reindexOp: ReindexSavedObject) => {
// Set back to writable if we ever got past this point.
if (reindexOp.attributes.lastCompletedStep >= ReindexStep.readonly) {
await callCluster('indices.putSettings', {
index: reindexOp.attributes.indexName,
body: { 'index.blocks.write': false },
});
}
// Stop consumers if we ever got past this point.
if (reindexOp.attributes.lastCompletedStep >= ReindexStep.indexConsumersStopped) {
await resumeConsumers(reindexOp);
}
};
// ------ Functions used to process the state machine
const validateNodesMinimumVersion = async (minMajor: number, minMinor: number) => {
const nodesResponse = await callCluster('transport.request', {
path: '/_nodes',
method: 'GET',
});
const outDatedNodes = Object.values(nodesResponse.nodes).filter((node: any) => {
const matches = node.version.match(VERSION_REGEX);
const major = parseInt(matches[1], 10);
const minor = parseInt(matches[2], 10);
// All ES nodes must be >= 6.7.0 to pause ML jobs
return !(major > minMajor || (major === minMajor && minor >= minMinor));
});
if (outDatedNodes.length > 0) {
const nodeList = JSON.stringify(outDatedNodes.map((n: any) => n.name));
throw new Error(
`Some nodes are not on minimum version (${minMajor}.${minMinor}.0) required: ${nodeList}`
);
}
};
const stopConsumers = async (reindexOp: ReindexSavedObject) => {
if (isMlIndex(reindexOp.attributes.indexName)) {
await stopMlJobs();
} else if (isWatcherIndex(reindexOp.attributes.indexName)) {
await stopWatcher();
}
return actions.updateReindexOp(reindexOp, {
lastCompletedStep: ReindexStep.indexConsumersStopped,
});
};
/**
* Sets the original index as readonly so new data can be indexed until the reindex
* is completed.
* @param reindexOp
*/
const setReadonly = async (reindexOp: ReindexSavedObject) => {
const { indexName } = reindexOp.attributes;
const putReadonly = await callCluster('indices.putSettings', {
index: indexName,
body: { 'index.blocks.write': true },
});
if (!putReadonly.acknowledged) {
throw new Error(`Index could not be set to readonly.`);
}
return actions.updateReindexOp(reindexOp, { lastCompletedStep: ReindexStep.readonly });
};
/**
* Creates a new index with the same mappings and settings as the original index.
* @param reindexOp
*/
const createNewIndex = async (reindexOp: ReindexSavedObject) => {
const { indexName, newIndexName } = reindexOp.attributes;
const flatSettings = await actions.getFlatSettings(indexName);
if (!flatSettings) {
throw Boom.notFound(`Index ${indexName} does not exist.`);
}
const { settings, mappings } = transformFlatSettings(flatSettings);
const createIndex = await callCluster('indices.create', {
index: newIndexName,
body: {
settings,
mappings,
},
});
if (!createIndex.acknowledged) {
throw Boom.badImplementation(`Index could not be created: ${newIndexName}`);
}
return actions.updateReindexOp(reindexOp, {
lastCompletedStep: ReindexStep.newIndexCreated,
});
};
/**
* Begins the reindex process via Elasticsearch's Reindex API.
* @param reindexOp
*/
const startReindexing = async (reindexOp: ReindexSavedObject) => {
const { indexName } = reindexOp.attributes;
const reindexBody = {
source: { index: indexName },
dest: { index: reindexOp.attributes.newIndexName },
} as any;
const booleanFieldPaths = await actions.getBooleanFieldPaths(indexName);
if (booleanFieldPaths.length) {
reindexBody.script = {
lang: 'painless',
source: `
// Updates a single field in a Map
void updateField(Map data, String fieldName) {
if (
data[fieldName] == 'yes' ||
data[fieldName] == '1' ||
(data[fieldName] instanceof Integer && data[fieldName] == 1) ||
data[fieldName] == 'on'
) {
data[fieldName] = true;
} else if (
data[fieldName] == 'no' ||
data[fieldName] == '0' ||
(data[fieldName] instanceof Integer && data[fieldName] == 0) ||
data[fieldName] == 'off'
) {
data[fieldName] = false;
}
}
// Recursively walks the fieldPath list and calls
void updateFieldPath(def data, List fieldPath) {
String pathHead = fieldPath[0];
if (fieldPath.getLength() == 1) {
if (data.get(pathHead) !== null) {
updateField(data, pathHead);
}
} else {
List fieldPathTail = fieldPath.subList(1, fieldPath.getLength());
if (data.get(pathHead) instanceof List) {
for (item in data[pathHead]) {
updateFieldPath(item, fieldPathTail);
}
} else if (data.get(pathHead) instanceof Map) {
updateFieldPath(data[pathHead], fieldPathTail);
}
}
}
for (fieldPath in params.booleanFieldPaths) {
updateFieldPath(ctx._source, fieldPath)
}
`,
params: { booleanFieldPaths },
};
}
const startReindex = (await callCluster('reindex', {
refresh: true,
waitForCompletion: false,
body: reindexBody,
})) as { task: string };
return actions.updateReindexOp(reindexOp, {
lastCompletedStep: ReindexStep.reindexStarted,
reindexTaskId: startReindex.task,
reindexTaskPercComplete: 0,
});
};
/**
* Polls Elasticsearch's Tasks API to see if the reindex operation has been completed.
* @param reindexOp
*/
const updateReindexStatus = async (reindexOp: ReindexSavedObject) => {
const taskId = reindexOp.attributes.reindexTaskId;
// Check reindexing task progress
const taskResponse = await callCluster('tasks.get', {
taskId,
waitForCompletion: false,
});
if (taskResponse.completed) {
const { count } = await callCluster('count', { index: reindexOp.attributes.indexName });
if (taskResponse.task.status.created < count) {
if (taskResponse.response.failures && taskResponse.response.failures.length > 0) {
const failureExample = JSON.stringify(taskResponse.response.failures[0]);
throw Boom.badData(`Reindexing failed with failures like: ${failureExample}`);
} else {
throw Boom.badData('Reindexing failed due to new documents created in original index.');
}
}
// Delete the task from ES .tasks index
const deleteTaskResp = await callCluster('delete', {
index: '.tasks',
type: 'task',
id: taskId,
});
if (deleteTaskResp.result !== 'deleted') {
throw Boom.badImplementation(`Could not delete reindexing task ${taskId}`);
}
// Update the status
return actions.updateReindexOp(reindexOp, {
lastCompletedStep: ReindexStep.reindexCompleted,
reindexTaskPercComplete: 1,
});
} else {
const perc = taskResponse.task.status.created / taskResponse.task.status.total;
return actions.updateReindexOp(reindexOp, {
reindexTaskPercComplete: perc,
});
}
};
/**
* Creates an alias that points the old index to the new index, deletes the old index.
* @param reindexOp
*/
const switchAlias = async (reindexOp: ReindexSavedObject) => {
const { indexName, newIndexName } = reindexOp.attributes;
const existingAliases = (await callCluster('indices.getAlias', {
index: indexName,
}))[indexName].aliases;
const extraAlises = Object.keys(existingAliases).map(aliasName => ({
add: { index: newIndexName, alias: aliasName, ...existingAliases[aliasName] },
}));
const aliasResponse = await callCluster('indices.updateAliases', {
body: {
actions: [
{ add: { index: newIndexName, alias: indexName } },
{ remove_index: { index: indexName } },
...extraAlises,
],
},
});
if (!aliasResponse.acknowledged) {
throw Boom.badImplementation(`Index aliases could not be created.`);
}
return actions.updateReindexOp(reindexOp, {
lastCompletedStep: ReindexStep.aliasCreated,
});
};
const resumeConsumers = async (reindexOp: ReindexSavedObject) => {
if (isMlIndex(reindexOp.attributes.indexName)) {
await resumeMlJobs();
} else if (isWatcherIndex(reindexOp.attributes.indexName)) {
await startWatcher();
}
// Only change the status if we're still in-progress (this function is also called when the reindex fails)
if (reindexOp.attributes.status === ReindexStatus.inProgress) {
return actions.updateReindexOp(reindexOp, {
lastCompletedStep: ReindexStep.indexConsumersStarted,
status: ReindexStatus.completed,
});
} else {
return reindexOp;
}
};
// ------ The service itself
return {
async detectReindexWarnings(indexName: string) {
const flatSettings = await actions.getFlatSettings(indexName);
if (!flatSettings) {
return null;
} else {
return getReindexWarnings(flatSettings);
}
},
async createReindexOperation(indexName: string) {
const indexExists = await callCluster('indices.exists', { index: indexName });
if (!indexExists) {
throw Boom.notFound(`Index ${indexName} does not exist in this cluster.`);
}
const existingReindexOps = await actions.findReindexOperations(indexName);
if (existingReindexOps.total !== 0) {
const existingOp = existingReindexOps.saved_objects[0];
if (existingOp.attributes.status === ReindexStatus.failed) {
// Delete the existing one if it failed to give a chance to retry.
await actions.deleteReindexOp(existingOp);
} else {
throw Boom.badImplementation(`A reindex operation already in-progress for ${indexName}`);
}
}
return actions.createReindexOp(indexName);
},
async findReindexOperation(indexName: string) {
const findResponse = await actions.findReindexOperations(indexName);
// Bail early if it does not exist or there is more than one.
if (findResponse.total === 0) {
return null;
} else if (findResponse.total > 1) {
throw Boom.badImplementation(`More than one reindex operation found for ${indexName}`);
}
return findResponse.saved_objects[0];
},
findAllByStatus: actions.findAllByStatus,
async processNextStep(reindexOp: ReindexSavedObject) {
return actions.runWhileLocked(reindexOp, async lockedReindexOp => {
try {
switch (lockedReindexOp.attributes.lastCompletedStep) {
case ReindexStep.created:
lockedReindexOp = await stopConsumers(lockedReindexOp);
break;
case ReindexStep.indexConsumersStopped:
lockedReindexOp = await setReadonly(lockedReindexOp);
break;
case ReindexStep.readonly:
lockedReindexOp = await createNewIndex(lockedReindexOp);
break;
case ReindexStep.newIndexCreated:
lockedReindexOp = await startReindexing(lockedReindexOp);
break;
case ReindexStep.reindexStarted:
lockedReindexOp = await updateReindexStatus(lockedReindexOp);
break;
case ReindexStep.reindexCompleted:
lockedReindexOp = await switchAlias(lockedReindexOp);
break;
case ReindexStep.aliasCreated:
lockedReindexOp = await resumeConsumers(lockedReindexOp);
break;
default:
break;
}
} catch (e) {
// Trap the exception and add the message to the object so the UI can display it.
lockedReindexOp = await actions.updateReindexOp(lockedReindexOp, {
status: ReindexStatus.failed,
errorMessage: e instanceof Error ? e.stack : e.toString(),
});
// Cleanup any changes, ignoring any errors.
await cleanupChanges(lockedReindexOp).catch(e => undefined);
}
return lockedReindexOp;
});
},
async pauseReindexOperation(indexName: string) {
const reindexOp = await this.findReindexOperation(indexName);
if (!reindexOp) {
throw new Error(`No reindex operation found for index ${indexName}`);
}
return actions.runWhileLocked(reindexOp, async op => {
if (op.attributes.status === ReindexStatus.paused) {
// Another node already paused the operation, don't do anything
return reindexOp;
} else if (op.attributes.status !== ReindexStatus.inProgress) {
throw new Error(`Reindex operation must be inProgress in order to be paused.`);
}
return actions.updateReindexOp(op, { status: ReindexStatus.paused });
});
},
async resumeReindexOperation(indexName: string) {
const reindexOp = await this.findReindexOperation(indexName);
if (!reindexOp) {
throw new Error(`No reindex operation found for index ${indexName}`);
}
return actions.runWhileLocked(reindexOp, async op => {
if (op.attributes.status === ReindexStatus.inProgress) {
// Another node already resumed the operation, don't do anything
return reindexOp;
} else if (op.attributes.status !== ReindexStatus.paused) {
throw new Error(`Reindex operation must be paused in order to be resumed.`);
}
return actions.updateReindexOp(op, { status: ReindexStatus.inProgress });
});
},
};
};
const isMlIndex = (indexName: string) =>
indexName.startsWith('.ml-state') || indexName.startsWith('.ml-anomalies');
const isWatcherIndex = (indexName: string) => indexName.startsWith('.watches');

27
x-pack/plugins/upgrade_assistant/server/lib/reindexing/types.ts Normal file
View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
interface Mapping {
type?: string;
properties?: MappingProperties;
}
export interface MappingProperties {
[key: string]: Mapping;
}
export interface TypeMapping extends Mapping {
_all?: { enabled: boolean };
}
export interface FlatSettings {
settings: {
[key: string]: string;
};
mappings: {
[type: string]: TypeMapping;
};
}

181
x-pack/plugins/upgrade_assistant/server/lib/reindexing/worker.ts Normal file
View file

@ -0,0 +1,181 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { CallCluster, CallClusterWithRequest } from 'src/legacy/core_plugins/elasticsearch';
import { Request, Server } from 'src/server/kbn_server';
import { SavedObjectsClient } from 'src/server/saved_objects';
import moment = require('moment');
import { ReindexSavedObject, ReindexStatus } from '../../../common/types';
import { CredentialStore } from './credential_store';
import { reindexActionsFactory } from './reindex_actions';
import { ReindexService, reindexServiceFactory } from './reindex_service';
const POLL_INTERVAL = 30000;
// If no nodes have been able to update this index in 2 minutes (due to missing credentials), set to paused.
const PAUSE_WINDOW = POLL_INTERVAL * 4;
const LOG_TAGS = ['upgrade_assistant', 'reindex_worker'];
/**
* A singleton worker that will coordinate two polling loops:
* (1) A longer loop that polls for reindex operations that are in progress. If any are found, loop (2) is started.
* (2) A tighter loop that pushes each in progress reindex operation through ReindexService.processNextStep. If all
* updated reindex operations are complete, this loop will terminate.
*
* The worker can also be forced to start loop (2) by calling forceRefresh(). This is done when we know a new reindex
* operation has been started.
*
* This worker can be ran on multiple nodes without conflicts or dropped jobs. Reindex operations are locked by the
* ReindexService and if any operation is locked longer than the ReindexService's timeout, it is assumed to have been
* locked by a node that is no longer running (crashed or shutdown). In this case, another node may safely acquire
* the lock for this reindex operation.
*/
export class ReindexWorker {
private static workerSingleton?: ReindexWorker;
private continuePolling: boolean = false;
private updateOperationLoopRunning: boolean = false;
private inProgressOps: ReindexSavedObject[] = [];
private readonly reindexService: ReindexService;
constructor(
private client: SavedObjectsClient,
private credentialStore: CredentialStore,
private callWithRequest: CallClusterWithRequest,
private callWithInternalUser: CallCluster,
private readonly log: Server['log']
) {
if (ReindexWorker.workerSingleton) {
throw new Error(`More than one ReindexWorker cannot be created.`);
}
this.reindexService = reindexServiceFactory(
this.callWithInternalUser,
reindexActionsFactory(this.client, this.callWithInternalUser)
);
ReindexWorker.workerSingleton = this;
}
/**
* Begins loop (1) to begin checking for in progress reindex operations.
*/
public start = () => {
this.log(['debug', ...LOG_TAGS], `Starting worker...`);
this.continuePolling = true;
this.pollForOperations();
};
/**
* Stops the worker from processing any further reindex operations.
*/
public stop = () => {
this.log(['debug', ...LOG_TAGS], `Stopping worker...`);
this.updateOperationLoopRunning = false;
this.continuePolling = false;
};
/**
* Should be called immediately after this server has started a new reindex operation.
*/
public forceRefresh = () => {
this.refresh();
};
/**
* Returns whether or not the given ReindexOperation is in the worker's queue.
*/
public includes = (reindexOp: ReindexSavedObject) => {
return this.inProgressOps.map(o => o.id).includes(reindexOp.id);
};
/**
* Runs an async loop until all inProgress jobs are complete or failed.
*/
private startUpdateOperationLoop = async () => {
this.updateOperationLoopRunning = true;
while (this.inProgressOps.length > 0) {
this.log(['debug', ...LOG_TAGS], `Updating ${this.inProgressOps.length} reindex operations`);
// Push each operation through the state machine and refresh.
await Promise.all(this.inProgressOps.map(this.processNextStep));
await this.refresh();
}
this.updateOperationLoopRunning = false;
};
private pollForOperations = async () => {
this.log(['debug', ...LOG_TAGS], `Polling for reindex operations`);
await this.refresh();
if (this.continuePolling) {
setTimeout(this.pollForOperations, POLL_INTERVAL);
}
};
private refresh = async () => {
this.inProgressOps = await this.reindexService.findAllByStatus(ReindexStatus.inProgress);
// If there are operations in progress and we're not already updating operations, kick off the update loop
if (!this.updateOperationLoopRunning) {
this.startUpdateOperationLoop();
}
};
private processNextStep = async (reindexOp: ReindexSavedObject) => {
const credential = this.credentialStore.get(reindexOp);
if (!credential) {
// Set to paused state if the job hasn't been updated in PAUSE_WINDOW.
// This indicates that no Kibana nodes currently have credentials to update this job.
const now = moment();
const updatedAt = moment(reindexOp.updated_at);
if (updatedAt < now.subtract(PAUSE_WINDOW)) {
return this.reindexService.pauseReindexOperation(reindexOp.attributes.indexName);
} else {
// If it has been updated recently, we assume another node has the necessary credentials,
// and this becomes a noop.
return reindexOp;
}
}
// Setup a ReindexService specific to these credentials.
const fakeRequest = { headers: credential } as Request;
const callCluster = this.callWithRequest.bind(null, fakeRequest) as CallCluster;
const actions = reindexActionsFactory(this.client, callCluster);
const service = reindexServiceFactory(callCluster, actions);
reindexOp = await swallowExceptions(service.processNextStep, this.log)(reindexOp);
// Update credential store with most recent state.
this.credentialStore.set(reindexOp, credential);
};
}
/**
* Swallows any exceptions that may occur during the reindex process. This prevents any errors from
* stopping the worker from continuing to process more jobs.
*/
const swallowExceptions = (
func: (reindexOp: ReindexSavedObject) => Promise<ReindexSavedObject>,
log: Server['log']
) => async (reindexOp: ReindexSavedObject) => {
try {
return await func(reindexOp);
} catch (e) {
if (reindexOp.attributes.locked) {
log(['debug', ...LOG_TAGS], `Skipping reindexOp with unexpired lock: ${reindexOp.id}`);
} else {
log(
['warning', ...LOG_TAGS],
`Error when trying to process reindexOp (${reindexOp.id}): ${e.toString()}`
);
}
return reindexOp;
}
};

4
x-pack/plugins/upgrade_assistant/server/routes/cluster_checkup.test.ts
View file

@ -17,7 +17,7 @@ MigrationApis.getUpgradeAssistantStatus = jest.fn();
* to ensure they're wired up to the lib functions correctly. Business logic is tested
* more thoroughly in the es_migration_apis test.
*/
describe('reindex template API', () => {
describe('cluster checkup API', () => {
const server = new Server();
server.plugins = {
elasticsearch: {
@ -29,7 +29,7 @@ describe('reindex template API', () => {
registerClusterCheckupRoutes(server);
describe('GET /api/upgrade_assistant/reindex/{indexName}.json', () => {
it('returns a template', async () => {
it('returns state', async () => {
MigrationApis.getUpgradeAssistantStatus.mockResolvedValue({
cluster: [],
indices: [],

193
x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.test.ts Normal file
View file

@ -0,0 +1,193 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Server } from 'hapi';
const mockReindexService = {
detectReindexWarnings: jest.fn(),
createReindexOperation: jest.fn(),
findAllInProgressOperations: jest.fn(),
findReindexOperation: jest.fn(),
processNextStep: jest.fn(),
resumeReindexOperation: jest.fn(),
};
jest.mock('../lib/reindexing', () => {
return {
reindexServiceFactory: () => mockReindexService,
};
});
import { ReindexSavedObject, ReindexStatus, ReindexWarning } from '../../common/types';
import { credentialStoreFactory } from '../lib/reindexing/credential_store';
import { registerReindexIndicesRoutes } from './reindex_indices';
// Need to require to get mock on named export to work.
// tslint:disable:no-var-requires
// const MigrationApis = require('../lib/es_migration_apis');
// MigrationApis.getUpgradeAssistantStatus = jest.fn();
/**
* Since these route callbacks are so thin, these serve simply as integration tests
* to ensure they're wired up to the lib functions correctly. Business logic is tested
* more thoroughly in the es_migration_apis test.
*/
describe('reindex template API', () => {
const server = new Server();
server.plugins = {
elasticsearch: {
getCluster: () => ({ callWithRequest: jest.fn() } as any),
} as any,
} as any;
server.config = () => ({ get: () => '' } as any);
server.decorate('request', 'getSavedObjectsClient', () => jest.fn());
const credentialStore = credentialStoreFactory();
const worker = {
includes: jest.fn(),
forceRefresh: jest.fn(),
} as any;
registerReindexIndicesRoutes(server, worker, credentialStore);
beforeEach(() => {
mockReindexService.detectReindexWarnings.mockReset();
mockReindexService.createReindexOperation.mockReset();
mockReindexService.findAllInProgressOperations.mockReset();
mockReindexService.findReindexOperation.mockReset();
mockReindexService.processNextStep.mockReset();
mockReindexService.resumeReindexOperation.mockReset();
worker.includes.mockReset();
worker.forceRefresh.mockReset();
// Reset the credentialMap
credentialStore.clear();
});
describe('GET /api/upgrade_assistant/reindex/{indexName}', () => {
it('returns the attributes of the reindex operation and reindex warnings', async () => {
mockReindexService.findReindexOperation.mockResolvedValueOnce({
attributes: { indexName: 'wowIndex', status: ReindexStatus.inProgress },
});
mockReindexService.detectReindexWarnings.mockResolvedValueOnce([ReindexWarning.allField]);
const resp = await server.inject({
method: 'GET',
url: `/api/upgrade_assistant/reindex/wowIndex`,
});
// It called into the service correctly
expect(mockReindexService.findReindexOperation).toHaveBeenCalledWith('wowIndex');
expect(mockReindexService.detectReindexWarnings).toHaveBeenCalledWith('wowIndex');
// It returned the right results
expect(resp.statusCode).toEqual(200);
const data = JSON.parse(resp.payload);
expect(data.reindexOp).toEqual({ indexName: 'wowIndex', status: ReindexStatus.inProgress });
expect(data.warnings).toEqual([0]);
});
it("returns null for both if reindex operation doesn't exist and index doesn't exist", async () => {
mockReindexService.findReindexOperation.mockResolvedValueOnce(null);
mockReindexService.detectReindexWarnings.mockResolvedValueOnce(null);
const resp = await server.inject({
method: 'GET',
url: `/api/upgrade_assistant/reindex/anIndex`,
});
expect(resp.statusCode).toEqual(200);
const data = JSON.parse(resp.payload);
expect(data).toEqual({ warnings: null, reindexOp: null });
});
});
describe('POST /api/upgrade_assistant/reindex/{indexName}', () => {
it('creates a new reindexOp', async () => {
mockReindexService.createReindexOperation.mockResolvedValueOnce({
attributes: { indexName: 'theIndex' },
});
const resp = await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/reindex/theIndex',
});
// It called create correctly
expect(mockReindexService.createReindexOperation).toHaveBeenCalledWith('theIndex');
// It returned the right results
expect(resp.statusCode).toEqual(200);
const data = JSON.parse(resp.payload);
expect(data).toEqual({ indexName: 'theIndex' });
});
it('calls worker.forceRefresh', async () => {
mockReindexService.createReindexOperation.mockResolvedValueOnce({
attributes: { indexName: 'theIndex' },
});
await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/reindex/theIndex',
});
expect(worker.forceRefresh).toHaveBeenCalled();
});
it('inserts headers into the credentialStore', async () => {
const reindexOp = {
attributes: { indexName: 'theIndex' },
} as ReindexSavedObject;
mockReindexService.createReindexOperation.mockResolvedValueOnce(reindexOp);
await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/reindex/theIndex',
headers: {
'kbn-auth-x': 'HERE!',
},
});
expect(credentialStore.get(reindexOp)!['kbn-auth-x']).toEqual('HERE!');
});
it('resumes a reindexOp if it is paused', async () => {
mockReindexService.findReindexOperation.mockResolvedValueOnce({
attributes: { indexName: 'theIndex', status: ReindexStatus.paused },
});
mockReindexService.resumeReindexOperation.mockResolvedValueOnce({
attributes: { indexName: 'theIndex', status: ReindexStatus.inProgress },
});
const resp = await server.inject({
method: 'POST',
url: '/api/upgrade_assistant/reindex/theIndex',
});
// It called resume correctly
expect(mockReindexService.resumeReindexOperation).toHaveBeenCalledWith('theIndex');
expect(mockReindexService.createReindexOperation).not.toHaveBeenCalled();
// It returned the right results
expect(resp.statusCode).toEqual(200);
const data = JSON.parse(resp.payload);
expect(data).toEqual({ indexName: 'theIndex', status: ReindexStatus.inProgress });
});
});
describe('DELETE /api/upgrade_assistant/reindex/{indexName}', () => {
it('returns a 501', async () => {
const resp = await server.inject({
method: 'DELETE',
url: '/api/upgrade_assistant/reindex/cancelMe',
});
expect(resp.statusCode).toEqual(501);
});
});
});

136
x-pack/plugins/upgrade_assistant/server/routes/reindex_indices.ts Normal file
View file

@ -0,0 +1,136 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import { Server } from 'hapi';
import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
import { SavedObjectsClient } from 'src/server/saved_objects';
import { ReindexStatus } from '../../common/types';
import { reindexServiceFactory, ReindexWorker } from '../lib/reindexing';
import { CredentialStore } from '../lib/reindexing/credential_store';
import { reindexActionsFactory } from '../lib/reindexing/reindex_actions';
export function registerReindexWorker(server: Server, credentialStore: CredentialStore) {
const { callWithRequest, callWithInternalUser } = server.plugins.elasticsearch.getCluster(
'admin'
);
const savedObjectsRepository = server.savedObjects.getSavedObjectsRepository(
callWithInternalUser
);
const savedObjectsClient = new server.savedObjects.SavedObjectsClient(
savedObjectsRepository
) as SavedObjectsClient;
// Cannot pass server.log directly because it's value changes during startup (?).
// Use this function to proxy through.
const log: Server['log'] = (
tags: string | string[],
data?: string | object | (() => any),
timestamp?: number
) => server.log(tags, data, timestamp);
const worker = new ReindexWorker(
savedObjectsClient,
credentialStore,
callWithRequest,
callWithInternalUser,
log
);
// Wait for ES connection before starting the polling loop.
server.plugins.elasticsearch.waitUntilReady().then(() => {
worker.start();
server.events.on('stop', () => worker.stop());
});
return worker;
}
export function registerReindexIndicesRoutes(
server: Server,
worker: ReindexWorker,
credentialStore: CredentialStore
) {
const { callWithRequest } = server.plugins.elasticsearch.getCluster('admin');
const BASE_PATH = '/api/upgrade_assistant/reindex';
// Start reindex for an index
server.route({
path: `${BASE_PATH}/{indexName}`,
method: 'POST',
async handler(request) {
const client = request.getSavedObjectsClient();
const { indexName } = request.params;
const callCluster = callWithRequest.bind(null, request) as CallCluster;
const reindexActions = reindexActionsFactory(client, callCluster);
const reindexService = reindexServiceFactory(callCluster, reindexActions);
try {
const existingOp = await reindexService.findReindexOperation(indexName);
// If the reindexOp already exists and it's paused, resume it. Otherwise create a new one.
const reindexOp =
existingOp && existingOp.attributes.status === ReindexStatus.paused
? await reindexService.resumeReindexOperation(indexName)
: await reindexService.createReindexOperation(indexName);
// Add users credentials for the worker to use
credentialStore.set(reindexOp, request.headers);
// Kick the worker on this node to immediately pickup the new reindex operation.
worker.forceRefresh();
return reindexOp.attributes;
} catch (e) {
if (!e.isBoom) {
return Boom.boomify(e, { statusCode: 500 });
}
return e;
}
},
});
// Get status
server.route({
path: `${BASE_PATH}/{indexName}`,
method: 'GET',
async handler(request) {
const client = request.getSavedObjectsClient();
const { indexName } = request.params;
const callCluster = callWithRequest.bind(null, request) as CallCluster;
const reindexActions = reindexActionsFactory(client, callCluster);
const reindexService = reindexServiceFactory(callCluster, reindexActions);
try {
const reindexOp = await reindexService.findReindexOperation(indexName);
const reindexWarnings = await reindexService.detectReindexWarnings(indexName);
return {
warnings: reindexWarnings,
reindexOp: reindexOp ? reindexOp.attributes : null,
};
} catch (e) {
if (!e.isBoom) {
return Boom.boomify(e, { statusCode: 500 });
}
return e;
}
},
});
// Cancel reindex
server.route({
path: `${BASE_PATH}/{indexName}`,
method: 'DELETE',
async handler(request) {
return Boom.notImplemented();
},
});
}

1
x-pack/scripts/functional_tests.js
View file

@ -20,4 +20,5 @@ require('@kbn/test').runTestsCli([
require.resolve('../test/saved_object_api_integration/security_and_spaces/config'),
require.resolve('../test/saved_object_api_integration/security_only/config'),
require.resolve('../test/saved_object_api_integration/spaces_only/config'),
require.resolve('../test/upgrade_assistant_integration/config'),
]);

77
x-pack/test/functional/es_archives/upgrade_assistant/reindex/data.json Normal file
View file

@ -0,0 +1,77 @@
{
"type": "doc",
"value": {
"index": "dummydata",
"type": "_doc",
"source": {
"@timestamp": "2018-10-30T18:51:56.792Z",
"host": {
"name": "foo.home",
"architecture": "x86_64",
"os": {
"version": "10.14",
"family": "darwin",
"build": "18A391",
"platform": "darwin"
}
},
"versions": [
"1.0.0",
"8.8.4"
],
"https": true,
"response_ms": 114
}
}
}
{
"type": "doc",
"value": {
"index": "dummydata",
"type": "_doc",
"source": {
"@timestamp": "2018-12-30T18:51:56.792Z",
"host": {
"name": "bar.home",
"architecture": "x86_64",
"os": {
"version": "10.12",
"family": "darwin",
"build": "18AXX",
"platform": "darwin"
}
},
"versions": [
"0.4"
],
"https": false,
"response_ms": 1567
}
}
}
{
"type": "doc",
"value": {
"index": "dummydata",
"type": "_doc",
"source": {
"@timestamp": "2018-01-30T18:51:56.792Z",
"host": {
"name": "qux.home",
"architecture": "x86_64",
"os": {
"version": "3.24",
"family": "linux",
"build": "YYY",
"platform": "linux"
}
},
"versions": [
],
"https": true,
"response_ms": 94
}
}
}

43
x-pack/test/upgrade_assistant_integration/config.js Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import path from 'path';
import {
EsProvider,
} from './services';
export default async function ({ readConfigFile }) {
// Read the Kibana API integration tests config file so that we can utilize its services.
const kibanaAPITestsConfig = await readConfigFile(require.resolve('../../../test/api_integration/config.js'));
const xPackFunctionalTestsConfig = await readConfigFile(require.resolve('../functional/config.js'));
const kibanaCommonConfig = await readConfigFile(require.resolve('../../../test/common/config.js'));
return {
testFiles: [require.resolve('./upgrade_assistant')],
servers: xPackFunctionalTestsConfig.get('servers'),
services: {
supertest: kibanaAPITestsConfig.get('services.supertest'),
es: EsProvider,
esArchiver: kibanaCommonConfig.get('services.esArchiver'),
},
esArchiver: xPackFunctionalTestsConfig.get('esArchiver'),
junit: {
reportName: 'X-Pack Upgrade Assistant Integration Tests',
},
kbnTestServer: {
...xPackFunctionalTestsConfig.get('kbnTestServer'),
serverArgs: [
...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'),
'--optimize.enabled=false',
],
},
esTestCluster: {
...xPackFunctionalTestsConfig.get('esTestCluster'),
dataArchive: path.resolve(__dirname, './fixtures/data_archives/upgrade_assistant.zip'),
}
};
}

BIN
x-pack/test/upgrade_assistant_integration/fixtures/data_archives/upgrade_assistant.zip Normal file
View file

Binary file not shown.

18
x-pack/test/upgrade_assistant_integration/services/es.js Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { format as formatUrl } from 'url';
import elasticsearch from 'elasticsearch';
export function EsProvider({ getService }) {
const config = getService('config');
return new elasticsearch.Client({
host: formatUrl(config.get('servers.elasticsearch')),
requestTimeout: config.get('timeouts.esRequestTimeout'),
});
}

7
x-pack/test/upgrade_assistant_integration/services/index.js Normal file
View file

@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { EsProvider } from './es';

13
x-pack/test/upgrade_assistant_integration/upgrade_assistant/index.js Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export default function ({ loadTestFile }) {
describe('upgrade assistant', function () {
this.tags('ciGroup5');
loadTestFile(require.resolve('./reindexing'));
});
}

169
x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js Normal file
View file

@ -0,0 +1,169 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from 'expect.js';
import { ReindexStatus, REINDEX_OP_TYPE, ReindexWarning } from '../../../plugins/upgrade_assistant/common/types';
export default function ({ getService }) {
const supertest = getService('supertest');
const esArchiver = getService('esArchiver');
const es = getService('es');
// Utility function that keeps polling API until reindex operation has completed or failed.
const waitForReindexToComplete = async (indexName) => {
console.log(`Waiting for reindex to complete...`);
let lastState;
while (true) {
lastState = (await supertest.get(`/api/upgrade_assistant/reindex/${indexName}`).expect(200)).body.reindexOp;
// Once the operation is completed or failed and unlocked, stop polling.
if (lastState.status !== ReindexStatus.inProgress && lastState.locked === null) {
break;
}
await new Promise(resolve => setTimeout(resolve, 1000));
}
return lastState;
};
describe('reindexing', () => {
afterEach(() => {
// Cleanup saved objects
return es.deleteByQuery({
index: '.kibana',
refresh: true,
body: {
query: {
"simple_query_string": {
query: REINDEX_OP_TYPE,
fields: ["type"]
}
}
}
});
});
it('should create a new index with the same documents', async () => {
await esArchiver.load('upgrade_assistant/reindex');
const { body } = await supertest
.post(`/api/upgrade_assistant/reindex/dummydata`)
.set('kbn-xsrf', 'xxx')
.expect(200);
expect(body.indexName).to.equal('dummydata');
expect(body.status).to.equal(ReindexStatus.inProgress);
const lastState = await waitForReindexToComplete('dummydata');
expect(lastState.errorMessage).to.equal(null);
expect(lastState.status).to.equal(ReindexStatus.completed);
const { newIndexName } = lastState;
const indexSummary = await es.indices.get({ index: 'dummydata' });
// The new index was created
expect(indexSummary[newIndexName]).to.be.an('object');
// The original index name is aliased to the new one
expect(indexSummary[newIndexName].aliases.dummydata).to.be.an('object');
// The number of documents in the new index matches what we expect
expect(
(await es.count({ index: lastState.newIndexName })).count
).to.be(3);
// Cleanup newly created index
await es.indices.delete({
index: lastState.newIndexName
});
});
it('should update any aliases', async () => {
await esArchiver.load('upgrade_assistant/reindex');
// Add aliases and ensure each returns the right number of docs
await es.indices.updateAliases({
body: {
actions: [
{ add: { index: 'dummydata', alias: 'myAlias' } },
{ add: { index: 'dummy*', alias: 'wildcardAlias' } },
{ add: { index: 'dummydata', alias: 'myHttpsAlias', filter: { term: { https: true } } } }
]
}
});
expect(
(await es.count({ index: 'myAlias' })).count
).to.be(3);
expect(
(await es.count({ index: 'wildcardAlias' })).count
).to.be(3);
expect(
(await es.count({ index: 'myHttpsAlias' })).count
).to.be(2);
// Reindex
await supertest
.post(`/api/upgrade_assistant/reindex/dummydata`)
.set('kbn-xsrf', 'xxx')
.expect(200);
const lastState = await waitForReindexToComplete('dummydata');
// The regular aliases should still return 3 docs
expect(
(await es.count({ index: 'myAlias' })).count
).to.be(3);
expect(
(await es.count({ index: 'wildcardAlias' })).count
).to.be(3);
// The filtered alias should still return 2 docs
expect(
(await es.count({ index: 'myHttpsAlias' })).count
).to.be(2);
// Cleanup newly created index
await es.indices.delete({
index: lastState.newIndexName
});
});
it('shows warnings for boolean fields', async () => {
const resp = await supertest.get(`/api/upgrade_assistant/reindex/boolean-test`);
expect(resp.body.warnings.includes(ReindexWarning.booleanFields)).to.be(true);
});
it('shows warnings for all meta field', async () => {
const resp = await supertest.get(`/api/upgrade_assistant/reindex/all-field-test`);
expect(resp.body.warnings.includes(ReindexWarning.allField)).to.be(true);
});
it('reindexes index with boolean fields', async () => {
const { body } = await supertest
.post(`/api/upgrade_assistant/reindex/boolean-test`)
.set('kbn-xsrf', 'xxx')
.expect(200);
expect(body.indexName).to.equal('boolean-test');
expect(body.status).to.equal(ReindexStatus.inProgress);
const lastState = await waitForReindexToComplete('boolean-test');
expect(lastState.errorMessage).to.equal(null);
expect(lastState.status).to.equal(ReindexStatus.completed);
});
it('reindexes index with _all field', async () => {
const { body } = await supertest
.post(`/api/upgrade_assistant/reindex/all-field-test`)
.set('kbn-xsrf', 'xxx')
.expect(200);
expect(body.indexName).to.equal('all-field-test');
expect(body.status).to.equal(ReindexStatus.inProgress);
const lastState = await waitForReindexToComplete('all-field-test');
expect(lastState.errorMessage).to.equal(null);
expect(lastState.status).to.equal(ReindexStatus.completed);
});
});
}