mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[kbn/test] add import/export support to KbnClient (#92526)
Co-authored-by: Tre' Seymour <wayne.seymour@elastic.co> Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: spalger <spalger@users.noreply.github.com>
This commit is contained in:
parent
9c527347ef
commit
0aabc317ec
34 changed files with 546 additions and 25 deletions
|
@ -655,6 +655,7 @@
|
||||||
"fetch-mock": "^7.3.9",
|
"fetch-mock": "^7.3.9",
|
||||||
"file-loader": "^4.2.0",
|
"file-loader": "^4.2.0",
|
||||||
"file-saver": "^1.3.8",
|
"file-saver": "^1.3.8",
|
||||||
|
"form-data": "^4.0.0",
|
||||||
"formsy-react": "^1.1.5",
|
"formsy-react": "^1.1.5",
|
||||||
"geckodriver": "^1.21.0",
|
"geckodriver": "^1.21.0",
|
||||||
"glob-watcher": "5.0.3",
|
"glob-watcher": "5.0.3",
|
||||||
|
|
|
@ -23,7 +23,6 @@ export {
|
||||||
KBN_P12_PATH,
|
KBN_P12_PATH,
|
||||||
KBN_P12_PASSWORD,
|
KBN_P12_PASSWORD,
|
||||||
} from './certs';
|
} from './certs';
|
||||||
export * from './kbn_client';
|
|
||||||
export * from './run';
|
export * from './run';
|
||||||
export * from './axios';
|
export * from './axios';
|
||||||
export * from './stdio';
|
export * from './stdio';
|
||||||
|
|
|
@ -7,7 +7,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib';
|
import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib';
|
||||||
|
|
||||||
|
|
|
@ -9,7 +9,8 @@
|
||||||
import { resolve } from 'path';
|
import { resolve } from 'path';
|
||||||
import { createReadStream } from 'fs';
|
import { createReadStream } from 'fs';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
|
import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
|
||||||
import { ES_CLIENT_HEADERS } from '../client_headers';
|
import { ES_CLIENT_HEADERS } from '../client_headers';
|
||||||
|
|
|
@ -10,7 +10,8 @@ import { resolve } from 'path';
|
||||||
import { createReadStream } from 'fs';
|
import { createReadStream } from 'fs';
|
||||||
import { Readable, Writable } from 'stream';
|
import { Readable, Writable } from 'stream';
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
import { createPromiseFromStreams } from '@kbn/utils';
|
import { createPromiseFromStreams } from '@kbn/utils';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
|
|
@ -17,8 +17,8 @@ import Url from 'url';
|
||||||
import readline from 'readline';
|
import readline from 'readline';
|
||||||
import Fs from 'fs';
|
import Fs from 'fs';
|
||||||
|
|
||||||
import { RunWithCommands, createFlagError, KbnClient, CA_CERT_PATH } from '@kbn/dev-utils';
|
import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils';
|
||||||
import { readConfigFile } from '@kbn/test';
|
import { readConfigFile, KbnClient } from '@kbn/test';
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
|
|
||||||
import { EsArchiver } from './es_archiver';
|
import { EsArchiver } from './es_archiver';
|
||||||
|
|
|
@ -7,7 +7,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
saveAction,
|
saveAction,
|
||||||
|
|
|
@ -9,7 +9,8 @@
|
||||||
import { inspect } from 'util';
|
import { inspect } from 'util';
|
||||||
|
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
import { Stats } from '../stats';
|
import { Stats } from '../stats';
|
||||||
import { deleteIndex } from './delete_index';
|
import { deleteIndex } from './delete_index';
|
||||||
import { ES_CLIENT_HEADERS } from '../../client_headers';
|
import { ES_CLIENT_HEADERS } from '../../client_headers';
|
||||||
|
|
|
@ -213,6 +213,13 @@ export const schema = Joi.object()
|
||||||
})
|
})
|
||||||
.default(),
|
.default(),
|
||||||
|
|
||||||
|
// settings for the saved objects svc
|
||||||
|
kbnArchiver: Joi.object()
|
||||||
|
.keys({
|
||||||
|
directory: Joi.string().default(defaultRelativeToConfigPath('fixtures/kbn_archiver')),
|
||||||
|
})
|
||||||
|
.default(),
|
||||||
|
|
||||||
// settings for the kibanaServer.uiSettings module
|
// settings for the kibanaServer.uiSettings module
|
||||||
uiSettings: Joi.object()
|
uiSettings: Joi.object()
|
||||||
.keys({
|
.keys({
|
||||||
|
|
|
@ -48,3 +48,7 @@ export { getUrl } from './jest/utils/get_url';
|
||||||
export { runCheckJestConfigsCli } from './jest/run_check_jest_configs_cli';
|
export { runCheckJestConfigsCli } from './jest/run_check_jest_configs_cli';
|
||||||
|
|
||||||
export { runJest } from './jest/run';
|
export { runJest } from './jest/run';
|
||||||
|
|
||||||
|
export * from './kbn_archiver_cli';
|
||||||
|
|
||||||
|
export * from './kbn_client';
|
||||||
|
|
149
packages/kbn-test/src/kbn_archiver_cli.ts
Normal file
149
packages/kbn-test/src/kbn_archiver_cli.ts
Normal file
|
@ -0,0 +1,149 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import Path from 'path';
|
||||||
|
import Url from 'url';
|
||||||
|
|
||||||
|
import { RunWithCommands, createFlagError, Flags } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
|
import { readConfigFile } from './functional_test_runner';
|
||||||
|
|
||||||
|
function getSinglePositionalArg(flags: Flags) {
|
||||||
|
const positional = flags._;
|
||||||
|
if (positional.length < 1) {
|
||||||
|
throw createFlagError('missing name of export to import');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (positional.length > 1) {
|
||||||
|
throw createFlagError(`extra positional arguments, expected 1, got [${positional}]`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return positional[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTypesFlag(flags: Flags) {
|
||||||
|
if (!flags.type || (typeof flags.type !== 'string' && !Array.isArray(flags.type))) {
|
||||||
|
throw createFlagError('--type is a required flag');
|
||||||
|
}
|
||||||
|
|
||||||
|
const types = typeof flags.type === 'string' ? [flags.type] : flags.type;
|
||||||
|
return types.reduce(
|
||||||
|
(acc: string[], type) => [...acc, ...type.split(',').map((t) => t.trim())],
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runKbnArchiverCli() {
|
||||||
|
new RunWithCommands({
|
||||||
|
description: 'Import/export saved objects from archives, for testing',
|
||||||
|
globalFlags: {
|
||||||
|
string: ['config', 'space', 'kibana-url', 'dir'],
|
||||||
|
help: `
|
||||||
|
--space space id to operate on, defaults to the default space
|
||||||
|
--config optional path to an FTR config file that will be parsed and used for defaults
|
||||||
|
--kibana-url set the url that kibana can be reached at, uses the "servers.kibana" setting from --config by default
|
||||||
|
--dir directory that contains exports to be imported, or where exports will be saved, uses the "kbnArchiver.directory"
|
||||||
|
setting from --config by default
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
async extendContext({ log, flags }) {
|
||||||
|
let config;
|
||||||
|
if (flags.config) {
|
||||||
|
if (typeof flags.config !== 'string') {
|
||||||
|
throw createFlagError('expected --config to be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
config = await readConfigFile(log, Path.resolve(flags.config));
|
||||||
|
}
|
||||||
|
|
||||||
|
let kibanaUrl;
|
||||||
|
if (flags['kibana-url']) {
|
||||||
|
if (typeof flags['kibana-url'] !== 'string') {
|
||||||
|
throw createFlagError('expected --kibana-url to be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
kibanaUrl = flags['kibana-url'];
|
||||||
|
} else if (config) {
|
||||||
|
kibanaUrl = Url.format(config.get('servers.kibana'));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!kibanaUrl) {
|
||||||
|
throw createFlagError(
|
||||||
|
'Either a --config file with `servers.kibana` defined, or a --kibana-url must be passed'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let importExportDir;
|
||||||
|
if (flags.dir) {
|
||||||
|
if (typeof flags.dir !== 'string') {
|
||||||
|
throw createFlagError('expected --dir to be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
importExportDir = flags.dir;
|
||||||
|
} else if (config) {
|
||||||
|
importExportDir = config.get('kbnArchiver.directory');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!importExportDir) {
|
||||||
|
throw createFlagError(
|
||||||
|
'--config does not include a kbnArchiver.directory, specify it or include --dir flag'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const space = flags.space;
|
||||||
|
if (!(space === undefined || typeof space === 'string')) {
|
||||||
|
throw createFlagError('--space must be a string');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
space,
|
||||||
|
kbnClient: new KbnClient({
|
||||||
|
log,
|
||||||
|
url: kibanaUrl,
|
||||||
|
importExportDir,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.command({
|
||||||
|
name: 'save',
|
||||||
|
usage: 'save <name>',
|
||||||
|
description: 'export saved objects from Kibana to a file',
|
||||||
|
flags: {
|
||||||
|
string: ['type'],
|
||||||
|
help: `
|
||||||
|
--type saved object type that should be fetched and stored in the archive, can
|
||||||
|
be specified multiple times or be a comma-separated list.
|
||||||
|
`,
|
||||||
|
},
|
||||||
|
async run({ kbnClient, flags, space }) {
|
||||||
|
await kbnClient.importExport.save(getSinglePositionalArg(flags), {
|
||||||
|
types: parseTypesFlag(flags),
|
||||||
|
space,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.command({
|
||||||
|
name: 'load',
|
||||||
|
usage: 'load <name>',
|
||||||
|
description: 'import a saved export to Kibana',
|
||||||
|
async run({ kbnClient, flags, space }) {
|
||||||
|
await kbnClient.importExport.load(getSinglePositionalArg(flags), { space });
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.command({
|
||||||
|
name: 'unload',
|
||||||
|
usage: 'unload <name>',
|
||||||
|
description: 'delete the saved objects saved in the archive from the Kibana index',
|
||||||
|
async run({ kbnClient, flags, space }) {
|
||||||
|
await kbnClient.importExport.unload(getSinglePositionalArg(flags), { space });
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.execute();
|
||||||
|
}
|
|
@ -6,19 +6,22 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ToolingLog } from '../tooling_log';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
|
||||||
import { KbnClientRequester, ReqOptions } from './kbn_client_requester';
|
import { KbnClientRequester, ReqOptions } from './kbn_client_requester';
|
||||||
import { KbnClientStatus } from './kbn_client_status';
|
import { KbnClientStatus } from './kbn_client_status';
|
||||||
import { KbnClientPlugins } from './kbn_client_plugins';
|
import { KbnClientPlugins } from './kbn_client_plugins';
|
||||||
import { KbnClientVersion } from './kbn_client_version';
|
import { KbnClientVersion } from './kbn_client_version';
|
||||||
import { KbnClientSavedObjects } from './kbn_client_saved_objects';
|
import { KbnClientSavedObjects } from './kbn_client_saved_objects';
|
||||||
import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings';
|
import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings';
|
||||||
|
import { KbnClientImportExport } from './kbn_client_import_export';
|
||||||
|
|
||||||
export interface KbnClientOptions {
|
export interface KbnClientOptions {
|
||||||
url: string;
|
url: string;
|
||||||
certificateAuthorities?: Buffer[];
|
certificateAuthorities?: Buffer[];
|
||||||
log: ToolingLog;
|
log: ToolingLog;
|
||||||
uiSettingDefaults?: UiSettingValues;
|
uiSettingDefaults?: UiSettingValues;
|
||||||
|
importExportDir?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class KbnClient {
|
export class KbnClient {
|
||||||
|
@ -27,6 +30,7 @@ export class KbnClient {
|
||||||
readonly version: KbnClientVersion;
|
readonly version: KbnClientVersion;
|
||||||
readonly savedObjects: KbnClientSavedObjects;
|
readonly savedObjects: KbnClientSavedObjects;
|
||||||
readonly uiSettings: KbnClientUiSettings;
|
readonly uiSettings: KbnClientUiSettings;
|
||||||
|
readonly importExport: KbnClientImportExport;
|
||||||
|
|
||||||
private readonly requester: KbnClientRequester;
|
private readonly requester: KbnClientRequester;
|
||||||
private readonly log: ToolingLog;
|
private readonly log: ToolingLog;
|
||||||
|
@ -56,6 +60,12 @@ export class KbnClient {
|
||||||
this.version = new KbnClientVersion(this.status);
|
this.version = new KbnClientVersion(this.status);
|
||||||
this.savedObjects = new KbnClientSavedObjects(this.log, this.requester);
|
this.savedObjects = new KbnClientSavedObjects(this.log, this.requester);
|
||||||
this.uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults);
|
this.uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults);
|
||||||
|
this.importExport = new KbnClientImportExport(
|
||||||
|
this.log,
|
||||||
|
this.requester,
|
||||||
|
this.savedObjects,
|
||||||
|
options.importExportDir
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
163
packages/kbn-test/src/kbn_client/kbn_client_import_export.ts
Normal file
163
packages/kbn-test/src/kbn_client/kbn_client_import_export.ts
Normal file
|
@ -0,0 +1,163 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { inspect } from 'util';
|
||||||
|
import Fs from 'fs/promises';
|
||||||
|
import Path from 'path';
|
||||||
|
|
||||||
|
import FormData from 'form-data';
|
||||||
|
import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils';
|
||||||
|
|
||||||
|
import { KbnClientRequester, uriencode, ReqOptions } from './kbn_client_requester';
|
||||||
|
import { KbnClientSavedObjects } from './kbn_client_saved_objects';
|
||||||
|
|
||||||
|
interface ImportApiResponse {
|
||||||
|
success: boolean;
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SavedObject {
|
||||||
|
id: string;
|
||||||
|
type: string;
|
||||||
|
[key: string]: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function parseArchive(path: string): Promise<SavedObject[]> {
|
||||||
|
return (await Fs.readFile(path, 'utf-8'))
|
||||||
|
.split('\n\n')
|
||||||
|
.filter((line) => !!line)
|
||||||
|
.map((line) => JSON.parse(line));
|
||||||
|
}
|
||||||
|
|
||||||
|
export class KbnClientImportExport {
|
||||||
|
constructor(
|
||||||
|
public readonly log: ToolingLog,
|
||||||
|
public readonly requester: KbnClientRequester,
|
||||||
|
public readonly savedObjects: KbnClientSavedObjects,
|
||||||
|
public readonly dir?: string
|
||||||
|
) {}
|
||||||
|
|
||||||
|
private resolvePath(path: string) {
|
||||||
|
if (!Path.extname(path)) {
|
||||||
|
path = `${path}.json`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.dir && !Path.isAbsolute(path)) {
|
||||||
|
throw new Error(
|
||||||
|
'unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.dir ? Path.resolve(this.dir, path) : path;
|
||||||
|
}
|
||||||
|
|
||||||
|
async load(name: string, options?: { space?: string }) {
|
||||||
|
const src = this.resolvePath(name);
|
||||||
|
this.log.debug('resolved import for', name, 'to', src);
|
||||||
|
|
||||||
|
const objects = await parseArchive(src);
|
||||||
|
this.log.info('importing', objects.length, 'saved objects', { space: options?.space });
|
||||||
|
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append('file', objects.map((obj) => JSON.stringify(obj)).join('\n'), 'import.ndjson');
|
||||||
|
|
||||||
|
// TODO: should we clear out the existing saved objects?
|
||||||
|
const resp = await this.req<ImportApiResponse>(options?.space, {
|
||||||
|
method: 'POST',
|
||||||
|
path: '/api/saved_objects/_import',
|
||||||
|
query: {
|
||||||
|
overwrite: true,
|
||||||
|
},
|
||||||
|
body: formData,
|
||||||
|
headers: formData.getHeaders(),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (resp.data.success) {
|
||||||
|
this.log.success('import success');
|
||||||
|
} else {
|
||||||
|
throw createFailError(`failed to import all saved objects: ${inspect(resp.data)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async unload(name: string, options?: { space?: string }) {
|
||||||
|
const src = this.resolvePath(name);
|
||||||
|
this.log.debug('unloading docs from archive at', src);
|
||||||
|
|
||||||
|
const objects = await parseArchive(src);
|
||||||
|
this.log.info('deleting', objects.length, 'objects', { space: options?.space });
|
||||||
|
|
||||||
|
const { deleted, missing } = await this.savedObjects.bulkDelete({
|
||||||
|
space: options?.space,
|
||||||
|
objects,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (missing) {
|
||||||
|
this.log.info(missing, 'saved objects were already deleted');
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log.success(deleted, 'saved objects deleted');
|
||||||
|
}
|
||||||
|
|
||||||
|
async save(name: string, options: { types: string[]; space?: string }) {
|
||||||
|
const dest = this.resolvePath(name);
|
||||||
|
this.log.debug('saving export to', dest);
|
||||||
|
|
||||||
|
const resp = await this.req(options.space, {
|
||||||
|
method: 'POST',
|
||||||
|
path: '/api/saved_objects/_export',
|
||||||
|
body: {
|
||||||
|
type: options.types,
|
||||||
|
excludeExportDetails: true,
|
||||||
|
includeReferencesDeep: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (typeof resp.data !== 'string') {
|
||||||
|
throw createFailError(`unexpected response from export API: ${inspect(resp.data)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const objects = resp.data
|
||||||
|
.split('\n')
|
||||||
|
.filter((l) => !!l)
|
||||||
|
.map((line) => JSON.parse(line));
|
||||||
|
|
||||||
|
const fileContents = objects
|
||||||
|
.map((obj) => {
|
||||||
|
const { sort: _, ...nonSortFields } = obj;
|
||||||
|
return JSON.stringify(nonSortFields, null, 2);
|
||||||
|
})
|
||||||
|
.join('\n\n');
|
||||||
|
|
||||||
|
await Fs.writeFile(dest, fileContents, 'utf-8');
|
||||||
|
|
||||||
|
this.log.success('Exported', objects.length, 'saved objects to', dest);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async req<T>(space: string | undefined, options: ReqOptions) {
|
||||||
|
if (!options.path.startsWith('/')) {
|
||||||
|
throw new Error('options.path must start with a /');
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await this.requester.request<T>({
|
||||||
|
...options,
|
||||||
|
path: space ? uriencode`/s/${space}` + options.path : options.path,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (!isAxiosResponseError(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw createFailError(
|
||||||
|
`${error.response.status} resp: ${inspect(error.response.data)}\nreq: ${inspect(
|
||||||
|
error.config
|
||||||
|
)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -8,10 +8,10 @@
|
||||||
|
|
||||||
import Url from 'url';
|
import Url from 'url';
|
||||||
import Https from 'https';
|
import Https from 'https';
|
||||||
import Axios, { AxiosResponse } from 'axios';
|
import Qs from 'querystring';
|
||||||
|
|
||||||
import { isAxiosRequestError, isAxiosResponseError } from '../axios';
|
import Axios, { AxiosResponse } from 'axios';
|
||||||
import { ToolingLog } from '../tooling_log';
|
import { ToolingLog, isAxiosRequestError, isAxiosResponseError } from '@kbn/dev-utils';
|
||||||
|
|
||||||
const isConcliftOnGetError = (error: any) => {
|
const isConcliftOnGetError = (error: any) => {
|
||||||
return (
|
return (
|
||||||
|
@ -52,6 +52,7 @@ export interface ReqOptions {
|
||||||
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
|
||||||
body?: any;
|
body?: any;
|
||||||
retries?: number;
|
retries?: number;
|
||||||
|
headers?: Record<string, string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
const delay = (ms: number) =>
|
const delay = (ms: number) =>
|
||||||
|
@ -102,9 +103,11 @@ export class KbnClientRequester {
|
||||||
data: options.body,
|
data: options.body,
|
||||||
params: options.query,
|
params: options.query,
|
||||||
headers: {
|
headers: {
|
||||||
|
...options.headers,
|
||||||
'kbn-xsrf': 'kbn-client',
|
'kbn-xsrf': 'kbn-client',
|
||||||
},
|
},
|
||||||
httpsAgent: this.httpsAgent,
|
httpsAgent: this.httpsAgent,
|
||||||
|
paramsSerializer: (params) => Qs.stringify(params),
|
||||||
});
|
});
|
||||||
|
|
||||||
return response;
|
return response;
|
|
@ -6,7 +6,12 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ToolingLog } from '../tooling_log';
|
import { inspect } from 'util';
|
||||||
|
|
||||||
|
import * as Rx from 'rxjs';
|
||||||
|
import { mergeMap } from 'rxjs/operators';
|
||||||
|
import { lastValueFrom } from '@kbn/std';
|
||||||
|
import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils';
|
||||||
|
|
||||||
import { KbnClientRequester, uriencode } from './kbn_client_requester';
|
import { KbnClientRequester, uriencode } from './kbn_client_requester';
|
||||||
|
|
||||||
|
@ -51,6 +56,38 @@ interface MigrateResponse {
|
||||||
result: Array<{ status: string }>;
|
result: Array<{ status: string }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface FindApiResponse {
|
||||||
|
saved_objects: Array<{
|
||||||
|
type: string;
|
||||||
|
id: string;
|
||||||
|
[key: string]: unknown;
|
||||||
|
}>;
|
||||||
|
total: number;
|
||||||
|
per_page: number;
|
||||||
|
page: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CleanOptions {
|
||||||
|
space?: string;
|
||||||
|
types: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface DeleteObjectsOptions {
|
||||||
|
space?: string;
|
||||||
|
objects: Array<{
|
||||||
|
type: string;
|
||||||
|
id: string;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function concurrently<T>(maxConcurrency: number, arr: T[], fn: (item: T) => Promise<void>) {
|
||||||
|
if (arr.length) {
|
||||||
|
await lastValueFrom(
|
||||||
|
Rx.from(arr).pipe(mergeMap(async (item) => await fn(item), maxConcurrency))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class KbnClientSavedObjects {
|
export class KbnClientSavedObjects {
|
||||||
constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {}
|
constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {}
|
||||||
|
|
||||||
|
@ -143,4 +180,67 @@ export class KbnClientSavedObjects {
|
||||||
|
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async clean(options: CleanOptions) {
|
||||||
|
this.log.debug('Cleaning all saved objects', { space: options.space });
|
||||||
|
|
||||||
|
let deleted = 0;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const resp = await this.requester.request<FindApiResponse>({
|
||||||
|
method: 'GET',
|
||||||
|
path: options.space
|
||||||
|
? uriencode`/s/${options.space}/api/saved_objects/_find`
|
||||||
|
: '/api/saved_objects/_find',
|
||||||
|
query: {
|
||||||
|
per_page: 1000,
|
||||||
|
type: options.types,
|
||||||
|
fields: 'none',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects');
|
||||||
|
const deletion = await this.bulkDelete({
|
||||||
|
space: options.space,
|
||||||
|
objects: resp.data.saved_objects,
|
||||||
|
});
|
||||||
|
deleted += deletion.deleted;
|
||||||
|
|
||||||
|
if (resp.data.total <= resp.data.per_page) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log.success('deleted', deleted, 'objects');
|
||||||
|
}
|
||||||
|
|
||||||
|
public async bulkDelete(options: DeleteObjectsOptions) {
|
||||||
|
let deleted = 0;
|
||||||
|
let missing = 0;
|
||||||
|
|
||||||
|
await concurrently(20, options.objects, async (obj) => {
|
||||||
|
try {
|
||||||
|
await this.requester.request({
|
||||||
|
method: 'DELETE',
|
||||||
|
path: options.space
|
||||||
|
? uriencode`/s/${options.space}/api/saved_objects/${obj.type}/${obj.id}`
|
||||||
|
: uriencode`/api/saved_objects/${obj.type}/${obj.id}`,
|
||||||
|
});
|
||||||
|
deleted++;
|
||||||
|
} catch (error) {
|
||||||
|
if (isAxiosResponseError(error)) {
|
||||||
|
if (error.response.status === 404) {
|
||||||
|
missing++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return { deleted, missing };
|
||||||
|
}
|
||||||
}
|
}
|
|
@ -6,7 +6,7 @@
|
||||||
* Side Public License, v 1.
|
* Side Public License, v 1.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ToolingLog } from '../tooling_log';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
|
||||||
import { KbnClientRequester, uriencode } from './kbn_client_requester';
|
import { KbnClientRequester, uriencode } from './kbn_client_requester';
|
||||||
|
|
10
scripts/kbn_archiver.js
Normal file
10
scripts/kbn_archiver.js
Normal file
|
@ -0,0 +1,10 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License
|
||||||
|
* 2.0 and the Server Side Public License, v 1; you may not use this file except
|
||||||
|
* in compliance with, at your election, the Elastic License 2.0 or the Server
|
||||||
|
* Side Public License, v 1.
|
||||||
|
*/
|
||||||
|
|
||||||
|
require('../src/setup_node_env');
|
||||||
|
require('@kbn/test').runKbnArchiverCli();
|
|
@ -7,7 +7,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import Url from 'url';
|
import Url from 'url';
|
||||||
import { KbnClient } from '@kbn/dev-utils';
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||||
|
|
||||||
|
@ -22,6 +22,7 @@ export function KibanaServerProvider({ getService }: FtrProviderContext) {
|
||||||
url,
|
url,
|
||||||
certificateAuthorities: config.get('servers.kibana.certificateAuthorities'),
|
certificateAuthorities: config.get('servers.kibana.certificateAuthorities'),
|
||||||
uiSettingDefaults: defaults,
|
uiSettingDefaults: defaults,
|
||||||
|
importExportDir: config.get('kbnArchiver.directory'),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (defaults) {
|
if (defaults) {
|
||||||
|
|
|
@ -7,7 +7,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import util from 'util';
|
import util from 'util';
|
||||||
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
export class Role {
|
export class Role {
|
||||||
constructor(private log: ToolingLog, private kibanaServer: KbnClient) {}
|
constructor(private log: ToolingLog, private kibanaServer: KbnClient) {}
|
||||||
|
|
|
@ -7,7 +7,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import util from 'util';
|
import util from 'util';
|
||||||
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
export class RoleMappings {
|
export class RoleMappings {
|
||||||
constructor(private log: ToolingLog, private kbnClient: KbnClient) {}
|
constructor(private log: ToolingLog, private kbnClient: KbnClient) {}
|
||||||
|
|
|
@ -7,7 +7,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import util from 'util';
|
import util from 'util';
|
||||||
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
|
|
||||||
export class User {
|
export class User {
|
||||||
constructor(private log: ToolingLog, private kbnClient: KbnClient) {}
|
constructor(private log: ToolingLog, private kbnClient: KbnClient) {}
|
||||||
|
|
|
@ -20,6 +20,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||||
const inspector = getService('inspector');
|
const inspector = getService('inspector');
|
||||||
const elasticChart = getService('elasticChart');
|
const elasticChart = getService('elasticChart');
|
||||||
const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']);
|
const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']);
|
||||||
|
|
||||||
const defaultSettings = {
|
const defaultSettings = {
|
||||||
defaultIndex: 'logstash-*',
|
defaultIndex: 'logstash-*',
|
||||||
};
|
};
|
||||||
|
@ -27,7 +28,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||||
describe('discover test', function describeIndexTests() {
|
describe('discover test', function describeIndexTests() {
|
||||||
before(async function () {
|
before(async function () {
|
||||||
log.debug('load kibana index with default index pattern');
|
log.debug('load kibana index with default index pattern');
|
||||||
await esArchiver.load('discover');
|
|
||||||
|
await kibanaServer.savedObjects.clean({ types: ['search'] });
|
||||||
|
await kibanaServer.importExport.load('discover');
|
||||||
|
|
||||||
// and load a set of makelogs data
|
// and load a set of makelogs data
|
||||||
await esArchiver.loadIfNeeded('logstash_functional');
|
await esArchiver.loadIfNeeded('logstash_functional');
|
||||||
|
|
51
test/functional/fixtures/kbn_archiver/discover.json
Normal file
51
test/functional/fixtures/kbn_archiver/discover.json
Normal file
File diff suppressed because one or more lines are too long
|
@ -6,7 +6,8 @@
|
||||||
*/
|
*/
|
||||||
/* eslint-disable no-console */
|
/* eslint-disable no-console */
|
||||||
import yargs from 'yargs';
|
import yargs from 'yargs';
|
||||||
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
import {
|
import {
|
||||||
CaseResponse,
|
CaseResponse,
|
||||||
CaseType,
|
CaseType,
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
|
|
||||||
import { Client } from '@elastic/elasticsearch';
|
import { Client } from '@elastic/elasticsearch';
|
||||||
import seedrandom from 'seedrandom';
|
import seedrandom from 'seedrandom';
|
||||||
import { KbnClient } from '@kbn/dev-utils';
|
import { KbnClient } from '@kbn/test';
|
||||||
import { AxiosResponse } from 'axios';
|
import { AxiosResponse } from 'axios';
|
||||||
import { EndpointDocGenerator, TreeOptions, Event } from './generate_data';
|
import { EndpointDocGenerator, TreeOptions, Event } from './generate_data';
|
||||||
import { firstNonNullValue } from './models/ecs_safety_helpers';
|
import { firstNonNullValue } from './models/ecs_safety_helpers';
|
||||||
|
|
|
@ -7,7 +7,7 @@
|
||||||
|
|
||||||
import { URL } from 'url';
|
import { URL } from 'url';
|
||||||
|
|
||||||
import { KbnClient, KbnClientOptions } from '@kbn/dev-utils';
|
import { KbnClient, KbnClientOptions } from '@kbn/test';
|
||||||
import fetch, { RequestInit } from 'node-fetch';
|
import fetch, { RequestInit } from 'node-fetch';
|
||||||
|
|
||||||
export class KbnClientWithApiKeySupport extends KbnClient {
|
export class KbnClientWithApiKeySupport extends KbnClient {
|
||||||
|
|
|
@ -10,7 +10,8 @@ import yargs from 'yargs';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { Client, ClientOptions } from '@elastic/elasticsearch';
|
import { Client, ClientOptions } from '@elastic/elasticsearch';
|
||||||
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
|
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
|
||||||
import { KbnClient, ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils';
|
import { ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
import { AxiosResponse } from 'axios';
|
import { AxiosResponse } from 'axios';
|
||||||
import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
|
import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
|
||||||
import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data';
|
import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data';
|
||||||
|
|
|
@ -7,7 +7,8 @@
|
||||||
|
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
import minimist from 'minimist';
|
import minimist from 'minimist';
|
||||||
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
|
import { ToolingLog } from '@kbn/dev-utils';
|
||||||
|
import { KbnClient } from '@kbn/test';
|
||||||
import bluebird from 'bluebird';
|
import bluebird from 'bluebird';
|
||||||
import { basename } from 'path';
|
import { basename } from 'path';
|
||||||
import { TRUSTED_APPS_CREATE_API, TRUSTED_APPS_LIST_API } from '../../../common/endpoint/constants';
|
import { TRUSTED_APPS_CREATE_API, TRUSTED_APPS_LIST_API } from '../../../common/endpoint/constants';
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
* 2.0.
|
* 2.0.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { KbnClient } from '@kbn/dev-utils';
|
import { KbnClient } from '@kbn/test';
|
||||||
import { ApiResponse, Client } from '@elastic/elasticsearch';
|
import { ApiResponse, Client } from '@elastic/elasticsearch';
|
||||||
import { SuperTest } from 'supertest';
|
import { SuperTest } from 'supertest';
|
||||||
import supertestAsPromised from 'supertest-as-promised';
|
import supertestAsPromised from 'supertest-as-promised';
|
||||||
|
|
|
@ -14954,6 +14954,15 @@ form-data@^3.0.0:
|
||||||
combined-stream "^1.0.8"
|
combined-stream "^1.0.8"
|
||||||
mime-types "^2.1.12"
|
mime-types "^2.1.12"
|
||||||
|
|
||||||
|
form-data@^4.0.0:
|
||||||
|
version "4.0.0"
|
||||||
|
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
|
||||||
|
integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
|
||||||
|
dependencies:
|
||||||
|
asynckit "^0.4.0"
|
||||||
|
combined-stream "^1.0.8"
|
||||||
|
mime-types "^2.1.12"
|
||||||
|
|
||||||
form-data@~2.3.2:
|
form-data@~2.3.2:
|
||||||
version "2.3.3"
|
version "2.3.3"
|
||||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
|
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue