[kbn/test] add import/export support to KbnClient (#92526) (#92935)

Co-authored-by: Tre' Seymour <wayne.seymour@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: spalger <spalger@users.noreply.github.com>

Co-authored-by: Spencer <email@spalger.com>
Co-authored-by: Tre' Seymour <wayne.seymour@elastic.co>
Co-authored-by: spalger <spalger@users.noreply.github.com>
This commit is contained in:
Kibana Machine 2021-02-25 20:34:11 -05:00 committed by GitHub
parent 9cbee5980e
commit f6e6149e66
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
34 changed files with 546 additions and 25 deletions

View file

@ -652,6 +652,7 @@
"fetch-mock": "^7.3.9",
"file-loader": "^4.2.0",
"file-saver": "^1.3.8",
"form-data": "^4.0.0",
"formsy-react": "^1.1.5",
"geckodriver": "^1.21.0",
"glob-watcher": "5.0.3",

View file

@ -23,7 +23,6 @@ export {
KBN_P12_PATH,
KBN_P12_PASSWORD,
} from './certs';
export * from './kbn_client';
export * from './run';
export * from './axios';
export * from './stdio';

View file

@ -7,7 +7,8 @@
*/
import { Client } from '@elastic/elasticsearch';
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib';

View file

@ -9,7 +9,8 @@
import { resolve } from 'path';
import { createReadStream } from 'fs';
import { Readable } from 'stream';
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { Client } from '@elastic/elasticsearch';
import { createPromiseFromStreams, concatStreamProviders } from '@kbn/utils';
import { ES_CLIENT_HEADERS } from '../client_headers';

View file

@ -10,7 +10,8 @@ import { resolve } from 'path';
import { createReadStream } from 'fs';
import { Readable, Writable } from 'stream';
import { Client } from '@elastic/elasticsearch';
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { createPromiseFromStreams } from '@kbn/utils';
import {

View file

@ -17,8 +17,8 @@ import Url from 'url';
import readline from 'readline';
import Fs from 'fs';
import { RunWithCommands, createFlagError, KbnClient, CA_CERT_PATH } from '@kbn/dev-utils';
import { readConfigFile } from '@kbn/test';
import { RunWithCommands, createFlagError, CA_CERT_PATH } from '@kbn/dev-utils';
import { readConfigFile, KbnClient } from '@kbn/test';
import { Client } from '@elastic/elasticsearch';
import { EsArchiver } from './es_archiver';

View file

@ -7,7 +7,8 @@
*/
import { Client } from '@elastic/elasticsearch';
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import {
saveAction,

View file

@ -9,7 +9,8 @@
import { inspect } from 'util';
import { Client } from '@elastic/elasticsearch';
import { ToolingLog, KbnClient } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { Stats } from '../stats';
import { deleteIndex } from './delete_index';
import { ES_CLIENT_HEADERS } from '../../client_headers';

View file

@ -213,6 +213,13 @@ export const schema = Joi.object()
})
.default(),
// settings for the saved objects svc
kbnArchiver: Joi.object()
.keys({
directory: Joi.string().default(defaultRelativeToConfigPath('fixtures/kbn_archiver')),
})
.default(),
// settings for the kibanaServer.uiSettings module
uiSettings: Joi.object()
.keys({

View file

@ -48,3 +48,7 @@ export { getUrl } from './jest/utils/get_url';
export { runCheckJestConfigsCli } from './jest/run_check_jest_configs_cli';
export { runJest } from './jest/run';
export * from './kbn_archiver_cli';
export * from './kbn_client';

View file

@ -0,0 +1,149 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import Path from 'path';
import Url from 'url';
import { RunWithCommands, createFlagError, Flags } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { readConfigFile } from './functional_test_runner';
function getSinglePositionalArg(flags: Flags) {
const positional = flags._;
if (positional.length < 1) {
throw createFlagError('missing name of export to import');
}
if (positional.length > 1) {
throw createFlagError(`extra positional arguments, expected 1, got [${positional}]`);
}
return positional[0];
}
function parseTypesFlag(flags: Flags) {
if (!flags.type || (typeof flags.type !== 'string' && !Array.isArray(flags.type))) {
throw createFlagError('--type is a required flag');
}
const types = typeof flags.type === 'string' ? [flags.type] : flags.type;
return types.reduce(
(acc: string[], type) => [...acc, ...type.split(',').map((t) => t.trim())],
[]
);
}
export function runKbnArchiverCli() {
new RunWithCommands({
description: 'Import/export saved objects from archives, for testing',
globalFlags: {
string: ['config', 'space', 'kibana-url', 'dir'],
help: `
--space space id to operate on, defaults to the default space
--config optional path to an FTR config file that will be parsed and used for defaults
--kibana-url set the url that kibana can be reached at, uses the "servers.kibana" setting from --config by default
--dir directory that contains exports to be imported, or where exports will be saved, uses the "kbnArchiver.directory"
setting from --config by default
`,
},
async extendContext({ log, flags }) {
let config;
if (flags.config) {
if (typeof flags.config !== 'string') {
throw createFlagError('expected --config to be a string');
}
config = await readConfigFile(log, Path.resolve(flags.config));
}
let kibanaUrl;
if (flags['kibana-url']) {
if (typeof flags['kibana-url'] !== 'string') {
throw createFlagError('expected --kibana-url to be a string');
}
kibanaUrl = flags['kibana-url'];
} else if (config) {
kibanaUrl = Url.format(config.get('servers.kibana'));
}
if (!kibanaUrl) {
throw createFlagError(
'Either a --config file with `servers.kibana` defined, or a --kibana-url must be passed'
);
}
let importExportDir;
if (flags.dir) {
if (typeof flags.dir !== 'string') {
throw createFlagError('expected --dir to be a string');
}
importExportDir = flags.dir;
} else if (config) {
importExportDir = config.get('kbnArchiver.directory');
}
if (!importExportDir) {
throw createFlagError(
'--config does not include a kbnArchiver.directory, specify it or include --dir flag'
);
}
const space = flags.space;
if (!(space === undefined || typeof space === 'string')) {
throw createFlagError('--space must be a string');
}
return {
space,
kbnClient: new KbnClient({
log,
url: kibanaUrl,
importExportDir,
}),
};
},
})
.command({
name: 'save',
usage: 'save <name>',
description: 'export saved objects from Kibana to a file',
flags: {
string: ['type'],
help: `
--type saved object type that should be fetched and stored in the archive, can
be specified multiple times or be a comma-separated list.
`,
},
async run({ kbnClient, flags, space }) {
await kbnClient.importExport.save(getSinglePositionalArg(flags), {
types: parseTypesFlag(flags),
space,
});
},
})
.command({
name: 'load',
usage: 'load <name>',
description: 'import a saved export to Kibana',
async run({ kbnClient, flags, space }) {
await kbnClient.importExport.load(getSinglePositionalArg(flags), { space });
},
})
.command({
name: 'unload',
usage: 'unload <name>',
description: 'delete the saved objects saved in the archive from the Kibana index',
async run({ kbnClient, flags, space }) {
await kbnClient.importExport.unload(getSinglePositionalArg(flags), { space });
},
})
.execute();
}

View file

@ -6,19 +6,22 @@
* Side Public License, v 1.
*/
import { ToolingLog } from '../tooling_log';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClientRequester, ReqOptions } from './kbn_client_requester';
import { KbnClientStatus } from './kbn_client_status';
import { KbnClientPlugins } from './kbn_client_plugins';
import { KbnClientVersion } from './kbn_client_version';
import { KbnClientSavedObjects } from './kbn_client_saved_objects';
import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings';
import { KbnClientImportExport } from './kbn_client_import_export';
export interface KbnClientOptions {
url: string;
certificateAuthorities?: Buffer[];
log: ToolingLog;
uiSettingDefaults?: UiSettingValues;
importExportDir?: string;
}
export class KbnClient {
@ -27,6 +30,7 @@ export class KbnClient {
readonly version: KbnClientVersion;
readonly savedObjects: KbnClientSavedObjects;
readonly uiSettings: KbnClientUiSettings;
readonly importExport: KbnClientImportExport;
private readonly requester: KbnClientRequester;
private readonly log: ToolingLog;
@ -56,6 +60,12 @@ export class KbnClient {
this.version = new KbnClientVersion(this.status);
this.savedObjects = new KbnClientSavedObjects(this.log, this.requester);
this.uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults);
this.importExport = new KbnClientImportExport(
this.log,
this.requester,
this.savedObjects,
options.importExportDir
);
}
/**

View file

@ -0,0 +1,163 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
import { inspect } from 'util';
import Fs from 'fs/promises';
import Path from 'path';
import FormData from 'form-data';
import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils';
import { KbnClientRequester, uriencode, ReqOptions } from './kbn_client_requester';
import { KbnClientSavedObjects } from './kbn_client_saved_objects';
interface ImportApiResponse {
success: boolean;
[key: string]: unknown;
}
interface SavedObject {
id: string;
type: string;
[key: string]: unknown;
}
async function parseArchive(path: string): Promise<SavedObject[]> {
return (await Fs.readFile(path, 'utf-8'))
.split('\n\n')
.filter((line) => !!line)
.map((line) => JSON.parse(line));
}
export class KbnClientImportExport {
constructor(
public readonly log: ToolingLog,
public readonly requester: KbnClientRequester,
public readonly savedObjects: KbnClientSavedObjects,
public readonly dir?: string
) {}
private resolvePath(path: string) {
if (!Path.extname(path)) {
path = `${path}.json`;
}
if (!this.dir && !Path.isAbsolute(path)) {
throw new Error(
'unable to resolve relative path to import/export without a configured dir, either path absolute path or specify --dir'
);
}
return this.dir ? Path.resolve(this.dir, path) : path;
}
async load(name: string, options?: { space?: string }) {
const src = this.resolvePath(name);
this.log.debug('resolved import for', name, 'to', src);
const objects = await parseArchive(src);
this.log.info('importing', objects.length, 'saved objects', { space: options?.space });
const formData = new FormData();
formData.append('file', objects.map((obj) => JSON.stringify(obj)).join('\n'), 'import.ndjson');
// TODO: should we clear out the existing saved objects?
const resp = await this.req<ImportApiResponse>(options?.space, {
method: 'POST',
path: '/api/saved_objects/_import',
query: {
overwrite: true,
},
body: formData,
headers: formData.getHeaders(),
});
if (resp.data.success) {
this.log.success('import success');
} else {
throw createFailError(`failed to import all saved objects: ${inspect(resp.data)}`);
}
}
async unload(name: string, options?: { space?: string }) {
const src = this.resolvePath(name);
this.log.debug('unloading docs from archive at', src);
const objects = await parseArchive(src);
this.log.info('deleting', objects.length, 'objects', { space: options?.space });
const { deleted, missing } = await this.savedObjects.bulkDelete({
space: options?.space,
objects,
});
if (missing) {
this.log.info(missing, 'saved objects were already deleted');
}
this.log.success(deleted, 'saved objects deleted');
}
async save(name: string, options: { types: string[]; space?: string }) {
const dest = this.resolvePath(name);
this.log.debug('saving export to', dest);
const resp = await this.req(options.space, {
method: 'POST',
path: '/api/saved_objects/_export',
body: {
type: options.types,
excludeExportDetails: true,
includeReferencesDeep: true,
},
});
if (typeof resp.data !== 'string') {
throw createFailError(`unexpected response from export API: ${inspect(resp.data)}`);
}
const objects = resp.data
.split('\n')
.filter((l) => !!l)
.map((line) => JSON.parse(line));
const fileContents = objects
.map((obj) => {
const { sort: _, ...nonSortFields } = obj;
return JSON.stringify(nonSortFields, null, 2);
})
.join('\n\n');
await Fs.writeFile(dest, fileContents, 'utf-8');
this.log.success('Exported', objects.length, 'saved objects to', dest);
}
private async req<T>(space: string | undefined, options: ReqOptions) {
if (!options.path.startsWith('/')) {
throw new Error('options.path must start with a /');
}
try {
return await this.requester.request<T>({
...options,
path: space ? uriencode`/s/${space}` + options.path : options.path,
});
} catch (error) {
if (!isAxiosResponseError(error)) {
throw error;
}
throw createFailError(
`${error.response.status} resp: ${inspect(error.response.data)}\nreq: ${inspect(
error.config
)}`
);
}
}
}

View file

@ -8,10 +8,10 @@
import Url from 'url';
import Https from 'https';
import Axios, { AxiosResponse } from 'axios';
import Qs from 'querystring';
import { isAxiosRequestError, isAxiosResponseError } from '../axios';
import { ToolingLog } from '../tooling_log';
import Axios, { AxiosResponse } from 'axios';
import { ToolingLog, isAxiosRequestError, isAxiosResponseError } from '@kbn/dev-utils';
const isConcliftOnGetError = (error: any) => {
return (
@ -52,6 +52,7 @@ export interface ReqOptions {
method: 'GET' | 'POST' | 'PUT' | 'DELETE';
body?: any;
retries?: number;
headers?: Record<string, string>;
}
const delay = (ms: number) =>
@ -102,9 +103,11 @@ export class KbnClientRequester {
data: options.body,
params: options.query,
headers: {
...options.headers,
'kbn-xsrf': 'kbn-client',
},
httpsAgent: this.httpsAgent,
paramsSerializer: (params) => Qs.stringify(params),
});
return response;

View file

@ -6,7 +6,12 @@
* Side Public License, v 1.
*/
import { ToolingLog } from '../tooling_log';
import { inspect } from 'util';
import * as Rx from 'rxjs';
import { mergeMap } from 'rxjs/operators';
import { lastValueFrom } from '@kbn/std';
import { ToolingLog, isAxiosResponseError, createFailError } from '@kbn/dev-utils';
import { KbnClientRequester, uriencode } from './kbn_client_requester';
@ -51,6 +56,38 @@ interface MigrateResponse {
result: Array<{ status: string }>;
}
interface FindApiResponse {
saved_objects: Array<{
type: string;
id: string;
[key: string]: unknown;
}>;
total: number;
per_page: number;
page: number;
}
interface CleanOptions {
space?: string;
types: string[];
}
interface DeleteObjectsOptions {
space?: string;
objects: Array<{
type: string;
id: string;
}>;
}
async function concurrently<T>(maxConcurrency: number, arr: T[], fn: (item: T) => Promise<void>) {
if (arr.length) {
await lastValueFrom(
Rx.from(arr).pipe(mergeMap(async (item) => await fn(item), maxConcurrency))
);
}
}
export class KbnClientSavedObjects {
constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {}
@ -143,4 +180,67 @@ export class KbnClientSavedObjects {
return data;
}
public async clean(options: CleanOptions) {
this.log.debug('Cleaning all saved objects', { space: options.space });
let deleted = 0;
while (true) {
const resp = await this.requester.request<FindApiResponse>({
method: 'GET',
path: options.space
? uriencode`/s/${options.space}/api/saved_objects/_find`
: '/api/saved_objects/_find',
query: {
per_page: 1000,
type: options.types,
fields: 'none',
},
});
this.log.info('deleting batch of', resp.data.saved_objects.length, 'objects');
const deletion = await this.bulkDelete({
space: options.space,
objects: resp.data.saved_objects,
});
deleted += deletion.deleted;
if (resp.data.total <= resp.data.per_page) {
break;
}
}
this.log.success('deleted', deleted, 'objects');
}
public async bulkDelete(options: DeleteObjectsOptions) {
let deleted = 0;
let missing = 0;
await concurrently(20, options.objects, async (obj) => {
try {
await this.requester.request({
method: 'DELETE',
path: options.space
? uriencode`/s/${options.space}/api/saved_objects/${obj.type}/${obj.id}`
: uriencode`/api/saved_objects/${obj.type}/${obj.id}`,
});
deleted++;
} catch (error) {
if (isAxiosResponseError(error)) {
if (error.response.status === 404) {
missing++;
return;
}
throw createFailError(`${error.response.status} resp: ${inspect(error.response.data)}`);
}
throw error;
}
});
return { deleted, missing };
}
}

View file

@ -6,7 +6,7 @@
* Side Public License, v 1.
*/
import { ToolingLog } from '../tooling_log';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClientRequester, uriencode } from './kbn_client_requester';

10
scripts/kbn_archiver.js Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/
require('../src/setup_node_env');
require('@kbn/test').runKbnArchiverCli();

View file

@ -7,7 +7,7 @@
*/
import Url from 'url';
import { KbnClient } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { FtrProviderContext } from '../../ftr_provider_context';
@ -22,6 +22,7 @@ export function KibanaServerProvider({ getService }: FtrProviderContext) {
url,
certificateAuthorities: config.get('servers.kibana.certificateAuthorities'),
uiSettingDefaults: defaults,
importExportDir: config.get('kbnArchiver.directory'),
});
if (defaults) {

View file

@ -7,7 +7,8 @@
*/
import util from 'util';
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
export class Role {
constructor(private log: ToolingLog, private kibanaServer: KbnClient) {}

View file

@ -7,7 +7,8 @@
*/
import util from 'util';
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
export class RoleMappings {
constructor(private log: ToolingLog, private kbnClient: KbnClient) {}

View file

@ -7,7 +7,8 @@
*/
import util from 'util';
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
export class User {
constructor(private log: ToolingLog, private kbnClient: KbnClient) {}

View file

@ -20,6 +20,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const inspector = getService('inspector');
const elasticChart = getService('elasticChart');
const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']);
const defaultSettings = {
defaultIndex: 'logstash-*',
};
@ -27,7 +28,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
describe('discover test', function describeIndexTests() {
before(async function () {
log.debug('load kibana index with default index pattern');
await esArchiver.load('discover');
await kibanaServer.savedObjects.clean({ types: ['search'] });
await kibanaServer.importExport.load('discover');
// and load a set of makelogs data
await esArchiver.loadIfNeeded('logstash_functional');

File diff suppressed because one or more lines are too long

View file

@ -6,7 +6,8 @@
*/
/* eslint-disable no-console */
import yargs from 'yargs';
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import {
CaseResponse,
CaseType,

View file

@ -7,7 +7,7 @@
import { Client } from '@elastic/elasticsearch';
import seedrandom from 'seedrandom';
import { KbnClient } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { AxiosResponse } from 'axios';
import { EndpointDocGenerator, TreeOptions, Event } from './generate_data';
import { firstNonNullValue } from './models/ecs_safety_helpers';

View file

@ -7,7 +7,7 @@
import { URL } from 'url';
import { KbnClient, KbnClientOptions } from '@kbn/dev-utils';
import { KbnClient, KbnClientOptions } from '@kbn/test';
import fetch, { RequestInit } from 'node-fetch';
export class KbnClientWithApiKeySupport extends KbnClient {

View file

@ -10,7 +10,8 @@ import yargs from 'yargs';
import fs from 'fs';
import { Client, ClientOptions } from '@elastic/elasticsearch';
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
import { KbnClient, ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils';
import { ToolingLog, CA_CERT_PATH } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { AxiosResponse } from 'axios';
import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
import { ANCESTRY_LIMIT, EndpointDocGenerator } from '../../common/endpoint/generate_data';

View file

@ -7,7 +7,8 @@
// @ts-ignore
import minimist from 'minimist';
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { ToolingLog } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import bluebird from 'bluebird';
import { basename } from 'path';
import { TRUSTED_APPS_CREATE_API, TRUSTED_APPS_LIST_API } from '../../../common/endpoint/constants';

View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { KbnClient } from '@kbn/dev-utils';
import { KbnClient } from '@kbn/test';
import { ApiResponse, Client } from '@elastic/elasticsearch';
import { SuperTest } from 'supertest';
import supertestAsPromised from 'supertest-as-promised';

View file

@ -14954,6 +14954,15 @@ form-data@^3.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@~2.3.2:
version "2.3.3"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"