[7.x] [dev-utils] implement basic KbnClient util for talking t… (#47003)

* [dev-utils] implement basic KbnClient util for talking to Kibana server

* update KbnClient to expose full KibanaServerService API

* expose request() function and uriencode helper

* [uiSettings] retry read on conflicts auto upgrading

* expose function for resolving a Kibana server url

* only use apis in test hooks

* run x-pack-ciGroup2 60 times

* log retries as errors so they are included in console output for job

* bump

* Revert "run x-pack-ciGroup2 60 times"

This reverts commit 6b6f392edf.

* refactor urlencode tag to be a little clearer

* support customizing maxAttempts in request method

packages/kbn-dev-utils/src/index.ts

@@ -23,3 +23,4 @@ export { createAbsolutePathSerializer } from './serializers';
 export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';
 export { run, createFailError, createFlagError, combineErrors, isFailError, Flags } from './run';
 export { REPO_ROOT } from './constants';
+export { KbnClient } from './kbn_client';

packages/kbn-dev-utils/src/kbn_client/errors.ts

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { AxiosError, AxiosResponse } from 'axios';
+
+export interface AxiosRequestError extends AxiosError {
+  response: undefined;
+}
+
+export interface AxiosResponseError<T> extends AxiosError {
+  response: AxiosResponse<T>;
+}
+
+export const isAxiosRequestError = (error: any): error is AxiosRequestError => {
+  return error && error.code === undefined && error.response === undefined;
+};
+
+export const isAxiosResponseError = (error: any): error is AxiosResponseError<any> => {
+  return error && error.code !== undefined && error.response !== undefined;
+};
+
+export const isConcliftOnGetError = (error: any) => {
+  return (
+    isAxiosResponseError(error) && error.config.method === 'GET' && error.response.status === 409
+  );
+};

packages/kbn-dev-utils/src/kbn_client/index.ts

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { KbnClient } from './kbn_client';
+export { uriencode } from './kbn_client_requester';

packages/kbn-dev-utils/src/kbn_client/kbn_client.ts

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ToolingLog } from '../tooling_log';
+import { KbnClientRequester, ReqOptions } from './kbn_client_requester';
+import { KbnClientStatus } from './kbn_client_status';
+import { KbnClientPlugins } from './kbn_client_plugins';
+import { KbnClientVersion } from './kbn_client_version';
+import { KbnClientSavedObjects } from './kbn_client_saved_objects';
+import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings';
+
+export class KbnClient {
+  private readonly requester = new KbnClientRequester(this.log, this.kibanaUrls);
+  readonly status = new KbnClientStatus(this.requester);
+  readonly plugins = new KbnClientPlugins(this.status);
+  readonly version = new KbnClientVersion(this.status);
+  readonly savedObjects = new KbnClientSavedObjects(this.log, this.requester);
+  readonly uiSettings = new KbnClientUiSettings(this.log, this.requester, this.uiSettingDefaults);
+
+  /**
+   * Basic Kibana server client that implements common behaviors for talking
+   * to the Kibana server from dev tooling.
+   *
+   * @param log ToolingLog
+   * @param kibanaUrls Array of kibana server urls to send requests to
+   * @param uiSettingDefaults Map of uiSetting values that will be merged with all uiSetting resets
+   */
+  constructor(
+    private readonly log: ToolingLog,
+    private readonly kibanaUrls: string[],
+    private readonly uiSettingDefaults?: UiSettingValues
+  ) {
+    if (!kibanaUrls.length) {
+      throw new Error('missing Kibana urls');
+    }
+  }
+
+  /**
+   * Make a direct request to the Kibana server
+   */
+  async request(options: ReqOptions) {
+    return await this.requester.request(options);
+  }
+
+  resolveUrl(relativeUrl: string) {
+    return this.requester.resolveUrl(relativeUrl);
+  }
+}

packages/kbn-dev-utils/src/kbn_client/kbn_client_plugins.ts

@@ -17,23 +17,28 @@
  * under the License.
  */
 
-export class KibanaServerVersion {
-  constructor(kibanaStatus) {
-    this.kibanaStatus = kibanaStatus;
-    this._cachedVersionNumber;
-  }
+import { KbnClientStatus } from './kbn_client_status';
 
-  async get() {
-    if (this._cachedVersionNumber) {
-      return this._cachedVersionNumber;
+const PLUGIN_STATUS_ID = /^plugin:(.+?)@/;
+
+export class KbnClientPlugins {
+  constructor(private readonly status: KbnClientStatus) {}
+  /**
+   * Get a list of plugin ids that are enabled on the server
+   */
+  public async getEnabledIds() {
+    const pluginIds: string[] = [];
+    const apiResp = await this.status.get();
+
+    for (const status of apiResp.status.statuses) {
+      if (status.id) {
+        const match = status.id.match(PLUGIN_STATUS_ID);
+        if (match) {
+          pluginIds.push(match[1]);
+        }
+      }
     }
 
-    const status = await this.kibanaStatus.get();
-    if (status && status.version && status.version.number) {
-      this._cachedVersionNumber = status.version.number + (status.version.build_snapshot ? '-SNAPSHOT' : '');
-      return this._cachedVersionNumber;
-    }
-
-    throw new Error(`Unable to fetch Kibana Server status, received ${JSON.stringify(status)}`);
+    return pluginIds;
   }
 }

packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts

@@ -0,0 +1,124 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Url from 'url';
+
+import Axios from 'axios';
+
+import { isAxiosRequestError, isConcliftOnGetError } from './errors';
+import { ToolingLog } from '../tooling_log';
+
+export const uriencode = (
+  strings: TemplateStringsArray,
+  ...values: Array<string | number | boolean>
+) => {
+  const queue = strings.slice();
+
+  if (queue.length === 0) {
+    throw new Error('how could strings passed to `uriencode` template tag be empty?');
+  }
+
+  if (queue.length !== values.length + 1) {
+    throw new Error('strings and values passed to `uriencode` template tag are unbalanced');
+  }
+
+  // pull the first string off the queue, there is one less item in `values`
+  // since the values are always wrapped in strings, so we shift the extra string
+  // off the queue to balance the queue and values array.
+  const leadingString = queue.shift()!;
+  return queue.reduce(
+    (acc, string, i) => `${acc}${encodeURIComponent(values[i])}${string}`,
+    leadingString
+  );
+};
+
+const DEFAULT_MAX_ATTEMPTS = 5;
+
+export interface ReqOptions {
+  description?: string;
+  path: string;
+  query?: Record<string, any>;
+  method: 'GET' | 'POST' | 'PUT' | 'DELETE';
+  body?: any;
+  attempt?: number;
+  maxAttempts?: number;
+}
+
+const delay = (ms: number) =>
+  new Promise(resolve => {
+    setTimeout(resolve, ms);
+  });
+
+export class KbnClientRequester {
+  constructor(private readonly log: ToolingLog, private readonly kibanaUrls: string[]) {}
+
+  private pickUrl() {
+    const url = this.kibanaUrls.shift()!;
+    this.kibanaUrls.push(url);
+    return url;
+  }
+
+  public resolveUrl(relativeUrl: string = '/') {
+    return Url.resolve(this.pickUrl(), relativeUrl);
+  }
+
+  async request<T>(options: ReqOptions): Promise<T> {
+    const url = Url.resolve(this.pickUrl(), options.path);
+    const description = options.description || `${options.method} ${url}`;
+    const attempt = options.attempt === undefined ? 1 : options.attempt;
+    const maxAttempts =
+      options.maxAttempts === undefined ? DEFAULT_MAX_ATTEMPTS : options.maxAttempts;
+
+    try {
+      const response = await Axios.request<T>({
+        method: options.method,
+        url,
+        data: options.body,
+        params: options.query,
+        headers: {
+          'kbn-xsrf': 'kbn-client',
+        },
+      });
+
+      return response.data;
+    } catch (error) {
+      let retryErrorMsg: string | undefined;
+      if (isAxiosRequestError(error)) {
+        retryErrorMsg = `[${description}] request failed (attempt=${attempt})`;
+      } else if (isConcliftOnGetError(error)) {
+        retryErrorMsg = `Conflict on GET (path=${options.path}, attempt=${attempt})`;
+      }
+
+      if (retryErrorMsg) {
+        if (attempt < maxAttempts) {
+          this.log.error(retryErrorMsg);
+          await delay(1000 * attempt);
+          return await this.request<T>({
+            ...options,
+            attempt: attempt + 1,
+          });
+        }
+
+        throw new Error(retryErrorMsg + ' and ran out of retries');
+      }
+
+      throw error;
+    }
+  }
+}

packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts

@@ -17,16 +17,9 @@
  * under the License.
  */
 
-import Url from 'url';
+import { ToolingLog } from '../tooling_log';
 
-import Axios, { AxiosRequestConfig } from 'axios';
-import { ToolingLog } from '@kbn/dev-utils';
-
-const joinPath = (...components: Array<string | undefined>) =>
-  `/${components
-    .filter((s): s is string => !!s)
-    .map(c => encodeURIComponent(c))
-    .join('/')}`;
+import { KbnClientRequester, uriencode } from './kbn_client_requester';
 
 type MigrationVersion = Record<string, string>;
 
@@ -64,15 +57,8 @@ interface UpdateOptions<Attributes> extends IndexOptions<Attributes> {
   id: string;
 }
 
-export class KibanaServerSavedObjects {
-  private readonly x = Axios.create({
-    baseURL: Url.resolve(this.url, '/api/saved_objects/'),
-    headers: {
-      'kbn-xsrf': 'KibanaServerSavedObjects',
-    },
-  });
-
-  constructor(private readonly url: string, private readonly log: ToolingLog) {}
+export class KbnClientSavedObjects {
+  constructor(private readonly log: ToolingLog, private readonly requester: KbnClientRequester) {}
 
   /**
    * Get an object
@@ -80,8 +66,9 @@ export class KibanaServerSavedObjects {
   public async get<Attributes extends Record<string, any>>(options: GetOptions) {
     this.log.debug('Gettings saved object: %j', options);
 
-    return await this.request<SavedObjectResponse<Attributes>>('get saved object', {
-      url: joinPath(options.type, options.id),
+    return await this.requester.request<SavedObjectResponse<Attributes>>({
+      description: 'get saved object',
+      path: uriencode`/api/saved_objects/${options.type}/${options.id}`,
       method: 'GET',
     });
   }
@@ -92,13 +79,16 @@ export class KibanaServerSavedObjects {
   public async create<Attributes extends Record<string, any>>(options: IndexOptions<Attributes>) {
     this.log.debug('Creating saved object: %j', options);
 
-    return await this.request<SavedObjectResponse<Attributes>>('update saved object', {
-      url: joinPath(options.type, options.id),
-      params: {
+    return await this.requester.request<SavedObjectResponse<Attributes>>({
+      description: 'update saved object',
+      path: options.id
+        ? uriencode`/api/saved_objects/${options.type}/${options.id}`
+        : uriencode`/api/saved_objects/${options.type}`,
+      query: {
         overwrite: options.overwrite,
       },
       method: 'POST',
-      data: {
+      body: {
         attributes: options.attributes,
         migrationVersion: options.migrationVersion,
         references: options.references,
@@ -112,13 +102,14 @@ export class KibanaServerSavedObjects {
   public async update<Attributes extends Record<string, any>>(options: UpdateOptions<Attributes>) {
     this.log.debug('Updating saved object: %j', options);
 
-    return await this.request<SavedObjectResponse<Attributes>>('update saved object', {
-      url: joinPath(options.type, options.id),
-      params: {
+    return await this.requester.request<SavedObjectResponse<Attributes>>({
+      description: 'update saved object',
+      path: uriencode`/api/saved_objects/${options.type}/${options.id}`,
+      query: {
         overwrite: options.overwrite,
       },
       method: 'PUT',
-      data: {
+      body: {
         attributes: options.attributes,
         migrationVersion: options.migrationVersion,
         references: options.references,
@@ -132,22 +123,10 @@ export class KibanaServerSavedObjects {
   public async delete(options: GetOptions) {
     this.log.debug('Deleting saved object %s/%s', options);
 
-    return await this.request('delete saved object', {
-      url: joinPath(options.type, options.id),
+    return await this.requester.request({
+      description: 'delete saved object',
+      path: uriencode`/api/saved_objects/${options.type}/${options.id}`,
       method: 'DELETE',
     });
   }
-
-  private async request<T>(desc: string, options: AxiosRequestConfig) {
-    try {
-      const resp = await this.x.request<T>(options);
-      return resp.data;
-    } catch (error) {
-      if (error.response) {
-        throw new Error(`Failed to ${desc}:\n${JSON.stringify(error.response.data, null, 2)}`);
-      }
-
-      throw error;
-    }
-  }
 }

packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts

@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { KbnClientRequester } from './kbn_client_requester';
+
+interface Status {
+  state: 'green' | 'red' | 'yellow';
+  title?: string;
+  id?: string;
+  icon: string;
+  message: string;
+  uiColor: string;
+  since: string;
+}
+
+interface ApiResponseStatus {
+  name: string;
+  uuid: string;
+  version: {
+    number: string;
+    build_hash: string;
+    build_number: number;
+    build_snapshot: boolean;
+  };
+  status: {
+    overall: Status;
+    statuses: Status[];
+  };
+  metrics: unknown;
+}
+
+export class KbnClientStatus {
+  constructor(private readonly requester: KbnClientRequester) {}
+
+  /**
+   * Get the full server status
+   */
+  async get() {
+    return await this.requester.request<ApiResponseStatus>({
+      method: 'GET',
+      path: 'api/status',
+    });
+  }
+
+  /**
+   * Get the overall/merged state
+   */
+  public async getOverallState() {
+    const status = await this.get();
+    return status.status.overall.state;
+  }
+}

packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts

@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ToolingLog } from '../tooling_log';
+
+import { KbnClientRequester, uriencode } from './kbn_client_requester';
+
+export type UiSettingValues = Record<string, string | number | boolean>;
+interface UiSettingsApiResponse {
+  settings: {
+    [key: string]: {
+      userValue: string | number | boolean;
+      isOverridden: boolean | undefined;
+    };
+  };
+}
+
+export class KbnClientUiSettings {
+  constructor(
+    private readonly log: ToolingLog,
+    private readonly requester: KbnClientRequester,
+    private readonly defaults?: UiSettingValues
+  ) {}
+
+  async get(setting: string) {
+    const all = await this.getAll();
+    const value = all.settings[setting] ? all.settings[setting].userValue : undefined;
+
+    this.log.verbose('uiSettings.value: %j', value);
+    return value;
+  }
+
+  /**
+   * Gets defaultIndex from the config doc.
+   */
+  async getDefaultIndex() {
+    return await this.get('defaultIndex');
+  }
+
+  /**
+   * Unset a uiSetting
+   */
+  async unset(setting: string) {
+    return await this.requester.request<any>({
+      path: uriencode`/api/kibana/settings/${setting}`,
+      method: 'DELETE',
+    });
+  }
+
+  /**
+   * Replace all uiSettings with the `doc` values, `doc` is merged
+   * with some defaults
+   */
+  async replace(doc: UiSettingValues) {
+    const all = await this.getAll();
+    for (const [name, { isOverridden }] of Object.entries(all.settings)) {
+      if (!isOverridden) {
+        await this.unset(name);
+      }
+    }
+
+    this.log.debug('replacing kibana config doc: %j', doc);
+
+    await this.requester.request({
+      method: 'POST',
+      path: '/api/kibana/settings',
+      body: {
+        changes: {
+          ...this.defaults,
+          ...doc,
+        },
+      },
+    });
+  }
+
+  /**
+   * Add fields to the config doc (like setting timezone and defaultIndex)
+   */
+  async update(updates: UiSettingValues) {
+    this.log.debug('applying update to kibana config: %j', updates);
+
+    await this.requester.request({
+      path: '/api/kibana/settings',
+      method: 'POST',
+      body: {
+        changes: updates,
+      },
+    });
+  }
+
+  private async getAll() {
+    return await this.requester.request<UiSettingsApiResponse>({
+      path: '/api/kibana/settings',
+      method: 'GET',
+    });
+  }
+}

packages/kbn-dev-utils/src/kbn_client/kbn_client_version.ts

@@ -17,26 +17,20 @@
  * under the License.
  */
 
-import { resolve as resolveUrl } from 'url';
+import { KbnClientStatus } from './kbn_client_status';
 
-import Wreck from '@hapi/wreck';
+export class KbnClientVersion {
+  private versionCache: string | undefined;
 
-const get = async url => {
-  const { payload } = await Wreck.get(url, { json: 'force' });
-  return payload;
-};
-
-export class KibanaServerStatus {
-  constructor(kibanaServerUrl) {
-    this.kibanaServerUrl = kibanaServerUrl;
-  }
+  constructor(private readonly status: KbnClientStatus) {}
 
   async get() {
-    return await get(resolveUrl(this.kibanaServerUrl, './api/status'));
-  }
+    if (this.versionCache !== undefined) {
+      return this.versionCache;
+    }
 
-  async getOverallState() {
-    const status = await this.get();
-    return status.status.overall.state;
+    const status = await this.status.get();
+    this.versionCache = status.version.number + (status.version.build_snapshot ? '-SNAPSHOT' : '');
+    return this.versionCache;
   }
 }

src/es_archiver/actions/empty_kibana_index.js

@@ -20,12 +20,11 @@ import {
   migrateKibanaIndex,
   deleteKibanaIndices,
   createStats,
-  getEnabledKibanaPluginIds
 } from '../lib';
 
-export async function emptyKibanaIndexAction({ client, log, kibanaUrl }) {
+export async function emptyKibanaIndexAction({ client, log, kbnClient }) {
   const stats = createStats('emptyKibanaIndex', log);
-  const kibanaPluginIds = await getEnabledKibanaPluginIds(kibanaUrl);
+  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
   await deleteKibanaIndices({ client, stats });
   await migrateKibanaIndex({ client, log, stats, kibanaPluginIds });

src/es_archiver/actions/load.js

@@ -35,7 +35,6 @@ import {
   createIndexDocRecordsStream,
   migrateKibanaIndex,
   Progress,
-  getEnabledKibanaPluginIds,
   createDefaultSpace,
 } from '../lib';
 
@@ -49,11 +48,11 @@ const pipeline = (...streams) => streams
       .pipe(dest)
   ));
 
-export async function loadAction({ name, skipExisting, client, dataDir, log, kibanaUrl }) {
+export async function loadAction({ name, skipExisting, client, dataDir, log, kbnClient }) {
   const inputDir = resolve(dataDir, name);
   const stats = createStats(name, log);
   const files = prioritizeMappings(await readDirectory(inputDir));
-  const kibanaPluginIds = await getEnabledKibanaPluginIds(kibanaUrl);
+  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
   // a single stream that emits records from all archive files, in
   // order, so that createIndexStream can track the state of indexes

src/es_archiver/actions/unload.js

@@ -32,13 +32,12 @@ import {
   createParseArchiveStreams,
   createFilterRecordsStream,
   createDeleteIndexStream,
-  getEnabledKibanaPluginIds,
 } from '../lib';
 
-export async function unloadAction({ name, client, dataDir, log, kibanaUrl }) {
+export async function unloadAction({ name, client, dataDir, log, kbnClient }) {
   const inputDir = resolve(dataDir, name);
   const stats = createStats(name, log);
-  const kibanaPluginIds = await getEnabledKibanaPluginIds(kibanaUrl);
+  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
   const files = prioritizeMappings(await readDirectory(inputDir));
   for (const filename of files) {

src/es_archiver/es_archiver.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import { KbnClient } from '@kbn/dev-utils';
+
 import {
   saveAction,
   loadAction,
@@ -31,7 +33,7 @@ export class EsArchiver {
     this.client = client;
     this.dataDir = dataDir;
     this.log = log;
-    this.kibanaUrl = kibanaUrl;
+    this.kbnClient = new KbnClient(log, [kibanaUrl]);
   }
 
   /**
@@ -73,7 +75,7 @@ export class EsArchiver {
       client: this.client,
       dataDir: this.dataDir,
       log: this.log,
-      kibanaUrl: this.kibanaUrl,
+      kbnClient: this.kbnClient,
     });
   }
 
@@ -89,7 +91,7 @@ export class EsArchiver {
       client: this.client,
       dataDir: this.dataDir,
       log: this.log,
-      kibanaUrl: this.kibanaUrl,
+      kbnClient: this.kbnClient,
     });
   }
 
@@ -144,7 +146,7 @@ export class EsArchiver {
     await emptyKibanaIndexAction({
       client: this.client,
       log: this.log,
-      kibanaUrl: this.kibanaUrl,
+      kbnClient: this.kbnClient,
     });
   }
 }

src/es_archiver/lib/index.js

@@ -53,7 +53,3 @@ export {
 export {
   Progress
 } from './progress';
-
-export {
-  getEnabledKibanaPluginIds,
-} from './kibana_plugins';

src/es_archiver/lib/kibana_plugins.ts

@@ -1,55 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import Axios from 'axios';
-
-const PLUGIN_STATUS_ID = /^plugin:(.+?)@/;
-const isString = (v: any): v is string => typeof v === 'string';
-
-/**
- * Get the list of enabled plugins from Kibana, used to determine which
- * uiExports to collect, whether we should clean or clean the kibana index,
- * and if we need to inject the default space document in new versions of
- * the index.
- *
- * This must be called before touching the Kibana index as Kibana becomes
- * unstable when the .kibana index is deleted/cleaned and the status API
- * will fail in situations where status.allowAnonymous=false and security
- * is enabled.
- */
-export async function getEnabledKibanaPluginIds(kibanaUrl: string): Promise<string[]> {
-  try {
-    const { data } = await Axios.get('/api/status', {
-      baseURL: kibanaUrl,
-    });
-
-    return (data.status.statuses as Array<{ id: string }>)
-      .map(({ id }) => {
-        const match = id.match(PLUGIN_STATUS_ID);
-        if (match) {
-          return match[1];
-        }
-      })
-      .filter(isString);
-  } catch (error) {
-    throw new Error(
-      `Unable to fetch Kibana status API response from Kibana at ${kibanaUrl}: ${error}`
-    );
-  }
-}

src/legacy/ui/ui_settings/ui_settings_service.js

@@ -174,10 +174,11 @@ export class UiSettingsService {
     } = options;
 
     const {
+      isConflictError,
       isNotFoundError,
       isForbiddenError,
       isEsUnavailableError,
-      isNotAuthorizedError
+      isNotAuthorizedError,
     } = this._savedObjectsClient.errors;
 
     const isIgnorableError = error => (
@@ -196,7 +197,14 @@ export class UiSettingsService {
           version: this._id,
           buildNum: this._buildNum,
           logWithMetadata: this._logWithMetadata,
-          onWriteError(error, attributes) {
+          async onWriteError(error, attributes) {
+            if (isConflictError(error)) {
+              // trigger `!failedUpgradeAttributes` check below, since another
+              // request caused the uiSettings object to be created so we can
+              // just re-read
+              return false;
+            }
+
             if (isNotAuthorizedError(error) || isForbiddenError(error)) {
               return attributes;
             }

test/common/services/kibana_server/kibana_server.ts

@@ -18,36 +18,24 @@
  */
 
 import Url from 'url';
+import { KbnClient } from '@kbn/dev-utils';
 
 import { FtrProviderContext } from '../../ftr_provider_context';
-// @ts-ignore not ts yet
-import { KibanaServerStatus } from './status';
-// @ts-ignore not ts yet
-import { KibanaServerUiSettings } from './ui_settings';
-// @ts-ignore not ts yet
-import { KibanaServerVersion } from './version';
-import { KibanaServerSavedObjects } from './saved_objects';
 
 export function KibanaServerProvider({ getService }: FtrProviderContext) {
   const log = getService('log');
   const config = getService('config');
   const lifecycle = getService('lifecycle');
-
   const url = Url.format(config.get('servers.kibana'));
+  const defaults = config.get('uiSettings.defaults');
 
-  return new (class KibanaServer {
-    public readonly status = new KibanaServerStatus(url);
-    public readonly version = new KibanaServerVersion(this.status);
-    public readonly savedObjects = new KibanaServerSavedObjects(url, log);
-    public readonly uiSettings = new KibanaServerUiSettings(
-      url,
-      log,
-      config.get('uiSettings.defaults'),
-      lifecycle
-    );
+  const kbn = new KbnClient(log, [url], defaults);
 
-    public resolveUrl(path = '/') {
-      return Url.resolve(url, path);
-    }
-  })();
+  if (defaults) {
+    lifecycle.on('beforeTests', async () => {
+      await kbn.uiSettings.update(defaults);
+    });
+  }
+
+  return kbn;
 }

test/common/services/kibana_server/ui_settings.js

@@ -1,84 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import Wreck from '@hapi/wreck';
-import { get } from 'lodash';
-
-export class KibanaServerUiSettings {
-  constructor(url, log, defaults, lifecycle) {
-    this._log = log;
-    this._defaults = defaults;
-    this._wreck = Wreck.defaults({
-      headers: { 'kbn-xsrf': 'ftr/services/uiSettings' },
-      baseUrl: url,
-      json: true,
-      redirects: 3,
-    });
-
-    if (this._defaults) {
-      lifecycle.on('beforeTests', async () => {
-        await this.update(defaults);
-      });
-    }
-  }
-
-  /**
-   * Gets defaultIndex from the config doc.
-   */
-  async getDefaultIndex() {
-    const { payload } = await this._wreck.get('/api/kibana/settings');
-    const defaultIndex = get(payload, 'settings.defaultIndex.userValue');
-    this._log.verbose('uiSettings.defaultIndex: %j', defaultIndex);
-    return defaultIndex;
-  }
-
-  async replace(doc) {
-    const { payload } = await this._wreck.get('/api/kibana/settings');
-
-    for (const key of Object.keys(payload.settings)) {
-      if (!payload.settings[key].isOverridden) {
-        await this._wreck.delete(`/api/kibana/settings/${key}`);
-      }
-    }
-
-    this._log.debug('replacing kibana config doc: %j', doc);
-
-    await this._wreck.post('/api/kibana/settings', {
-      payload: {
-        changes: {
-          ...this._defaults,
-          ...doc,
-        },
-      },
-    });
-  }
-
-  /**
-   * Add fields to the config doc (like setting timezone and defaultIndex)
-   * @return {Promise} A promise that is resolved when elasticsearch has a response
-   */
-  async update(updates) {
-    this._log.debug('applying update to kibana config: %j', updates);
-    await this._wreck.post('/api/kibana/settings', {
-      payload: {
-        changes: updates,
-      },
-    });
-  }
-}

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/pagerduty.ts

@@ -17,6 +17,7 @@ import {
 export default function pagerdutyTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
 
   describe('pagerduty action', () => {
     let simulatedActionId = '';
@@ -24,11 +25,9 @@ export default function pagerdutyTest({ getService }: FtrProviderContext) {
 
     // need to wait for kibanaServer to settle ...
     before(() => {
-      const kibanaServer = getService('kibanaServer');
-      const kibanaUrl = kibanaServer.status && kibanaServer.status.kibanaServerUrl;
-      pagerdutySimulatorURL = `${kibanaUrl}${getExternalServiceSimulatorPath(
-        ExternalServiceSimulator.PAGERDUTY
-      )}`;
+      pagerdutySimulatorURL = kibanaServer.resolveUrl(
+        getExternalServiceSimulatorPath(ExternalServiceSimulator.PAGERDUTY)
+      );
     });
 
     after(() => esArchiver.unload('empty_kibana'));

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/slack.ts

@@ -17,6 +17,7 @@ import {
 export default function slackTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
 
   describe('slack action', () => {
     let simulatedActionId = '';
@@ -24,11 +25,9 @@ export default function slackTest({ getService }: FtrProviderContext) {
 
     // need to wait for kibanaServer to settle ...
     before(() => {
-      const kibanaServer = getService('kibanaServer');
-      const kibanaUrl = kibanaServer.status && kibanaServer.status.kibanaServerUrl;
-      slackSimulatorURL = `${kibanaUrl}${getExternalServiceSimulatorPath(
-        ExternalServiceSimulator.SLACK
-      )}`;
+      slackSimulatorURL = kibanaServer.resolveUrl(
+        getExternalServiceSimulatorPath(ExternalServiceSimulator.SLACK)
+      );
     });
 
     after(() => esArchiver.unload('empty_kibana'));

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/webhook.ts

@@ -28,6 +28,7 @@ function parsePort(url: Record<string, string>): Record<string, string | null |
 export default function webhookTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
 
   async function createWebhookAction(
     urlWithCreds: string,
@@ -65,10 +66,9 @@ export default function webhookTest({ getService }: FtrProviderContext) {
 
     // need to wait for kibanaServer to settle ...
     before(() => {
-      const kibanaServer = getService('kibanaServer');
-      const kibanaUrl = kibanaServer.status && kibanaServer.status.kibanaServerUrl;
-      const webhookServiceUrl = getExternalServiceSimulatorPath(ExternalServiceSimulator.WEBHOOK);
-      webhookSimulatorURL = `${kibanaUrl}${webhookServiceUrl}`;
+      webhookSimulatorURL = kibanaServer.resolveUrl(
+        getExternalServiceSimulatorPath(ExternalServiceSimulator.WEBHOOK)
+      );
     });
 
     after(() => esArchiver.unload('empty_kibana'));