[ftr] implement support for accessing ES through CCS (#126547)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-test/BUILD.bazel

@@ -53,6 +53,7 @@ RUNTIME_DEPS = [
     "@npm//execa",
     "@npm//exit-hook",
     "@npm//form-data",
+    "@npm//get-port",
     "@npm//getopts",
     "@npm//globby",
     "@npm//he",
@@ -90,6 +91,7 @@ TYPES_DEPS = [
   "@npm//del",
   "@npm//exit-hook",
   "@npm//form-data",
+  "@npm//get-port",
   "@npm//getopts",
   "@npm//jest",
   "@npm//jest-cli",

packages/kbn-test/src/es/es_client_for_testing.ts

@@ -27,6 +27,22 @@ export interface EsClientForTestingOptions extends Omit<ClientOptions, 'node' |
   isCloud?: boolean;
 }
 
+export function createRemoteEsClientForFtrConfig(
+  config: Config,
+  overrides?: Omit<EsClientForTestingOptions, 'esUrl'>
+) {
+  const ccsConfig = config.get('esTestCluster.ccs');
+  if (!ccsConfig) {
+    throw new Error('FTR config is missing esTestCluster.ccs');
+  }
+
+  return createEsClientForTesting({
+    esUrl: ccsConfig.remoteClusterUrl,
+    requestTimeout: config.get('timeouts.esRequestTimeout'),
+    ...overrides,
+  });
+}
+
 export function createEsClientForFtrConfig(
   config: Config,
   overrides?: Omit<EsClientForTestingOptions, 'esUrl'>

packages/kbn-test/src/es/index.ts

@@ -9,5 +9,9 @@
 export { createTestEsCluster } from './test_es_cluster';
 export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './test_es_cluster';
 export { esTestConfig } from './es_test_config';
-export { createEsClientForTesting, createEsClientForFtrConfig } from './es_client_for_testing';
+export {
+  createEsClientForTesting,
+  createEsClientForFtrConfig,
+  createRemoteEsClientForFtrConfig,
+} from './es_client_for_testing';
 export type { EsClientForTestingOptions } from './es_client_for_testing';

packages/kbn-test/src/es/test_es_cluster.ts

@@ -136,7 +136,15 @@ export interface CreateTestEsClusterOptions {
    * }
    */
   port?: number;
+  /**
+   * Should this ES cluster use SSL?
+   */
   ssl?: boolean;
+  /**
+   * Explicit transport port for a single node to run on, or a string port range to use eg. '9300-9400'
+   * defaults to the transport port from `packages/kbn-test/src/es/es_test_config.ts`
+   */
+  transportPort?: number | string;
 }
 
 export function createTestEsCluster<
@@ -155,13 +163,14 @@ export function createTestEsCluster<
     esJavaOpts,
     clusterName: customClusterName = 'es-test-cluster',
     ssl,
+    transportPort,
   } = options;
 
   const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`;
 
   const defaultEsArgs = [
     `cluster.name=${clusterName}`,
-    `transport.port=${esTestConfig.getTransportPort()}`,
+    `transport.port=${transportPort ?? esTestConfig.getTransportPort()}`,
     // For multi-node clusters, we make all nodes master-eligible by default.
     ...(nodes.length > 1
       ? ['discovery.type=zen', `cluster.initial_master_nodes=${nodes.map((n) => n.name).join(',')}`]

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -192,12 +192,17 @@ export const schema = Joi.object()
 
     esTestCluster: Joi.object()
       .keys({
-        license: Joi.string().default('basic'),
+        license: Joi.valid('basic', 'trial', 'gold').default('basic'),
         from: Joi.string().default('snapshot'),
-        serverArgs: Joi.array(),
+        serverArgs: Joi.array().items(Joi.string()),
         esJavaOpts: Joi.string(),
         dataArchive: Joi.string(),
         ssl: Joi.boolean().default(false),
+        ccs: Joi.object().keys({
+          remoteClusterUrl: Joi.string().uri({
+            scheme: /https?/,
+          }),
+        }),
       })
       .default(),
 
@@ -290,6 +295,7 @@ export const schema = Joi.object()
     security: Joi.object()
       .keys({
         roles: Joi.object().default(),
+        remoteEsRoles: Joi.object(),
         defaultRoles: Joi.array()
           .items(Joi.string())
           .when('$primary', {

packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts

@@ -8,6 +8,7 @@
 
 import { resolve } from 'path';
 import type { ToolingLog } from '@kbn/dev-utils';
+import getPort from 'get-port';
 import { KIBANA_ROOT } from './paths';
 import type { Config } from '../../functional_test_runner/';
 import { createTestEsCluster } from '../../es';
@@ -15,32 +16,102 @@ import { createTestEsCluster } from '../../es';
 interface RunElasticsearchOptions {
   log: ToolingLog;
   esFrom?: string;
-}
-export async function runElasticsearch({
-  config,
-  options,
-}: {
   config: Config;
-  options: RunElasticsearchOptions;
-}) {
-  const { log, esFrom } = options;
-  const ssl = config.get('esTestCluster.ssl');
-  const license = config.get('esTestCluster.license');
-  const esArgs = config.get('esTestCluster.serverArgs');
-  const esJavaOpts = config.get('esTestCluster.esJavaOpts');
+}
+
+interface CcsConfig {
+  remoteClusterUrl: string;
+}
+
+type EsConfig = ReturnType<typeof getEsConfig>;
+
+function getEsConfig({
+  config,
+  esFrom = config.get('esTestCluster.from'),
+}: RunElasticsearchOptions) {
+  const ssl = !!config.get('esTestCluster.ssl');
+  const license: 'basic' | 'trial' | 'gold' = config.get('esTestCluster.license');
+  const esArgs: string[] = config.get('esTestCluster.serverArgs') ?? [];
+  const esJavaOpts: string | undefined = config.get('esTestCluster.esJavaOpts');
   const isSecurityEnabled = esArgs.includes('xpack.security.enabled=true');
 
-  const cluster = createTestEsCluster({
-    port: config.get('servers.elasticsearch.port'),
-    password: isSecurityEnabled ? 'changeme' : config.get('servers.elasticsearch.password'),
+  const port: number | undefined = config.get('servers.elasticsearch.port');
+  const ccsConfig: CcsConfig | undefined = config.get('esTestCluster.ccs');
+
+  const password: string | undefined = isSecurityEnabled
+    ? 'changeme'
+    : config.get('servers.elasticsearch.password');
+
+  const dataArchive: string | undefined = config.get('esTestCluster.dataArchive');
+
+  return {
+    ssl,
     license,
-    log,
-    basePath: resolve(KIBANA_ROOT, '.es'),
-    esFrom: esFrom || config.get('esTestCluster.from'),
-    dataArchive: config.get('esTestCluster.dataArchive'),
     esArgs,
     esJavaOpts,
-    ssl,
+    isSecurityEnabled,
+    esFrom,
+    port,
+    password,
+    dataArchive,
+    ccsConfig,
+  };
+}
+
+export async function runElasticsearch(
+  options: RunElasticsearchOptions
+): Promise<() => Promise<void>> {
+  const { log } = options;
+  const config = getEsConfig(options);
+
+  if (!config.ccsConfig) {
+    const node = await startEsNode(log, 'ftr', config);
+    return async () => {
+      await node.cleanup();
+    };
+  }
+
+  const remotePort = await getPort();
+  const remoteNode = await startEsNode(log, 'ftr-remote', {
+    ...config,
+    port: parseInt(new URL(config.ccsConfig.remoteClusterUrl).port, 10),
+    transportPort: remotePort,
+  });
+
+  const localNode = await startEsNode(log, 'ftr-local', {
+    ...config,
+    esArgs: [...config.esArgs, `cluster.remote.ftr-remote.seeds=localhost:${remotePort}`],
+  });
+
+  return async () => {
+    await localNode.cleanup();
+    await remoteNode.cleanup();
+  };
+}
+
+async function startEsNode(
+  log: ToolingLog,
+  name: string,
+  config: EsConfig & { transportPort?: number }
+) {
+  const cluster = createTestEsCluster({
+    clusterName: `cluster-${name}`,
+    esArgs: config.esArgs,
+    esFrom: config.esFrom,
+    esJavaOpts: config.esJavaOpts,
+    license: config.license,
+    password: config.password,
+    port: config.port,
+    ssl: config.ssl,
+    log,
+    basePath: resolve(KIBANA_ROOT, '.es'),
+    nodes: [
+      {
+        name,
+        dataArchive: config.dataArchive,
+      },
+    ],
+    transportPort: config.transportPort,
   });
 
   await cluster.start();

packages/kbn-test/src/functional_tests/tasks.ts

@@ -108,10 +108,10 @@ export async function runTests(options: RunTestsParams) {
       await withProcRunner(log, async (procs) => {
         const config = await readConfigFile(log, options.esVersion, configPath);
 
-        let es;
+        let shutdownEs;
         try {
           if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') {
-            es = await runElasticsearch({ config, options: { ...options, log } });
+            shutdownEs = await runElasticsearch({ ...options, log, config });
           }
           await runKibanaServer({ procs, config, options });
           await runFtr({ configPath, options: { ...options, log } });
@@ -125,8 +125,8 @@ export async function runTests(options: RunTestsParams) {
 
             await procs.stop('kibana');
           } finally {
-            if (es) {
-              await es.cleanup();
+            if (shutdownEs) {
+              await shutdownEs();
             }
           }
         }
@@ -166,7 +166,7 @@ export async function startServers({ ...options }: StartServerOptions) {
   await withProcRunner(log, async (procs) => {
     const config = await readConfigFile(log, options.esVersion, options.config);
 
-    const es = await runElasticsearch({ config, options: opts });
+    const shutdownEs = await runElasticsearch({ ...opts, config });
     await runKibanaServer({
       procs,
       config,
@@ -190,7 +190,7 @@ export async function startServers({ ...options }: StartServerOptions) {
     log.success(makeSuccessMessage(options));
 
     await procs.waitForAllToStop();
-    await es.cleanup();
+    await shutdownEs();
   });
 }
 

packages/kbn-test/src/index.ts

@@ -36,6 +36,7 @@ export {
   createTestEsCluster,
   createEsClientForTesting,
   createEsClientForFtrConfig,
+  createRemoteEsClientForFtrConfig,
 } from './es';
 
 export {

scripts/functional_tests.js

@@ -9,7 +9,7 @@
 require('../src/setup_node_env');
 require('@kbn/test').runTestsCli([
   require.resolve('../test/functional/config.js'),
-  require.resolve('../test/functional_ccs/config.js'),
+  require.resolve('../test/functional_ccs/config.ts'),
   require.resolve('../test/plugin_functional/config.ts'),
   require.resolve('../test/ui_capabilities/newsfeed_err/config.ts'),
   require.resolve('../test/new_visualize_flow/config.ts'),

test/common/services/es_archiver.ts

@@ -8,11 +8,12 @@
 
 import { EsArchiver } from '@kbn/es-archiver';
 import { FtrProviderContext } from '../ftr_provider_context';
-import * as KibanaServer from './kibana_server';
+import * as KibanaServer from '../../common/services/kibana_server';
 
 export function EsArchiverProvider({ getService }: FtrProviderContext): EsArchiver {
   const config = getService('config');
   const client = getService('es');
+
   const log = getService('log');
   const kibanaServer = getService('kibanaServer');
   const retry = getService('retry');

test/common/services/security/system_indices_user.ts

@@ -6,25 +6,18 @@
  * Side Public License, v 1.
  */
 
-import { systemIndicesSuperuser, createEsClientForFtrConfig } from '@kbn/test';
+import { Client } from '@elastic/elasticsearch';
+import { ToolingLog } from '@kbn/dev-utils';
+import {
+  systemIndicesSuperuser,
+  createEsClientForFtrConfig,
+  createRemoteEsClientForFtrConfig,
+} from '@kbn/test';
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 const SYSTEM_INDICES_SUPERUSER_ROLE = 'system_indices_superuser';
 
-export async function createSystemIndicesUser(ctx: FtrProviderContext) {
-  const log = ctx.getService('log');
-  const config = ctx.getService('config');
-
-  const enabled = !config
-    .get('esTestCluster.serverArgs')
-    .some((arg: string) => arg === 'xpack.security.enabled=false');
-
-  if (!enabled) {
-    return;
-  }
-
-  const es = createEsClientForFtrConfig(config);
-
+async function ensureSystemIndicesUser(es: Client, log: ToolingLog) {
   // There are cases where the test config file doesn't have security disabled
   // but tests are still executed on ES without security. Checking this case
   // by trying to fetch the users list.
@@ -67,3 +60,24 @@ export async function createSystemIndicesUser(ctx: FtrProviderContext) {
 
   await es.close();
 }
+
+export async function createSystemIndicesUser(ctx: FtrProviderContext) {
+  const log = ctx.getService('log');
+  const config = ctx.getService('config');
+
+  const enabled = !config
+    .get('esTestCluster.serverArgs')
+    .some((arg: string) => arg === 'xpack.security.enabled=false');
+
+  if (!enabled) {
+    return;
+  }
+
+  const localEs = createEsClientForFtrConfig(config);
+  await ensureSystemIndicesUser(localEs, log);
+
+  if (config.get('esTestCluster.ccs')) {
+    const remoteEs = createRemoteEsClientForFtrConfig(config);
+    await ensureSystemIndicesUser(remoteEs, log);
+  }
+}

test/common/services/security/test_user.ts

@@ -90,6 +90,28 @@ export async function createTestUserService(ctx: FtrProviderContext, role: Role,
       await role.create(name, definition);
     }
 
+    // when configured to setup remote roles, load the remote es service and set them up directly via es
+    const remoteEsRoles: undefined | Record<string, any> = config.get('security.remoteEsRoles');
+    if (remoteEsRoles) {
+      let remoteEs;
+      try {
+        remoteEs = ctx.getService('remoteEs' as 'es');
+      } catch (error) {
+        throw new Error(
+          'unable to load `remoteEs` cluster, which should provide an ES client configured to talk to the remote cluster. Include that service from another FTR config or fix the error it is throwing on creation: ' +
+            error.message
+        );
+      }
+
+      for (const [name, body] of Object.entries(remoteEsRoles)) {
+        log.info(`creating ${name} role on remote cluster`);
+        await remoteEs.security.putRole({
+          name,
+          ...body,
+        });
+      }
+    }
+
     // delete the test_user if present (will it error if the user doesn't exist?)
     try {
       await user.delete(TEST_USER_NAME);

test/functional/fixtures/kbn_archiver/date_nested_ccs.json

@@ -2,10 +2,10 @@
   "attributes": {
     "fields": "[]",
     "timeFieldName": "nested.timestamp",
-    "title": "remote:date-nested"
+    "title": "ftr-remote:date-nested"
   },
   "coreMigrationVersion": "8.2.0",
-  "id": "remote:date-nested",
+  "id": "ftr-remote:date-nested",
   "migrationVersion": {
     "index-pattern": "8.0.0"
   },

test/functional_ccs/apps/discover/data_view_ccs.ts

@@ -6,13 +6,13 @@
  * Side Public License, v 1.
  */
 
-import { FtrProviderContext } from './ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const testSubjects = getService('testSubjects');
   const kibanaServer = getService('kibanaServer');
-  const esArchiver = getService('esArchiver');
+  const remoteEsArchiver = getService('remoteEsArchiver');
 
   const security = getService('security');
   const PageObjects = getPageObjects(['common', 'discover', 'header', 'timePicker']);
@@ -30,8 +30,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   // FLAKY: https://github.com/elastic/kibana/issues/126658
   describe.skip('discover integration with data view editor', function describeIndexTests() {
     before(async function () {
-      await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
-      await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
+      await security.testUser.setRoles([
+        'kibana_admin',
+        'test_logstash_reader',
+        'ccs_remote_search',
+      ]);
+      await remoteEsArchiver.loadIfNeeded(
+        'test/functional/fixtures/es_archiver/logstash_functional'
+      );
       await kibanaServer.savedObjects.clean({ types: ['saved-search', 'index-pattern'] });
       await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');
       await PageObjects.timePicker.setDefaultAbsoluteRangeViaUiSettings();
@@ -45,7 +51,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     });
 
     it('use ccs to create a new data view', async function () {
-      const dataViewToCreate = 'remote:logstash';
+      const dataViewToCreate = 'ftr-remote:logstash';
       await createDataView(dataViewToCreate);
       await PageObjects.header.waitUntilLoadingHasFinished();
       await retry.waitForWithTimeout(

test/functional_ccs/apps/discover/index.ts

@@ -6,38 +6,24 @@
  * Side Public License, v 1.
  */
 
-import { FtrProviderContext } from './ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, loadTestFile }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const browser = getService('browser');
-  const esClient = getService('es');
 
   describe('discover app css', function () {
     this.tags('ciGroup6');
 
-    before(async function () {
-      await esClient.cluster.putSettings({
-        persistent: {
-          cluster: {
-            remote: {
-              remote: {
-                skip_unavailable: 'true',
-                seeds: ['localhost:9300'],
-              },
-            },
-          },
-        },
-      });
-      return browser.setWindowSize(1300, 800);
+    before(async () => {
+      await browser.setWindowSize(1300, 800);
     });
 
-    after(function unloadMakelogs() {
-      // Make sure to clean up the cluster setting from the before above.
-      return esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
-    });
+    loadTestFile(require.resolve('./data_view_ccs'));
+    loadTestFile(require.resolve('./saved_queries_ccs'));
 
-    loadTestFile(require.resolve('./_data_view_ccs'));
-    loadTestFile(require.resolve('./_saved_queries_ccs'));
+    after(async () => {
+      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+    });
   });
 }

test/functional_ccs/apps/discover/saved_queries_ccs.ts

@@ -8,12 +8,12 @@
 
 import expect from '@kbn/expect';
 
-import { FtrProviderContext } from './ftr_provider_context';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const log = getService('log');
-  const esArchiver = getService('esArchiver');
+  const remoteEsArchiver = getService('remoteEsArchiver');
   const kibanaServer = getService('kibanaServer');
   const PageObjects = getPageObjects(['common', 'discover', 'timePicker']);
   const browser = getService('browser');
@@ -47,8 +47,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       await kibanaServer.importExport.load(
         'test/functional/fixtures/kbn_archiver/date_nested_ccs.json'
       );
-      await esArchiver.load('test/functional/fixtures/es_archiver/date_nested');
-      await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional');
+      await remoteEsArchiver.load('test/functional/fixtures/es_archiver/date_nested');
+      await remoteEsArchiver.load('test/functional/fixtures/es_archiver/logstash_functional');
 
       await kibanaServer.uiSettings.replace(defaultSettings);
       log.debug('discover');
@@ -61,8 +61,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       await kibanaServer.importExport.unload(
         'test/functional/fixtures/kbn_archiver/date_nested_ccs'
       );
-      await esArchiver.unload('test/functional/fixtures/es_archiver/date_nested');
-      await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
+      await remoteEsArchiver.unload('test/functional/fixtures/es_archiver/date_nested');
+      await remoteEsArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional');
       await PageObjects.common.unsetTime();
     });
 
@@ -87,12 +87,12 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
         expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false);
         expect(await queryBar.getQueryString()).to.eql('');
 
-        await PageObjects.discover.selectIndexPattern('remote:date-nested');
+        await PageObjects.discover.selectIndexPattern('ftr-remote:date-nested');
 
         expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false);
         expect(await queryBar.getQueryString()).to.eql('');
 
-        await PageObjects.discover.selectIndexPattern('remote:logstash-*');
+        await PageObjects.discover.selectIndexPattern('ftr-remote:logstash-*');
 
         expect(await filterBar.hasFilter('extension.raw', 'jpg')).to.be(false);
         expect(await queryBar.getQueryString()).to.eql('');

test/functional_ccs/config.js

@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { services } from '../functional/services';
-
-export default async function ({ readConfigFile }) {
-  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
-
-  return {
-    ...functionalConfig.getAll(),
-
-    testFiles: [require.resolve('./apps/discover')],
-
-    services,
-
-    junit: {
-      reportName: 'Kibana CCS Tests',
-    },
-  };
-}

test/functional_ccs/config.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test';
+import { services } from './services';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [require.resolve('./apps/discover')],
+
+    services,
+
+    junit: {
+      reportName: 'Kibana CCS Tests',
+    },
+
+    security: {
+      ...functionalConfig.get('security'),
+      remoteEsRoles: {
+        ccs_remote_search: {
+          indices: [
+            {
+              names: ['*'],
+              privileges: ['read', 'view_index_metadata', 'read_cross_cluster'],
+            },
+          ],
+        },
+      },
+      defaultRoles: [...(functionalConfig.get('security.defaultRoles') ?? []), 'ccs_remote_search'],
+    },
+
+    esTestCluster: {
+      ...functionalConfig.get('esTestCluster'),
+      ccs: {
+        remoteClusterUrl:
+          process.env.REMOTE_CLUSTER_URL ??
+          `http://elastic:changeme@localhost:${
+            functionalConfig.get('servers.elasticsearch.port') + 1
+          }`,
+      },
+    },
+  };
+}

test/functional_ccs/ftr_provider_context.ts

@@ -7,7 +7,7 @@
  */
 
 import { GenericFtrProviderContext } from '@kbn/test';
-import { services } from '../../../functional/services';
-import { pageObjects } from '../../../functional/page_objects';
+import { services } from './services';
+import { pageObjects } from '../functional/page_objects';
 
 export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;

test/functional_ccs/services/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { services as functionalServices } from '../../functional/services';
+import { RemoteEsProvider } from './remote_es';
+import { RemoteEsArchiverProvider } from './remote_es_archiver';
+
+export const services = {
+  ...functionalServices,
+  remoteEs: RemoteEsProvider,
+  remoteEsArchiver: RemoteEsArchiverProvider,
+};

test/functional_ccs/services/remote_es.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+import { systemIndicesSuperuser, createRemoteEsClientForFtrConfig } from '@kbn/test';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+/**
+ * Kibana-specific @elastic/elasticsearch client instance.
+ */
+export function RemoteEsProvider({ getService }: FtrProviderContext): Client {
+  const config = getService('config');
+
+  return createRemoteEsClientForFtrConfig(config, {
+    // Use system indices user so tests can write to system indices
+    authOverride: systemIndicesSuperuser,
+  });
+}

test/functional_ccs/services/remote_es_archiver.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { EsArchiver } from '@kbn/es-archiver';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export function RemoteEsArchiverProvider({ getService }: FtrProviderContext): EsArchiver {
+  const remoteEs = getService('remoteEs');
+  const log = getService('log');
+  const kibanaServer = getService('kibanaServer');
+
+  return new EsArchiver({
+    client: remoteEs,
+    log,
+    kbnClient: kibanaServer,
+  });
+}