Functional test runner creates system_indices_superuser (#124008)

This PR fixes the functional test runner for execution against cloud (and other existing deployments) by making sure the system_indices_superuser exists.

packages/kbn-test/src/es/es_client_for_testing.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as Url from 'url';
+import * as Fs from 'fs';
+
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { Client as EsClient, ClientOptions, HttpConnection } from '@elastic/elasticsearch';
+import type { Config } from '../functional_test_runner';
+
+/** options for creating es instances used in functional testing scenarios */
+export interface EsClientForTestingOptions extends Omit<ClientOptions, 'node' | 'nodes' | 'tls'> {
+  /** url of es instance */
+  esUrl: string;
+  /** overwrite the auth embedded in the url to use a different user in this client instance */
+  authOverride?: { username: string; password: string };
+  /**
+   * are we running tests against cloud? this is automatically determined
+   * by checking for the TEST_CLOUD environment variable but can be overriden
+   * for special cases
+   */
+  isCloud?: boolean;
+}
+
+export function createEsClientForFtrConfig(
+  config: Config,
+  overrides?: Omit<EsClientForTestingOptions, 'esUrl'>
+) {
+  const esUrl = Url.format(config.get('servers.elasticsearch'));
+  return createEsClientForTesting({
+    esUrl,
+    requestTimeout: config.get('timeouts.esRequestTimeout'),
+    ...overrides,
+  });
+}
+
+export function createEsClientForTesting(options: EsClientForTestingOptions) {
+  const { esUrl, authOverride, isCloud = !!process.env.TEST_CLOUD, ...otherOptions } = options;
+
+  const url = options.authOverride
+    ? Url.format({
+        ...Url.parse(options.esUrl),
+        auth: `${options.authOverride.username}:${options.authOverride.password}`,
+      })
+    : options.esUrl;
+
+  return new EsClient({
+    Connection: HttpConnection,
+    tls: isCloud ? undefined : { ca: Fs.readFileSync(CA_CERT_PATH) },
+
+    ...otherOptions,
+
+    // force nodes config
+    nodes: [url],
+  });
+}

packages/kbn-test/src/es/index.ts

@@ -10,3 +10,5 @@ export { createTestEsCluster } from './test_es_cluster';
 export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './test_es_cluster';
 export { esTestConfig } from './es_test_config';
 export { convertToKibanaClient } from './client_to_kibana_client';
+export { createEsClientForTesting, createEsClientForFtrConfig } from './es_client_for_testing';
+export type { EsClientForTestingOptions } from './es_client_for_testing';

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -6,7 +6,6 @@
  * Side Public License, v 1.
  */
 
-import type { Client as EsClient } from '@elastic/elasticsearch';
 import { ToolingLog } from '@kbn/dev-utils';
 
 import { Suite, Test } from './fake_mocha_types';
@@ -24,6 +23,7 @@ import {
   SuiteTracker,
   EsVersion,
 } from './lib';
+import { createEsClientForFtrConfig } from '../es';
 
 export class FunctionalTestRunner {
   public readonly lifecycle = new Lifecycle();
@@ -61,27 +61,9 @@ export class FunctionalTestRunner {
         ...readProviderSpec('PageObject', config.get('pageObjects')),
       ]);
 
-      // validate es version
       if (providers.hasService('es')) {
-        const es = (await providers.getService('es')) as unknown as EsClient;
-        let esInfo;
-        try {
-          esInfo = await es.info();
-        } catch (error) {
-          throw new Error(
-            `attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ${error.stack}`
-          );
-        }
-
-        if (!this.esVersion.eql(esInfo.version.number)) {
-          throw new Error(
-            `ES reports a version number "${
-              esInfo.version.number
-            }" which doesn't match supplied es version "${this.esVersion.toString()}"`
-          );
-        }
+        await this.validateEsVersion(config);
       }
-
       await providers.loadAll();
 
       const customTestRunner = config.get('testRunner');
@@ -100,6 +82,33 @@ export class FunctionalTestRunner {
     });
   }
 
+  private async validateEsVersion(config: Config) {
+    const es = createEsClientForFtrConfig(config);
+
+    let esInfo;
+    try {
+      esInfo = await es.info();
+    } catch (error) {
+      throw new Error(
+        `attempted to use the "es" service to fetch Elasticsearch version info but the request failed: ${error.stack}`
+      );
+    } finally {
+      try {
+        await es.close();
+      } catch {
+        // noop
+      }
+    }
+
+    if (!this.esVersion.eql(esInfo.version.number)) {
+      throw new Error(
+        `ES reports a version number "${
+          esInfo.version.number
+        }" which doesn't match supplied es version "${this.esVersion.toString()}"`
+      );
+    }
+  }
+
   async getTestStats() {
     return await this._run(async (config, coreProviders) => {
       if (config.get('testRunner')) {

packages/kbn-test/src/index.ts

@@ -25,8 +25,19 @@ export { runTests, startServers } from './functional_tests/tasks';
 // @internal
 export { KIBANA_ROOT } from './functional_tests/lib/paths';
 
-export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './es';
-export { esTestConfig, createTestEsCluster, convertToKibanaClient } from './es';
+export type {
+  CreateTestEsClusterOptions,
+  EsTestCluster,
+  ICluster,
+  EsClientForTestingOptions,
+} from './es';
+export {
+  esTestConfig,
+  createTestEsCluster,
+  convertToKibanaClient,
+  createEsClientForTesting,
+  createEsClientForFtrConfig,
+} from './es';
 
 export {
   kbnTestConfig,

test/common/services/elasticsearch.ts

@@ -6,12 +6,9 @@
  * Side Public License, v 1.
  */
 
-import { format as formatUrl } from 'url';
-import fs from 'fs';
-import { Client, HttpConnection } from '@elastic/elasticsearch';
-import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { Client } from '@elastic/elasticsearch';
 
-import { systemIndicesSuperuser } from '@kbn/test';
+import { systemIndicesSuperuser, createEsClientForFtrConfig } from '@kbn/test';
 import { FtrProviderContext } from '../ftr_provider_context';
 
 /*
@@ -20,26 +17,8 @@ import { FtrProviderContext } from '../ftr_provider_context';
 export function ElasticsearchProvider({ getService }: FtrProviderContext): Client {
   const config = getService('config');
 
-  const esUrl = formatUrl({
-    ...config.get('servers.elasticsearch'),
+  return createEsClientForFtrConfig(config, {
     // Use system indices user so tests can write to system indices
-    auth: `${systemIndicesSuperuser.username}:${systemIndicesSuperuser.password}`,
+    authOverride: systemIndicesSuperuser,
   });
-
-  if (process.env.TEST_CLOUD) {
-    return new Client({
-      nodes: [esUrl],
-      requestTimeout: config.get('timeouts.esRequestTimeout'),
-      Connection: HttpConnection,
-    });
-  } else {
-    return new Client({
-      tls: {
-        ca: fs.readFileSync(CA_CERT_PATH, 'utf-8'),
-      },
-      nodes: [esUrl],
-      requestTimeout: config.get('timeouts.esRequestTimeout'),
-      Connection: HttpConnection,
-    });
-  }
 }

test/common/services/security/security.ts

@@ -11,6 +11,7 @@ import { User } from './user';
 import { RoleMappings } from './role_mappings';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import { createTestUserService, TestUserSupertestProvider, TestUser } from './test_user';
+import { createSystemIndicesUser } from './system_indices_user';
 
 export class SecurityService {
   constructor(
@@ -28,6 +29,7 @@ export async function SecurityServiceProvider(ctx: FtrProviderContext) {
 
   const role = new Role(log, kibanaServer);
   const user = new User(log, kibanaServer);
+  await createSystemIndicesUser(ctx);
   const testUser = await createTestUserService(ctx, role, user);
   const testUserSupertest = TestUserSupertestProvider(ctx);
   const roleMappings = new RoleMappings(log, kibanaServer);

test/common/services/security/system_indices_user.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { systemIndicesSuperuser, createEsClientForFtrConfig } from '@kbn/test';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+const SYSTEM_INDICES_SUPERUSER_ROLE = 'system_indices_superuser';
+
+export async function createSystemIndicesUser(ctx: FtrProviderContext) {
+  const log = ctx.getService('log');
+  const config = ctx.getService('config');
+
+  const enabled = !config
+    .get('esTestCluster.serverArgs')
+    .some((arg: string) => arg === 'xpack.security.enabled=false');
+
+  if (!enabled) {
+    return;
+  }
+
+  const es = createEsClientForFtrConfig(config);
+
+  log.debug('===============creating system indices role and user===============');
+
+  await es.security.putRole({
+    name: SYSTEM_INDICES_SUPERUSER_ROLE,
+    refresh: 'wait_for',
+    cluster: ['all'],
+    indices: [
+      {
+        names: ['*'],
+        privileges: ['all'],
+        allow_restricted_indices: true,
+      },
+    ],
+    applications: [
+      {
+        application: '*',
+        privileges: ['*'],
+        resources: ['*'],
+      },
+    ],
+    run_as: ['*'],
+  });
+
+  await es.security.putUser({
+    username: systemIndicesSuperuser.username,
+    refresh: 'wait_for',
+    password: systemIndicesSuperuser.password,
+    roles: [SYSTEM_INDICES_SUPERUSER_ROLE],
+  });
+
+  await es.close();
+}

x-pack/plugins/apm/ftr_e2e/cypress/plugins/index.ts

@@ -4,10 +4,8 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import Fs from 'fs';
-import { Client, HttpConnection } from '@elastic/elasticsearch';
 import { apm, createLogger, LogLevel } from '@elastic/apm-synthtrace';
-import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { createEsClientForTesting } from '@kbn/test';
 
 // ***********************************************************
 // This example plugins/index.ts can be used to load plugins
@@ -29,15 +27,10 @@ const plugin: Cypress.PluginConfig = (on, config) => {
   // `on` is used to hook into various events Cypress emits
   // `config` is the resolved Cypress config
 
-  const node = config.env.ES_NODE;
-  const requestTimeout = config.env.ES_REQUEST_TIMEOUT;
-  const isCloud = config.env.TEST_CLOUD;
-
-  const client = new Client({
-    node,
-    requestTimeout,
-    Connection: HttpConnection,
-    ...(isCloud ? { tls: { ca: Fs.readFileSync(CA_CERT_PATH, 'utf-8') } } : {}),
+  const client = createEsClientForTesting({
+    esUrl: config.env.ES_NODE,
+    requestTimeout: config.env.ES_REQUEST_TIMEOUT,
+    isCloud: !!config.env.TEST_CLOUD,
   });
 
   const synthtraceEsClient = new apm.ApmSynthtraceEsClient(

x-pack/test/functional/apps/security/users.ts

@@ -44,7 +44,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
 
       // In Cloud default users are defined in file realm, such users aren't exposed through the Users API.
       if (isCloudEnvironment()) {
-        expect(Object.keys(users)).to.eql(['test_user']);
+        expect(users).to.not.have.property('elastic');
+        expect(users).to.not.have.property('kibana_system');
+        expect(users).to.not.have.property('kibana');
       } else {
         expect(users.elastic.roles).to.eql(['superuser']);
         expect(users.elastic.reserved).to.be(true);

x-pack/test/rule_registry/common/services/cluster_client.ts

@@ -5,10 +5,8 @@
  * 2.0.
  */
 
-import { format as formatUrl } from 'url';
-import fs from 'fs';
-import { Client, HttpConnection, Transport } from '@elastic/elasticsearch';
-import { CA_CERT_PATH } from '@kbn/dev-utils';
+import { Client, Transport } from '@elastic/elasticsearch';
+import { createEsClientForFtrConfig } from '@kbn/test';
 import type {
   TransportRequestParams,
   TransportRequestOptions,
@@ -35,22 +33,7 @@ export function clusterClientProvider({ getService }: FtrProviderContext): Clien
     }
   }
 
-  if (process.env.TEST_CLOUD) {
-    return new Client({
-      nodes: [formatUrl(config.get('servers.elasticsearch'))],
-      requestTimeout: config.get('timeouts.esRequestTimeout'),
-      Transport: KibanaTransport,
-      Connection: HttpConnection,
-    });
-  } else {
-    return new Client({
-      tls: {
-        ca: fs.readFileSync(CA_CERT_PATH, 'utf-8'),
-      },
-      nodes: [formatUrl(config.get('servers.elasticsearch'))],
-      requestTimeout: config.get('timeouts.esRequestTimeout'),
-      Transport: KibanaTransport,
-      Connection: HttpConnection,
-    });
-  }
+  return createEsClientForFtrConfig(config, {
+    Transport: KibanaTransport,
+  });
 }