Support for superuser not having write access (#123337) (#123639)

Co-authored-by: Nicolas Chaulet <nicolas.chaulet@elastic.co>
Co-authored-by: Timothy Sullivan <tsullivan@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit 23739f5f4d)

# Conflicts:
#	src/core/server/saved_objects/migrations/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-es/src/index.ts

@@ -10,3 +10,4 @@
 export { run } from './cli';
 // @ts-expect-error not typed yet
 export { Cluster } from './cluster';
+export { SYSTEM_INDICES_SUPERUSER } from './utils';

packages/kbn-es/src/utils/index.ts

@@ -14,6 +14,6 @@ export { findMostRecentlyChanged } from './find_most_recently_changed';
 // @ts-expect-error not typed yet
 export { extractConfigFiles } from './extract_config_files';
 // @ts-expect-error not typed yet
-export { NativeRealm } from './native_realm';
+export { NativeRealm, SYSTEM_INDICES_SUPERUSER } from './native_realm';
 export { buildSnapshot } from './build_snapshot';
 export { archiveForPlatform } from './build_snapshot';

packages/kbn-es/src/utils/native_realm.js

@@ -11,6 +11,9 @@ const chalk = require('chalk');
 
 const { log: defaultLog } = require('./log');
 
+export const SYSTEM_INDICES_SUPERUSER =
+  process.env.TEST_ES_SYSTEM_INDICES_USER || 'system_indices_superuser';
+
 exports.NativeRealm = class NativeRealm {
   constructor({ elasticPassword, port, log = defaultLog, ssl = false, caCert }) {
     const auth = { username: 'elastic', password: elasticPassword };
@@ -57,11 +60,12 @@ exports.NativeRealm = class NativeRealm {
     }
 
     const reservedUsers = await this.getReservedUsers();
-    await Promise.all(
-      reservedUsers.map(async (user) => {
+    await Promise.all([
+      ...reservedUsers.map(async (user) => {
         await this.setPassword(user, options[`password.${user}`]);
-      })
-    );
+      }),
+      this._createSystemIndicesUser(),
+    ]);
   }
 
   async getReservedUsers(retryOpts = {}) {
@@ -113,4 +117,39 @@ exports.NativeRealm = class NativeRealm {
       return await this._autoRetry(nextOpts, fn);
     }
   }
+
+  async _createSystemIndicesUser() {
+    if (!(await this.isSecurityEnabled())) {
+      this._log.info('security is not enabled, unable to create role and user');
+      return;
+    }
+
+    await this._client.security.putRole({
+      name: SYSTEM_INDICES_SUPERUSER,
+      refresh: 'wait_for',
+      cluster: ['all'],
+      indices: [
+        {
+          names: ['*'],
+          privileges: ['all'],
+          allow_restricted_indices: true,
+        },
+      ],
+      applications: [
+        {
+          application: '*',
+          privileges: ['*'],
+          resources: ['*'],
+        },
+      ],
+      run_as: ['*'],
+    });
+
+    await this._client.security.putUser({
+      username: SYSTEM_INDICES_SUPERUSER,
+      refresh: 'wait_for',
+      password: this._elasticPassword,
+      roles: [SYSTEM_INDICES_SUPERUSER],
+    });
+  }
 };

packages/kbn-es/src/utils/native_realm.test.js

@@ -21,6 +21,8 @@ const mockClient = {
   security: {
     changePassword: jest.fn(),
     getUser: jest.fn(),
+    putRole: jest.fn(),
+    putUser: jest.fn(),
   },
 };
 Client.mockImplementation(() => mockClient);

packages/kbn-test/src/index.ts

@@ -28,7 +28,13 @@ export { KIBANA_ROOT } from './functional_tests/lib/paths';
 export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './es';
 export { esTestConfig, createTestEsCluster, convertToKibanaClient } from './es';
 
-export { kbnTestConfig, kibanaServerTestUser, kibanaTestUser, adminTestUser } from './kbn';
+export {
+  kbnTestConfig,
+  kibanaServerTestUser,
+  kibanaTestUser,
+  adminTestUser,
+  systemIndicesSuperuser,
+} from './kbn';
 
 export { readConfigFile } from './functional_test_runner/lib/config/read_config_file';
 

packages/kbn-test/src/kbn/index.ts

@@ -7,4 +7,9 @@
  */
 
 export { kbnTestConfig } from './kbn_test_config';
-export { kibanaTestUser, kibanaServerTestUser, adminTestUser } from './users';
+export {
+  kibanaTestUser,
+  kibanaServerTestUser,
+  adminTestUser,
+  systemIndicesSuperuser,
+} from './users';

packages/kbn-test/src/kbn/users.ts

@@ -6,6 +6,9 @@
  * Side Public License, v 1.
  */
 
+// @ts-expect-error no types
+import { SYSTEM_INDICES_SUPERUSER } from '@kbn/es';
+
 const env = process.env;
 
 export const kibanaTestUser = {
@@ -22,3 +25,11 @@ export const adminTestUser = {
   username: env.TEST_ES_USER || 'elastic',
   password: env.TEST_ES_PASS || 'changeme',
 };
+
+/**
+ * User with higher privileges than regular superuser role for writing to system indices
+ */
+export const systemIndicesSuperuser = {
+  username: SYSTEM_INDICES_SUPERUSER,
+  password: env.TEST_ES_PASS || 'changeme',
+};

src/core/server/saved_objects/migrations/integration_tests/batch_size_bytes.test.ts

@@ -51,7 +51,8 @@ async function fetchDocuments(esClient: ElasticsearchClient, index: string) {
 
 const assertMigratedDocuments = (arr: any[], target: any[]) => target.every((v) => arr.includes(v));
 
-describe('migration v2', () => {
+// dataArchive not compatible with ES 8.0+
+describe.skip('migration v2', () => {
   let esServer: kbnTestServer.TestElasticsearchUtils;
   let root: Root;
   let startES: () => Promise<kbnTestServer.TestElasticsearchUtils>;

src/core/server/saved_objects/migrations/integration_tests/batch_size_bytes_exceeds_es_content_length.test.ts

@@ -21,6 +21,7 @@ async function removeLogFile() {
 }
 
 // un-skip after https://github.com/elastic/kibana/issues/116111
+// dataArchive not compatible with ES 8.0+
 describe.skip('migration v2', () => {
   let esServer: kbnTestServer.TestElasticsearchUtils;
   let root: Root;

src/core/test_helpers/kbn_server.ts

@@ -13,7 +13,7 @@ import {
   CreateTestEsClusterOptions,
   esTestConfig,
   kibanaServerTestUser,
-  kibanaTestUser,
+  systemIndicesSuperuser,
 } from '@kbn/test';
 import { defaultsDeep } from 'lodash';
 import { BehaviorSubject } from 'rxjs';
@@ -76,7 +76,9 @@ export function createRootWithSettings(
  * @param path
  */
 export function getSupertest(root: Root, method: HttpMethod, path: string) {
-  const testUserCredentials = Buffer.from(`${kibanaTestUser.username}:${kibanaTestUser.password}`);
+  const testUserCredentials = Buffer.from(
+    `${systemIndicesSuperuser.username}:${systemIndicesSuperuser.password}`
+  );
   return supertest((root as any).server.http.httpServer.server.listener)
     [method](path)
     .set('Authorization', `Basic ${testUserCredentials.toString('base64')}`);

test/api_integration/services/supertest.ts

@@ -6,6 +6,8 @@
  * Side Public License, v 1.
  */
 
+import { systemIndicesSuperuser } from '@kbn/test';
+
 import { FtrProviderContext } from 'test/functional/ftr_provider_context';
 import { format as formatUrl } from 'url';
 
@@ -20,7 +22,11 @@ export function KibanaSupertestProvider({ getService }: FtrProviderContext) {
 export function ElasticsearchSupertestProvider({ getService }: FtrProviderContext) {
   const config = getService('config');
   const esServerConfig = config.get('servers.elasticsearch');
-  const elasticSearchServerUrl = formatUrl(esServerConfig);
+  const elasticSearchServerUrl = formatUrl({
+    ...esServerConfig,
+    // Use system indices user so tests can write to system indices
+    auth: `${systemIndicesSuperuser.username}:${systemIndicesSuperuser.password}`,
+  });
 
   let agentOptions = {};
   if ('certificateAuthorities' in esServerConfig) {

test/common/services/elasticsearch.ts

@@ -11,6 +11,7 @@ import fs from 'fs';
 import { Client, HttpConnection } from '@elastic/elasticsearch';
 import { CA_CERT_PATH } from '@kbn/dev-utils';
 
+import { systemIndicesSuperuser } from '@kbn/test';
 import { FtrProviderContext } from '../ftr_provider_context';
 
 /*
@@ -19,9 +20,15 @@ import { FtrProviderContext } from '../ftr_provider_context';
 export function ElasticsearchProvider({ getService }: FtrProviderContext): Client {
   const config = getService('config');
 
+  const esUrl = formatUrl({
+    ...config.get('servers.elasticsearch'),
+    // Use system indices user so tests can write to system indices
+    auth: `${systemIndicesSuperuser.username}:${systemIndicesSuperuser.password}`,
+  });
+
   if (process.env.TEST_CLOUD) {
     return new Client({
-      nodes: [formatUrl(config.get('servers.elasticsearch'))],
+      nodes: [esUrl],
       requestTimeout: config.get('timeouts.esRequestTimeout'),
       Connection: HttpConnection,
     });
@@ -30,7 +37,7 @@ export function ElasticsearchProvider({ getService }: FtrProviderContext): Clien
       tls: {
         ca: fs.readFileSync(CA_CERT_PATH, 'utf-8'),
       },
-      nodes: [formatUrl(config.get('servers.elasticsearch'))],
+      nodes: [esUrl],
       requestTimeout: config.get('timeouts.esRequestTimeout'),
       Connection: HttpConnection,
     });

x-pack/plugins/fleet/server/integration_tests/reset_preconfiguration.test.ts

@@ -208,7 +208,8 @@ describe('Fleet preconfiguration rest', () => {
     });
   });
 
-  describe('Reset one preconfigured policy', () => {
+  // SKIP: https://github.com/elastic/kibana/issues/123528
+  describe.skip('Reset one preconfigured policy', () => {
     const POLICY_ID = 'test-12345';
 
     it('Works and reset one preconfigured policies if the policy is already deleted (with a ghost package policy)', async () => {

x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts

@@ -197,12 +197,17 @@ export default function (providerContext: FtrProviderContext) {
     for (const scenario of scenarios) {
       it(`Should write the correct event.agent_id_status for ${scenario.name}`, async () => {
         // Create an API key
-        const apiKeyRes = await es.security.createApiKey({
-          body: {
-            name: `test api key`,
-            ...(scenario.apiKey || {}),
+        const apiKeyRes = await es.security.createApiKey(
+          {
+            body: {
+              name: `test api key`,
+              ...(scenario.apiKey || {}),
+            },
           },
-        });
+          {
+            headers: { 'es-security-runas-user': 'elastic' }, // run as elastic suer
+          }
+        );
 
         const res = await indexUsingApiKey(
           {

x-pack/test/fleet_api_integration/apis/epm/install_overrides.ts

@@ -63,7 +63,9 @@ export default function (providerContext: FtrProviderContext) {
           method: 'GET',
           path: `/_component_template/${templateName}@mappings`,
         },
-        { meta: true }
+        {
+          meta: true,
+        }
       ));
 
       // The mappings override provided in the package is set in the mappings component template
@@ -128,6 +130,8 @@ export default function (providerContext: FtrProviderContext) {
         },
         { meta: true }
       ));
+      // omit routings
+      delete body.template.settings.index.routing;
 
       expect(body).to.eql({
         template: {

x-pack/test/reporting_api_integration/reporting_and_security/ilm_migration_apis.ts

@@ -13,7 +13,6 @@ import { ILM_POLICY_NAME } from '../../../plugins/reporting/common/constants';
 // eslint-disable-next-line import/no-default-export
 export default function ({ getService }: FtrProviderContext) {
   const es = getService('es');
-  const supertest = getService('supertest');
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const reportingAPI = getService('reportingAPI');
   const security = getService('security');
@@ -28,10 +27,42 @@ export default function ({ getService }: FtrProviderContext) {
     `,index:aac3e500-f2c7-11ea-8250-fb138aa491e7,query:(language:kuery,query:'')` +
     `,version:!t),sort:!((order_date:desc)),trackTotalHits:!t),title:'EC SEARCH from DEFAULT')`;
 
+  const runMigrate = async () => {
+    await reportingAPI.migrateReportingIndices(
+      reportingAPI.REPORTING_USER_USERNAME,
+      reportingAPI.REPORTING_USER_PASSWORD
+    );
+  };
+
   describe('ILM policy migration APIs', () => {
     before(async () => {
+      await security.role.create(reportingAPI.REPORTING_ROLE, {
+        metadata: {},
+        elasticsearch: {
+          cluster: ['manage_ilm'],
+          indices: [
+            { names: ['ecommerce'], privileges: ['read'], allow_restricted_indices: false },
+            { names: ['.reporting-*'], privileges: ['all'], allow_restricted_indices: true },
+          ],
+          run_as: [],
+        },
+        kibana: [
+          {
+            base: [],
+            feature: {
+              dashboard: ['minimal_read', 'download_csv_report', 'generate_report'],
+              discover: ['minimal_read', 'generate_report'],
+              canvas: ['minimal_read', 'generate_report'],
+              visualize: ['minimal_read', 'generate_report'],
+            },
+            spaces: ['*'],
+          },
+        ],
+      });
+      await reportingAPI.createTestReportingUser();
+
       await reportingAPI.initLogs();
-      await reportingAPI.migrateReportingIndices(); // ensure that the ILM policy exists for the first test
+      await runMigrate(); // ensure that the ILM policy exists for the first test
     });
 
     after(async () => {
@@ -40,47 +71,80 @@ export default function ({ getService }: FtrProviderContext) {
 
     afterEach(async () => {
       await reportingAPI.deleteAllReports();
-      await reportingAPI.migrateReportingIndices(); // ensure that the ILM policy exists
+      await runMigrate(); // ensure that the ILM policy exists
     });
 
     it('detects when no migration is needed', async () => {
-      expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('ok');
+      expect(
+        await reportingAPI.checkIlmMigrationStatus(
+          reportingAPI.REPORTING_USER_USERNAME,
+          reportingAPI.REPORTING_USER_PASSWORD
+        )
+      ).to.eql('ok');
 
       // try creating a report
-      await supertest
+      await supertestWithoutAuth
         .post(`/api/reporting/generate/csv_searchsource`)
+        .auth(reportingAPI.REPORTING_USER_USERNAME, reportingAPI.REPORTING_USER_PASSWORD)
         .set('kbn-xsrf', 'xxx')
         .send({ jobParams: JOB_PARAMS_RISON_CSV });
 
-      expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('ok');
+      expect(
+        await reportingAPI.checkIlmMigrationStatus(
+          reportingAPI.REPORTING_USER_USERNAME,
+          reportingAPI.REPORTING_USER_PASSWORD
+        )
+      ).to.eql('ok');
     });
 
     it('detects when reporting indices should be migrated due to missing ILM policy', async () => {
       await reportingAPI.makeAllReportingIndicesUnmanaged();
       await es.ilm.deleteLifecycle({ name: ILM_POLICY_NAME });
 
-      await supertest
+      await supertestWithoutAuth
         .post(`/api/reporting/generate/csv_searchsource`)
+        .auth(reportingAPI.REPORTING_USER_USERNAME, reportingAPI.REPORTING_USER_PASSWORD)
         .set('kbn-xsrf', 'xxx')
         .send({ jobParams: JOB_PARAMS_RISON_CSV });
 
-      expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('policy-not-found');
+      expect(
+        await reportingAPI.checkIlmMigrationStatus(
+          reportingAPI.REPORTING_USER_USERNAME,
+          reportingAPI.REPORTING_USER_PASSWORD
+        )
+      ).to.eql('policy-not-found');
       // assert that migration fixes this
-      await reportingAPI.migrateReportingIndices();
-      expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('ok');
+      await runMigrate();
+      expect(
+        await reportingAPI.checkIlmMigrationStatus(
+          reportingAPI.REPORTING_USER_USERNAME,
+          reportingAPI.REPORTING_USER_PASSWORD
+        )
+      ).to.eql('ok');
     });
 
     it('detects when reporting indices should be migrated due to unmanaged indices', async () => {
       await reportingAPI.makeAllReportingIndicesUnmanaged();
-      await supertest
+      await supertestWithoutAuth
         .post(`/api/reporting/generate/csv_searchsource`)
+        .auth(reportingAPI.REPORTING_USER_USERNAME, reportingAPI.REPORTING_USER_PASSWORD)
         .set('kbn-xsrf', 'xxx')
         .send({ jobParams: JOB_PARAMS_RISON_CSV });
 
-      expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('indices-not-managed-by-policy');
+      expect(
+        await reportingAPI.checkIlmMigrationStatus(
+          reportingAPI.REPORTING_USER_USERNAME,
+          reportingAPI.REPORTING_USER_PASSWORD
+        )
+      ).to.eql('indices-not-managed-by-policy');
       // assert that migration fixes this
-      await reportingAPI.migrateReportingIndices();
-      expect(await reportingAPI.checkIlmMigrationStatus()).to.eql('ok');
+      await runMigrate();
+      expect(
+        await reportingAPI.checkIlmMigrationStatus(
+          reportingAPI.REPORTING_USER_USERNAME,
+          reportingAPI.REPORTING_USER_PASSWORD
+        )
+      ).to.eql('ok');
     });
 
     it('does not override an existing ILM policy', async () => {
@@ -109,7 +173,7 @@ export default function ({ getService }: FtrProviderContext) {
         body: customLifecycle,
       });
 
-      await reportingAPI.migrateReportingIndices();
+      await runMigrate();
 
       const {
         [ILM_POLICY_NAME]: { policy },

x-pack/test/reporting_api_integration/services/scenarios.ts

@@ -37,6 +37,7 @@ export function createScenarios({ getService }: Pick<FtrProviderContext, 'getSer
   const DATA_ANALYST_PASSWORD = 'data_analyst-password';
   const REPORTING_USER_USERNAME = 'reporting_user';
   const REPORTING_USER_PASSWORD = 'reporting_user-password';
+  const REPORTING_ROLE = 'test_reporting_user';
 
   const logTaskManagerHealth = async () => {
     // Check task manager health for analyzing test failures. See https://github.com/elastic/kibana/issues/114946
@@ -90,7 +91,7 @@ export function createScenarios({ getService }: Pick<FtrProviderContext, 'getSer
   };
 
   const createTestReportingUserRole = async () => {
-    await security.role.create('test_reporting_user', {
+    await security.role.create(REPORTING_ROLE, {
       metadata: {},
       elasticsearch: {
         cluster: [],
@@ -127,9 +128,9 @@ export function createScenarios({ getService }: Pick<FtrProviderContext, 'getSer
   };
 
   const createTestReportingUser = async () => {
-    await security.user.create('reporting_user', {
-      password: 'reporting_user-password',
-      roles: ['test_reporting_user'],
+    await security.user.create(REPORTING_USER_USERNAME, {
+      password: REPORTING_USER_PASSWORD,
+      roles: [REPORTING_ROLE],
       full_name: 'Reporting User',
     });
   };
@@ -202,18 +203,29 @@ export function createScenarios({ getService }: Pick<FtrProviderContext, 'getSer
     });
   };
 
-  const checkIlmMigrationStatus = async () => {
+  const checkIlmMigrationStatus = async (username: string, password: string) => {
     log.debug('ReportingAPI.checkIlmMigrationStatus');
-    const { body } = await supertest
+    const { body } = await supertestWithoutAuth
       .get(API_GET_ILM_POLICY_STATUS)
+      .auth(username, password)
       .set('kbn-xsrf', 'xxx')
       .expect(200);
     return body.status;
   };
 
-  const migrateReportingIndices = async () => {
+  const migrateReportingIndices = async (username: string, password: string) => {
     log.debug('ReportingAPI.migrateReportingIndices');
-    await supertest.put(API_MIGRATE_ILM_POLICY_URL).set('kbn-xsrf', 'xxx').expect(200);
+    try {
+      await supertestWithoutAuth
+        .put(API_MIGRATE_ILM_POLICY_URL)
+        .auth(username, password)
+        .set('kbn-xsrf', 'xxx')
+        .expect(200);
+    } catch (err) {
+      log.error(`Could not migrate Reporting indices!`);
+      log.error(err);
+      throw err;
+    }
   };
 
   const makeAllReportingIndicesUnmanaged = async () => {
@@ -239,6 +251,7 @@ export function createScenarios({ getService }: Pick<FtrProviderContext, 'getSer
     DATA_ANALYST_PASSWORD,
     REPORTING_USER_USERNAME,
     REPORTING_USER_PASSWORD,
+    REPORTING_ROLE,
     routes: {
       API_GET_ILM_POLICY_STATUS,
       API_MIGRATE_ILM_POLICY_URL,

x-pack/test/upgrade_assistant_integration/config.js

@@ -5,8 +5,6 @@
  * 2.0.
  */
 
-import path from 'path';
-
 export default async function ({ readConfigFile }) {
   // Read the Kibana API integration tests config file so that we can utilize its services.
   const kibanaAPITestsConfig = await readConfigFile(
@@ -32,7 +30,8 @@ export default async function ({ readConfigFile }) {
     kbnTestServer: xPackFunctionalTestsConfig.get('kbnTestServer'),
     esTestCluster: {
       ...xPackFunctionalTestsConfig.get('esTestCluster'),
-      dataArchive: path.resolve(__dirname, './fixtures/data_archives/upgrade_assistant.zip'),
+      // this archive can not be loaded into 8.0+
+      // dataArchive: path.resolve(__dirname, './fixtures/data_archives/upgrade_assistant.zip'),
     },
   };
 }