[Serverless FTRs] Ensure security index exists before running the tests (#169926)

packages/kbn-es/src/utils/docker.test.ts

@@ -40,6 +40,7 @@ import {
   SERVERLESS_JWKS_PATH,
 } from '../paths';
 import * as waitClusterUtil from './wait_until_cluster_ready';
+import * as waitForSecurityIndexUtil from './wait_for_security_index';
 
 jest.mock('execa');
 const execa = jest.requireMock('execa');
@@ -53,6 +54,10 @@ jest.mock('./wait_until_cluster_ready', () => ({
   waitUntilClusterReady: jest.fn(),
 }));
 
+jest.mock('./wait_for_security_index', () => ({
+  waitForSecurityIndex: jest.fn(),
+}));
+
 const log = new ToolingLog();
 const logWriter = new ToolingLogCollectingWriter();
 log.setWriters([logWriter]);
@@ -63,6 +68,7 @@ const serverlessDir = 'stateless';
 const serverlessObjectStorePath = `${baseEsPath}/${serverlessDir}`;
 
 const waitUntilClusterReadyMock = jest.spyOn(waitClusterUtil, 'waitUntilClusterReady');
+const waitForSecurityIndexMock = jest.spyOn(waitForSecurityIndexUtil, 'waitForSecurityIndex');
 
 beforeEach(() => {
   jest.resetAllMocks();
@@ -560,6 +566,32 @@ describe('runServerlessCluster()', () => {
     expect(waitUntilClusterReadyMock.mock.calls[0][0].expectedStatus).toEqual('green');
     expect(waitUntilClusterReadyMock.mock.calls[0][0].readyTimeout).toEqual(undefined);
   });
+
+  test(`should wait for the security index`, async () => {
+    waitForSecurityIndexMock.mockResolvedValue();
+    mockFs({
+      [baseEsPath]: {},
+    });
+    execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
+
+    await runServerlessCluster(log, { basePath: baseEsPath, waitForReady: true });
+    expect(waitForSecurityIndexMock).toHaveBeenCalledTimes(1);
+    expect(waitForSecurityIndexMock.mock.calls[0][0].readyTimeout).toEqual(undefined);
+  });
+
+  test(`should not wait for the security index when security is disabled`, async () => {
+    mockFs({
+      [baseEsPath]: {},
+    });
+    execa.mockImplementation(() => Promise.resolve({ stdout: '' }));
+
+    await runServerlessCluster(log, {
+      basePath: baseEsPath,
+      waitForReady: true,
+      esArgs: ['xpack.security.enabled=false'],
+    });
+    expect(waitForSecurityIndexMock).not.toHaveBeenCalled();
+  });
 });
 
 describe('stopServerlessCluster()', () => {

packages/kbn-es/src/utils/docker.ts

@@ -21,6 +21,7 @@ import {
   kibanaDevServiceAccount,
 } from '@kbn/dev-utils';
 
+import { waitForSecurityIndex } from './wait_for_security_index';
 import { createCliError } from '../errors';
 import { EsClusterExecOptions } from '../cluster_exec_options';
 import {
@@ -687,6 +688,10 @@ export async function runServerlessCluster(log: ToolingLog, options: ServerlessO
         : {}),
     });
     await waitUntilClusterReady({ client, expectedStatus: 'green', log });
+    if (!options.esArgs || !options.esArgs.includes('xpack.security.enabled=false')) {
+      // If security is not disabled, make sure the security index exists before running the test to avoid flakiness
+      await waitForSecurityIndex({ client, log });
+    }
   }
 
   if (!options.background) {

packages/kbn-es/src/utils/wait_for_security_index.test.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { ToolingLog, ToolingLogCollectingWriter } from '@kbn/tooling-log';
+import { waitForSecurityIndex } from './wait_for_security_index';
+
+jest.mock('@elastic/elasticsearch', () => {
+  return {
+    Client: jest.fn(),
+  };
+});
+
+const log = new ToolingLog();
+const logWriter = new ToolingLogCollectingWriter();
+log.setWriters([logWriter]);
+
+const createApiKey = jest.fn();
+const invalidateApiKey = jest.fn();
+
+beforeEach(() => {
+  jest.resetAllMocks();
+  jest
+    .requireMock('@elastic/elasticsearch')
+    .Client.mockImplementation(() => ({ security: { createApiKey, invalidateApiKey } }));
+  log.indent(-log.getIndent());
+  logWriter.messages.length = 0;
+  createApiKey.mockResolvedValue({ id: 'test-id' });
+});
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+describe('waitForSecurityIndex', () => {
+  test(`waits for the security request to succeed`, async () => {
+    invalidateApiKey.mockImplementationOnce(() => Promise.reject(new Error('foo')));
+    invalidateApiKey.mockImplementationOnce(() => Promise.resolve({}));
+
+    const client = new Client({});
+
+    await waitForSecurityIndex({ client, log });
+    expect(invalidateApiKey).toHaveBeenCalledTimes(2);
+    expect(logWriter.messages).toMatchInlineSnapshot(`
+      Array [
+        " info waiting for ES cluster to bootstrap the security index",
+        " warn waiting for ES cluster to bootstrap the security index, attempt 1 failed with: foo",
+        " succ ES security index is ready",
+      ]
+    `);
+  }, 10000);
+
+  test(`rejects when 'readyTimeout' is exceeded`, async () => {
+    invalidateApiKey.mockImplementationOnce(() => Promise.reject(new Error('foo')));
+    invalidateApiKey.mockImplementationOnce(() => Promise.reject(new Error('foo')));
+    const client = new Client({});
+    await expect(waitForSecurityIndex({ client, log, readyTimeout: 1000 })).rejects.toThrow(
+      'ES cluster failed to bootstrap the security index with the 1 second timeout'
+    );
+  });
+});

packages/kbn-es/src/utils/wait_for_security_index.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { ToolingLog } from '@kbn/tooling-log';
+const DEFAULT_READY_TIMEOUT = 10 * 1000; // 10 seconds
+
+export interface WaitOptions {
+  client: Client;
+  log: ToolingLog;
+  readyTimeout?: number;
+}
+
+/**
+ * General method to wait for the ES cluster status to be yellow or green
+ */
+export async function waitForSecurityIndex({
+  client,
+  log,
+  readyTimeout = DEFAULT_READY_TIMEOUT,
+}: WaitOptions) {
+  let attempt = 0;
+  const start = Date.now();
+
+  // The loop will continue until timeout even if SIGINT is signaled, so force exit
+  process.on('SIGINT', () => process.exit());
+
+  log.info(`waiting for ES cluster to bootstrap the security index`);
+
+  // Hack to force the creation of the index `.security-7` index
+  const response = await client.security.createApiKey({ name: 'test-api-key-to-delete' });
+  while (true) {
+    attempt += 1;
+
+    try {
+      await client.security.invalidateApiKey({ ids: [response.id] });
+      log.success('ES security index is ready');
+      return;
+    } catch (error) {
+      const timeSinceStart = Date.now() - start;
+      if (timeSinceStart > readyTimeout) {
+        const sec = readyTimeout / 1000;
+        throw new Error(
+          `ES cluster failed to bootstrap the security index with the ${sec} second timeout`
+        );
+      }
+
+      log.warning(
+        `waiting for ES cluster to bootstrap the security index, attempt ${attempt} failed with: ${error?.message}`
+      );
+
+      const waitSec = attempt * 1.5;
+      await new Promise((resolve) => setTimeout(resolve, waitSec * 1000));
+    }
+  }
+}