[Feature Flags] Retry provider setup (#214200)

## Summary

We identified that on some occasions, the Feature Flags provider times
out when setting up, and, since we don't restart the Kibana server, it
never sets it up.

This PR adds a retry logic to try to set the provider in case there's an
error.

cc @pmuellr as he found out about this bug

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

src/core/packages/feature-flags/server-internal/src/feature_flags_service.test.ts

@@ -38,7 +38,9 @@ describe('FeatureFlagsService Server', () => {
   });
 
   afterEach(async () => {
+    jest.useRealTimers();
     await featureFlagsService.stop();
+    jest.spyOn(OpenFeature, 'setProviderAndWait').mockRestore(); // Make sure that we clean up any previous mocked implementations
     jest.clearAllMocks();
     await OpenFeature.clearProviders();
   });
@@ -47,7 +49,7 @@ describe('FeatureFlagsService Server', () => {
     test('appends a provider (no async operation)', () => {
       expect.assertions(1);
       const { setProvider } = featureFlagsService.setup();
-      const spy = jest.spyOn(OpenFeature, 'setProvider');
+      const spy = jest.spyOn(OpenFeature, 'setProviderAndWait');
       const fakeProvider = { metadata: { name: 'fake provider' } } as Provider;
       setProvider(fakeProvider);
       expect(spy).toHaveBeenCalledWith(fakeProvider);

src/core/packages/feature-flags/server-internal/src/feature_flags_service.ts

@@ -25,6 +25,7 @@ import {
 import deepMerge from 'deepmerge';
 import { filter, switchMap, startWith, Subject } from 'rxjs';
 import { get } from 'lodash';
+import { setProviderWithRetries } from './set_provider_with_retries';
 import { type FeatureFlagsConfig, featureFlagsConfig } from './feature_flags_config';
 
 /**
@@ -77,7 +78,7 @@ export class FeatureFlagsService {
         if (OpenFeature.providerMetadata !== NOOP_PROVIDER.metadata) {
           throw new Error('A provider has already been set. This API cannot be called twice.');
         }
-        OpenFeature.setProvider(provider);
+        setProviderWithRetries(provider, this.logger);
       },
       appendContext: (contextToAppend) => this.appendContext(contextToAppend),
     };

src/core/packages/feature-flags/server-internal/src/set_provider_with_retries.test.ts

@@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OpenFeature, type Provider } from '@openfeature/server-sdk';
+import { setProviderWithRetries } from './set_provider_with_retries';
+import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
+
+describe('setProviderWithRetries', () => {
+  const fakeProvider = { metadata: { name: 'fake provider' } } as Provider;
+  let logger: MockedLogger;
+
+  beforeEach(() => {
+    jest.useFakeTimers();
+    logger = loggerMock.create();
+  });
+
+  afterEach(() => {
+    jest.clearAllTimers();
+    jest.clearAllMocks();
+    jest.useRealTimers();
+  });
+
+  test('sets the provider and logs the success', async () => {
+    expect.assertions(3);
+    const spy = jest.spyOn(OpenFeature, 'setProviderAndWait');
+
+    setProviderWithRetries(fakeProvider, logger);
+
+    expect(spy).toHaveBeenCalledWith(fakeProvider);
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    await jest.runAllTimersAsync();
+
+    expect(logger.info.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Feature flags provider successfully set up.",
+        ],
+      ]
+    `);
+  });
+
+  test('should retry up to 5 times (and does not throw/reject)', async () => {
+    expect.assertions(15);
+    const spy = jest
+      .spyOn(OpenFeature, 'setProviderAndWait')
+      .mockRejectedValue(new Error('Something went terribly wrong!'));
+
+    setProviderWithRetries(fakeProvider, logger);
+
+    expect(spy).toHaveBeenCalledWith(fakeProvider);
+
+    // Initial attempt
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    // 5 retries
+    for (let i = 0; i < 5; i++) {
+      await jest.advanceTimersByTimeAsync(1000 * Math.pow(2, i)); // exponential backoff of factor 2
+      expect(spy).toHaveBeenCalledTimes(i + 2);
+      expect(logger.warn).toHaveBeenCalledTimes(i + 2);
+    }
+
+    // Given up retrying
+    await jest.advanceTimersByTimeAsync(32000);
+    expect(spy).toHaveBeenCalledTimes(6);
+
+    expect(logger.warn.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 5 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 4 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 3 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 2 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 1 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 0 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+      ]
+    `);
+    expect(logger.error.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+      ]
+    `);
+  });
+});

src/core/packages/feature-flags/server-internal/src/set_provider_with_retries.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Logger } from '@kbn/logging';
+import { type Provider, OpenFeature } from '@openfeature/server-sdk';
+import pRetry from 'p-retry';
+
+/**
+ * Handles the setting of the Feature Flags provider and any retries that may be required.
+ * This method is intentionally synchronous (no async/await) to avoid holding Kibana's startup on the feature flags setup.
+ * @param provider The OpenFeature provider to set up.
+ * @param logger You know, for logging.
+ */
+export function setProviderWithRetries(provider: Provider, logger: Logger): void {
+  pRetry(() => OpenFeature.setProviderAndWait(provider), {
+    retries: 5,
+    onFailedAttempt: (error) => {
+      logger.warn(
+        `Failed to set up the feature flags provider: ${error.message}. Retrying ${error.retriesLeft} times more...`,
+        { error }
+      );
+    },
+  })
+    .then(() => {
+      logger.info('Feature flags provider successfully set up.');
+    })
+    .catch((error) => {
+      logger.error(`Failed to set up the feature flags provider: ${error.message}`, {
+        error,
+      });
+    });
+}

src/core/packages/feature-flags/server-internal/tsconfig.json

@@ -20,5 +20,6 @@
     "@kbn/core-base-server-mocks",
     "@kbn/config-schema",
     "@kbn/config-mocks",
+    "@kbn/logging-mocks",
   ]
 }