[8.16] [Feature Flags] Retry provider setup (#214200) (#214300)

# Backport This will backport the following commits from `main` to `8.16`: - [[Feature Flags] Retry provider setup (#214200)](https://github.com/elastic/kibana/pull/214200)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)
2025-04-24 09:48:58 -04:00 · 2025-03-13 11:30:18 +01:00 · 2025-03-13 11:30:18 +01:00 · a17cb1c163
commit a17cb1c163
parent d648da2ded
5 changed files with 168 additions and 2 deletions
--- a/packages/core/feature-flags/core-feature-flags-server-internal/src/feature_flags_service.test.ts
+++ b/packages/core/feature-flags/core-feature-flags-server-internal/src/feature_flags_service.test.ts
@ -36,7 +36,9 @@ describe('FeatureFlagsService Server', () => {
  });

  afterEach(async () => {
+    jest.useRealTimers();
    await featureFlagsService.stop();
+    jest.spyOn(OpenFeature, 'setProviderAndWait').mockRestore(); // Make sure that we clean up any previous mocked implementations
    jest.clearAllMocks();
    await OpenFeature.clearProviders();
  });
@ -45,7 +47,7 @@ describe('FeatureFlagsService Server', () => {
    test('appends a provider (no async operation)', () => {
      expect.assertions(1);
      const { setProvider } = featureFlagsService.setup();
-      const spy = jest.spyOn(OpenFeature, 'setProvider');
+      const spy = jest.spyOn(OpenFeature, 'setProviderAndWait');
      const fakeProvider = { metadata: { name: 'fake provider' } } as Provider;
      setProvider(fakeProvider);
      expect(spy).toHaveBeenCalledWith(fakeProvider);
--- a/packages/core/feature-flags/core-feature-flags-server-internal/src/feature_flags_service.ts
+++ b/packages/core/feature-flags/core-feature-flags-server-internal/src/feature_flags_service.ts
@ -24,6 +24,7 @@ import {
 } from '@openfeature/server-sdk';
 import deepMerge from 'deepmerge';
 import { filter, switchMap, startWith, Subject } from 'rxjs';
+import { setProviderWithRetries } from './set_provider_with_retries';
 import { type FeatureFlagsConfig, featureFlagsConfig } from './feature_flags_config';

 /**
@ -76,7 +77,7 @@ export class FeatureFlagsService {
        if (OpenFeature.providerMetadata !== NOOP_PROVIDER.metadata) {
          throw new Error('A provider has already been set. This API cannot be called twice.');
        }
-        OpenFeature.setProvider(provider);
+        setProviderWithRetries(provider, this.logger);
      },
      appendContext: (contextToAppend) => this.appendContext(contextToAppend),
    };
--- a/packages/core/feature-flags/core-feature-flags-server-internal/src/set_provider_with_retries.test.ts
+++ b/packages/core/feature-flags/core-feature-flags-server-internal/src/set_provider_with_retries.test.ts
@ -0,0 +1,124 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { OpenFeature, type Provider } from '@openfeature/server-sdk';
+import { setProviderWithRetries } from './set_provider_with_retries';
+import { loggerMock, type MockedLogger } from '@kbn/logging-mocks';
+
+describe('setProviderWithRetries', () => {
+  const fakeProvider = { metadata: { name: 'fake provider' } } as Provider;
+  let logger: MockedLogger;
+
+  beforeEach(() => {
+    jest.useFakeTimers();
+    logger = loggerMock.create();
+  });
+
+  afterEach(() => {
+    jest.clearAllTimers();
+    jest.clearAllMocks();
+    jest.useRealTimers();
+  });
+
+  test('sets the provider and logs the success', async () => {
+    expect.assertions(3);
+    const spy = jest.spyOn(OpenFeature, 'setProviderAndWait');
+
+    setProviderWithRetries(fakeProvider, logger);
+
+    expect(spy).toHaveBeenCalledWith(fakeProvider);
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    await jest.runAllTimersAsync();
+
+    expect(logger.info.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Feature flags provider successfully set up.",
+        ],
+      ]
+    `);
+  });
+
+  test('should retry up to 5 times (and does not throw/reject)', async () => {
+    expect.assertions(15);
+    const spy = jest
+      .spyOn(OpenFeature, 'setProviderAndWait')
+      .mockRejectedValue(new Error('Something went terribly wrong!'));
+
+    setProviderWithRetries(fakeProvider, logger);
+
+    expect(spy).toHaveBeenCalledWith(fakeProvider);
+
+    // Initial attempt
+    expect(spy).toHaveBeenCalledTimes(1);
+
+    // 5 retries
+    for (let i = 0; i < 5; i++) {
+      await jest.advanceTimersByTimeAsync(1000 * Math.pow(2, i)); // exponential backoff of factor 2
+      expect(spy).toHaveBeenCalledTimes(i + 2);
+      expect(logger.warn).toHaveBeenCalledTimes(i + 2);
+    }
+
+    // Given up retrying
+    await jest.advanceTimersByTimeAsync(32000);
+    expect(spy).toHaveBeenCalledTimes(6);
+
+    expect(logger.warn.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 5 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 4 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 3 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 2 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 1 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!. Retrying 0 times more...",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+      ]
+    `);
+    expect(logger.error.mock.calls).toMatchInlineSnapshot(`
+      Array [
+        Array [
+          "Failed to set up the feature flags provider: Something went terribly wrong!",
+          Object {
+            "error": [Error: Something went terribly wrong!],
+          },
+        ],
+      ]
+    `);
+  });
+});
--- a/packages/core/feature-flags/core-feature-flags-server-internal/src/set_provider_with_retries.ts
+++ b/packages/core/feature-flags/core-feature-flags-server-internal/src/set_provider_with_retries.ts
@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { Logger } from '@kbn/logging';
+import { type Provider, OpenFeature } from '@openfeature/server-sdk';
+import pRetry from 'p-retry';
+
+/**
+ * Handles the setting of the Feature Flags provider and any retries that may be required.
+ * This method is intentionally synchronous (no async/await) to avoid holding Kibana's startup on the feature flags setup.
+ * @param provider The OpenFeature provider to set up.
+ * @param logger You know, for logging.
+ */
+export function setProviderWithRetries(provider: Provider, logger: Logger): void {
+  pRetry(() => OpenFeature.setProviderAndWait(provider), {
+    retries: 5,
+    onFailedAttempt: (error) => {
+      logger.warn(
+        `Failed to set up the feature flags provider: ${error.message}. Retrying ${error.retriesLeft} times more...`,
+        { error }
+      );
+    },
+  })
+    .then(() => {
+      logger.info('Feature flags provider successfully set up.');
+    })
+    .catch((error) => {
+      logger.error(`Failed to set up the feature flags provider: ${error.message}`, {
+        error,
+      });
+    });
+}
--- a/packages/core/feature-flags/core-feature-flags-server-internal/tsconfig.json
+++ b/packages/core/feature-flags/core-feature-flags-server-internal/tsconfig.json
@ -20,5 +20,6 @@
    "@kbn/core-base-server-mocks",
    "@kbn/config-schema",
    "@kbn/config-mocks",
+    "@kbn/logging-mocks",
  ]
 }