Set refresh according to stateful vs stateless when indexing alert documents (#201209)

In this PR, I'm making the change so when Kibana is running with Elasticsearch stateful we set refresh to `wait_for` (instead of `true`) so we are not putting too much pressure on the Elasticsearch indices when under load. ## To verify Very using the Cloud deployment and Serverless project created from this PR 1. Create an always firing ES Query rule 2. Create an always firing security detection rule w/ alert suppression 3. Verify the ECH cluster logs and observe `*** Refresh value when indexing alerts: wait_for` and `*** Rule registry - refresh value when indexing alerts: wait_for` messages 4. Verify the serverless project logs on QA overview and observe `*** Refresh value when indexing alerts: true` and `*** Rule registry - refresh value when indexing alerts: true` messages ## To-Do - [x] Revert commit 7c19b458e6 that was added for testing purposes --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
2025-04-23 09:19:04 -04:00 · 2024-11-28 12:10:56 -05:00 · 2024-11-28 12:10:56 -05:00 · a4cb330af2
commit a4cb330af2
parent 9724c46be1
30 changed files with 188 additions and 37 deletions
--- a/x-pack/plugins/alerting/kibana.jsonc
+++ b/x-pack/plugins/alerting/kibana.jsonc
@ -35,12 +35,11 @@
      "usageCollection",
      "security",
      "monitoringCollection",
-      "spaces",
-      "serverless"
+      "spaces"
    ],
    "extraPublicDirs": [
      "common",
      "common/parse_duration"
    ]
  }
-}
+}
--- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts
+++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts
@ -347,6 +347,7 @@ describe('Alerts Client', () => {
          rule: alertRuleData,
          kibanaVersion: '8.9.0',
          spaceId: 'space1',
+          isServerless: false,
          dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts }),
        };
        maintenanceWindowsService.getMaintenanceWindows.mockReturnValue({
@ -543,10 +544,58 @@ describe('Alerts Client', () => {
      });

      describe('persistAlerts()', () => {
-        test('should index new alerts', async () => {
-          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>(
-            alertsClientParams
-          );
+        test('should index new alerts with refresh: wait_for in stateful', async () => {
+          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({
+            ...alertsClientParams,
+            isServerless: false,
+          });
+
+          await alertsClient.initializeExecution(defaultExecutionOpts);
+
+          // Report 2 new alerts
+          const alertExecutorService = alertsClient.factory();
+          alertExecutorService.create('1').scheduleActions('default');
+          alertExecutorService.create('2').scheduleActions('default');
+
+          await alertsClient.processAlerts(processAlertsOpts);
+          alertsClient.logAlerts(logAlertsOpts);
+
+          await alertsClient.persistAlerts();
+
+          const { alertsToReturn } = alertsClient.getAlertsToSerialize();
+          const uuid1 = alertsToReturn['1'].meta?.uuid;
+          const uuid2 = alertsToReturn['2'].meta?.uuid;
+
+          expect(clusterClient.bulk).toHaveBeenCalledWith({
+            index: '.alerts-test.alerts-default',
+            refresh: 'wait_for',
+            require_alias: !useDataStreamForAlerts,
+            body: [
+              {
+                create: { _id: uuid1, ...(useDataStreamForAlerts ? {} : { require_alias: true }) },
+              },
+              // new alert doc
+              getNewIndexedAlertDoc({ [ALERT_UUID]: uuid1 }),
+              {
+                create: { _id: uuid2, ...(useDataStreamForAlerts ? {} : { require_alias: true }) },
+              },
+              // new alert doc
+              getNewIndexedAlertDoc({ [ALERT_UUID]: uuid2, [ALERT_INSTANCE_ID]: '2' }),
+            ],
+          });
+          expect(maintenanceWindowsService.getMaintenanceWindows).toHaveBeenCalledWith({
+            eventLogger: alertingEventLogger,
+            request: fakeRequest,
+            ruleTypeCategory: 'test',
+            spaceId: 'space1',
+          });
+        });
+
+        test('should index new alerts with refresh: true in stateless', async () => {
+          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({
+            ...alertsClientParams,
+            isServerless: true,
+          });

          await alertsClient.initializeExecution(defaultExecutionOpts);

@ -659,7 +708,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -732,7 +781,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -867,7 +916,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -940,7 +989,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -1039,7 +1088,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -1196,7 +1245,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -1314,7 +1363,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -1518,7 +1567,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -1602,6 +1651,7 @@ describe('Alerts Client', () => {
                shouldWrite: false,
              },
            },
+            isServerless: false,
            request: fakeRequest,
            namespace: 'default',
            rule: alertRuleData,
@ -2451,7 +2501,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -2725,7 +2775,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -2826,7 +2876,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
@ -2923,7 +2973,7 @@ describe('Alerts Client', () => {

          expect(clusterClient.bulk).toHaveBeenCalledWith({
            index: '.alerts-test.alerts-default',
-            refresh: true,
+            refresh: 'wait_for',
            require_alias: !useDataStreamForAlerts,
            body: [
              {
--- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts
+++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts
@ -73,6 +73,7 @@ export interface AlertsClientParams extends CreateAlertsClientParams {
  elasticsearchClientPromise: Promise<ElasticsearchClient>;
  kibanaVersion: string;
  dataStreamAdapter: DataStreamAdapter;
+  isServerless: boolean;
 }

 interface AlertsAffectedByMaintenanceWindows {
@ -109,6 +110,7 @@ export class AlertsClient<
  private runTimestampString: string | undefined;
  private rule: AlertRule;
  private ruleType: UntypedNormalizedRuleType;
+  private readonly isServerless: boolean;

  private indexTemplateAndPattern: IIndexPatternString;

@ -143,6 +145,7 @@ export class AlertsClient<
    this._isUsingDataStreams = this.options.dataStreamAdapter.isUsingDataStreams();
    this.ruleInfoMessage = `for ${this.ruleType.id}:${this.options.rule.id} '${this.options.rule.name}'`;
    this.logTags = { tags: [this.ruleType.id, this.options.rule.id, 'alerts-client'] };
+    this.isServerless = options.isServerless;
  }

  public async initializeExecution(opts: InitializeExecutionOpts) {
@ -555,7 +558,9 @@ export class AlertsClient<

      try {
        const response = await esClient.bulk({
-          refresh: true,
+          // On serverless we can force a refresh to we don't wait for the longer refresh interval
+          // When too many refresh calls are done in a short period of time, they are throttled by stateless Elasticsearch
+          refresh: this.isServerless ? true : 'wait_for',
          index: this.indexTemplateAndPattern.alias,
          require_alias: !this.isUsingDataStreams(),
          body: bulkBody,
--- a/x-pack/plugins/alerting/server/alerts_client/lib/initialize_alerts_client.test.ts
+++ b/x-pack/plugins/alerting/server/alerts_client/lib/initialize_alerts_client.test.ts
@ -100,6 +100,7 @@ describe('initializeAlertsClient', () => {
        ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
        ruleRunMetricsStore,
        spaceId: 'default',
+        isServerless: false,
      },
      executionId: 'abc',
      logger,
@ -159,6 +160,7 @@ describe('initializeAlertsClient', () => {
        ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
        ruleRunMetricsStore,
        spaceId: 'default',
+        isServerless: false,
      },
      executionId: 'abc',
      logger,
@ -219,6 +221,7 @@ describe('initializeAlertsClient', () => {
        ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
        ruleRunMetricsStore,
        spaceId: 'default',
+        isServerless: false,
      },
      executionId: 'abc',
      logger,
@ -288,6 +291,7 @@ describe('initializeAlertsClient', () => {
        ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
        ruleRunMetricsStore,
        spaceId: 'default',
+        isServerless: false,
      },
      executionId: 'abc',
      logger,
--- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts
+++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts
@ -275,6 +275,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -308,6 +309,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$: test$,
+            isServerless: false,
          });

          await retryUntil(
@ -350,6 +352,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil('error log called', async () => logger.error.mock.calls.length > 0);
@ -372,6 +375,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil('error log called', async () => logger.error.mock.calls.length > 0);
@ -453,6 +457,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -495,6 +500,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -1516,6 +1522,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: true,
          });

          await retryUntil(
@ -1561,6 +1568,7 @@ describe('Alerts Service', () => {
            maintenanceWindowsService,
            namespace: 'default',
            spaceId: 'default',
+            isServerless: true,
            rule: {
              consumer: 'bar',
              executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28',
@ -1586,6 +1594,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -1629,6 +1638,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -1693,6 +1703,7 @@ describe('Alerts Service', () => {
            maintenanceWindowsService,
            namespace: 'default',
            spaceId: 'default',
+            isServerless: false,
            rule: {
              consumer: 'bar',
              executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28',
@ -1736,6 +1747,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -1824,6 +1836,7 @@ describe('Alerts Service', () => {
            maintenanceWindowsService,
            namespace: 'default',
            spaceId: 'default',
+            isServerless: false,
            rule: {
              consumer: 'bar',
              executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28',
@ -1870,6 +1883,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -1911,6 +1925,7 @@ describe('Alerts Service', () => {
            maintenanceWindowsService,
            namespace: 'default',
            spaceId: 'default',
+            isServerless: false,
            rule: {
              consumer: 'bar',
              executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28',
@ -1960,6 +1975,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -2013,6 +2029,7 @@ describe('Alerts Service', () => {
            maintenanceWindowsService,
            namespace: 'default',
            spaceId: 'default',
+            isServerless: false,
            rule: {
              consumer: 'bar',
              executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28',
@ -2067,6 +2084,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -2134,6 +2152,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -2205,6 +2224,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -2283,6 +2303,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });
          alertsService.register(TestRegistrationContext);

@ -2341,6 +2362,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -2362,6 +2384,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -2383,6 +2406,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -2412,6 +2436,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -2444,6 +2469,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -2481,6 +2507,7 @@ describe('Alerts Service', () => {
            kibanaVersion: '8.8.0',
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil(
@ -2516,6 +2543,7 @@ describe('Alerts Service', () => {
            timeoutMs: 10,
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0);
@ -2533,6 +2561,7 @@ describe('Alerts Service', () => {
            timeoutMs: 10,
            dataStreamAdapter,
            elasticsearchAndSOAvailability$,
+            isServerless: false,
          });

          await retryUntil('debug logger called', async () => logger.debug.mock.calls.length > 0);
--- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts
+++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts
@ -59,6 +59,7 @@ interface AlertsServiceParams {
  timeoutMs?: number;
  dataStreamAdapter: DataStreamAdapter;
  elasticsearchAndSOAvailability$: Observable<boolean>;
+  isServerless: boolean;
 }

 export interface CreateAlertsClientParams extends LegacyAlertsClientParams {
@ -120,6 +121,7 @@ export type PublicFrameworkAlertsService = PublicAlertsService & {

 export class AlertsService implements IAlertsService {
  private initialized: boolean;
+  private isServerless: boolean;
  private isInitializing: boolean = false;
  private resourceInitializationHelper: ResourceInstallationHelper;
  private registeredContexts: Map<string, IRuleTypeAlerts> = new Map();
@ -129,6 +131,7 @@ export class AlertsService implements IAlertsService {
  constructor(private readonly options: AlertsServiceParams) {
    this.initialized = false;

+    this.isServerless = options.isServerless;
    this.dataStreamAdapter = options.dataStreamAdapter;

    // Kick off initialization of common assets and save the promise
@ -245,6 +248,7 @@ export class AlertsService implements IAlertsService {
      spaceId: opts.spaceId,
      kibanaVersion: this.options.kibanaVersion,
      dataStreamAdapter: this.dataStreamAdapter,
+      isServerless: this.isServerless,
    });
  }

--- a/x-pack/plugins/alerting/server/plugin.ts
+++ b/x-pack/plugins/alerting/server/plugin.ts
@ -61,7 +61,6 @@ import type { PluginSetup as UnifiedSearchServerPluginSetup } from '@kbn/unified
 import { PluginStart as DataPluginStart } from '@kbn/data-plugin/server';
 import { MonitoringCollectionSetup } from '@kbn/monitoring-collection-plugin/server';
 import { SharePluginStart } from '@kbn/share-plugin/server';
-import { ServerlessPluginSetup } from '@kbn/serverless/server';

 import { RuleTypeRegistry } from './rule_type_registry';
 import { TaskRunnerFactory } from './task_runner';
@ -198,7 +197,6 @@ export interface AlertingPluginsSetup {
  data: DataPluginSetup;
  features: FeaturesPluginSetup;
  unifiedSearch: UnifiedSearchServerPluginSetup;
-  serverless?: ServerlessPluginSetup;
 }

 export interface AlertingPluginsStart {
@ -213,7 +211,6 @@ export interface AlertingPluginsStart {
  data: DataPluginStart;
  dataViews: DataViewsPluginStart;
  share: SharePluginStart;
-  serverless?: ServerlessPluginSetup;
 }

 export class AlertingPlugin {
@ -239,6 +236,7 @@ export class AlertingPlugin {
  private pluginStop$: Subject<void>;
  private dataStreamAdapter?: DataStreamAdapter;
  private backfillClient?: BackfillClient;
+  private readonly isServerless: boolean;
  private nodeRoles: PluginInitializerContext['node']['roles'];
  private readonly connectorAdapterRegistry = new ConnectorAdapterRegistry();

@ -256,6 +254,7 @@ export class AlertingPlugin {
    this.kibanaVersion = initializerContext.env.packageInfo.version;
    this.inMemoryMetrics = new InMemoryMetrics(initializerContext.logger.get('in_memory_metrics'));
    this.pluginStop$ = new ReplaySubject(1);
+    this.isServerless = initializerContext.env.packageInfo.buildFlavor === 'serverless';
  }

  public setup(
@ -268,7 +267,7 @@ export class AlertingPlugin {

    const elasticsearchAndSOAvailability$ = getElasticsearchAndSOAvailability(core.status.core$);

-    const useDataStreamForAlerts = !!plugins.serverless;
+    const useDataStreamForAlerts = this.isServerless;
    this.dataStreamAdapter = getDataStreamAdapter({ useDataStreamForAlerts });

    core.capabilities.registerProvider(() => {
@ -282,7 +281,7 @@ export class AlertingPlugin {
      };
    });

-    plugins.features.registerKibanaFeature(getRulesSettingsFeature(!!plugins.serverless));
+    plugins.features.registerKibanaFeature(getRulesSettingsFeature(this.isServerless));

    plugins.features.registerKibanaFeature(maintenanceWindowFeature);

@ -330,6 +329,7 @@ export class AlertingPlugin {
            .getStartServices()
            .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser),
          elasticsearchAndSOAvailability$,
+          isServerless: this.isServerless,
        });
      }
    }
@ -410,7 +410,7 @@ export class AlertingPlugin {
      getAlertIndicesAlias: createGetAlertIndicesAliasFn(this.ruleTypeRegistry!),
      encryptedSavedObjects: plugins.encryptedSavedObjects,
      config$: plugins.unifiedSearch.autocomplete.getInitializerContextConfig().create(),
-      isServerless: !!plugins.serverless,
+      isServerless: this.isServerless,
      docLinks: core.docLinks,
    });

@ -562,7 +562,7 @@ export class AlertingPlugin {
      logger: this.logger,
      savedObjectsService: core.savedObjects,
      securityService: core.security,
-      isServerless: !!plugins.serverless,
+      isServerless: this.isServerless,
    });

    maintenanceWindowClientFactory.initialize({
@ -620,7 +620,7 @@ export class AlertingPlugin {
      rulesSettingsService: new RulesSettingsService({
        cacheInterval: this.config.rulesSettings.cacheInterval,
        getRulesSettingsClientWithRequest,
-        isServerless: !!plugins.serverless,
+        isServerless: this.isServerless,
        logger,
      }),
      savedObjects: core.savedObjects,
@ -629,6 +629,7 @@ export class AlertingPlugin {
      supportsEphemeralTasks: plugins.taskManager.supportsEphemeralTasks(),
      uiSettings: core.uiSettings,
      usageCounter: this.usageCounter,
+      isServerless: this.isServerless,
    });

    this.eventLogService!.registerSavedObjectProvider(RULE_SAVED_OBJECT_TYPE, (request) => {
--- a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.test.ts
@ -131,6 +131,7 @@ const alertsService = new AlertsService({
  elasticsearchClientPromise: Promise.resolve(clusterClient),
  dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts }),
  elasticsearchAndSOAvailability$,
+  isServerless: false,
 });
 const backfillClient = backfillClientMock.create();
 const dataPlugin = dataPluginMock.createStartContract();
@ -176,6 +177,7 @@ const taskRunnerFactoryInitializerParams: TaskRunnerFactoryInitializerParamsType
  supportsEphemeralTasks: false,
  uiSettings: uiSettingsService,
  usageCounter: mockUsageCounter,
+  isServerless: false,
 };

 const mockedTaskInstance: ConcreteTaskInstance = {
@ -459,7 +461,7 @@ describe('Ad Hoc Task Runner', () => {

    expect(clusterClient.bulk).toHaveBeenCalledWith({
      index: '.alerts-test.alerts-default',
-      refresh: true,
+      refresh: 'wait_for',
      require_alias: !useDataStreamForAlerts,
      body: [
        {
--- a/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts
+++ b/x-pack/plugins/alerting/server/task_runner/ad_hoc_task_runner.ts
@ -185,6 +185,7 @@ export class AdHocTaskRunner implements CancellableTask {
      ruleLogPrefix: ruleLabel,
      ruleRunMetricsStore,
      spaceId: adHocRunData.spaceId,
+      isServerless: this.context.isServerless,
    };
    const alertsClient = await initializeAlertsClient<
      RuleTypeParams,
--- a/x-pack/plugins/alerting/server/task_runner/rule_type_runner.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/rule_type_runner.test.ts
@ -189,6 +189,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -231,6 +232,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -295,6 +297,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -337,6 +340,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: true,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -404,6 +408,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -465,6 +470,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -507,6 +513,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -567,6 +574,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -609,6 +617,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -669,6 +678,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -706,6 +716,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -748,6 +759,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -819,6 +831,7 @@ describe('RuleTypeRunner', () => {
          ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
          ruleRunMetricsStore,
          spaceId: 'default',
+          isServerless: false,
        },
        alertsClient,
        executionId: 'abc',
@ -861,6 +874,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -932,6 +946,7 @@ describe('RuleTypeRunner', () => {
            ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
            ruleRunMetricsStore,
            spaceId: 'default',
+            isServerless: false,
          },
          alertsClient,
          executionId: 'abc',
@ -975,6 +990,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -1036,6 +1052,7 @@ describe('RuleTypeRunner', () => {
            ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
            ruleRunMetricsStore,
            spaceId: 'default',
+            isServerless: false,
          },
          alertsClient,
          executionId: 'abc',
@ -1079,6 +1096,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
@ -1140,6 +1158,7 @@ describe('RuleTypeRunner', () => {
            ruleLogPrefix: `${RULE_TYPE_ID}:${RULE_ID}: '${RULE_NAME}'`,
            ruleRunMetricsStore,
            spaceId: 'default',
+            isServerless: false,
          },
          alertsClient,
          executionId: 'abc',
@ -1183,6 +1202,7 @@ describe('RuleTypeRunner', () => {
        startedAtOverridden: false,
        previousStartedAt: null,
        spaceId: 'default',
+        isServerless: false,
        rule: {
          id: RULE_ID,
          name: mockedRule.name,
--- a/x-pack/plugins/alerting/server/task_runner/rule_type_runner.ts
+++ b/x-pack/plugins/alerting/server/task_runner/rule_type_runner.ts
@ -285,6 +285,7 @@ export class RuleTypeRunner<
                    ...(context.queryDelaySec ? { queryDelay: context.queryDelaySec } : {}),
                    ...(startedAtOverridden ? { forceNow: startedAt } : {}),
                  }),
+                isServerless: context.isServerless,
              })
            )
          );
--- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts
@ -196,6 +196,7 @@ describe('Task Runner', () => {
    supportsEphemeralTasks: false,
    uiSettings: uiSettingsService,
    usageCounter: mockUsageCounter,
+    isServerless: false,
  };

  const ephemeralTestParams: Array<
--- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts
@ -317,6 +317,7 @@ export class TaskRunner<
      ruleLogPrefix: ruleLabel,
      ruleRunMetricsStore,
      spaceId,
+      isServerless: this.context.isServerless,
    };
    const alertsClient = await withAlertingSpan('alerting:initialize-alerts-client', () =>
      initializeAlertsClient<
--- a/x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts
@ -225,6 +225,7 @@ describe('Task Runner', () => {
      supportsEphemeralTasks: false,
      uiSettings: uiSettingsService,
      usageCounter: mockUsageCounter,
+      isServerless: false,
    };

    describe(`using ${label} for alert indices`, () => {
@ -412,6 +413,7 @@ describe('Task Runner', () => {
          elasticsearchClientPromise: Promise.resolve(clusterClient),
          dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts }),
          elasticsearchAndSOAvailability$,
+          isServerless: false,
        });
        elasticsearchAndSOAvailability$.next(true);

@ -544,6 +546,7 @@ describe('Task Runner', () => {
          elasticsearchClientPromise: Promise.resolve(clusterClient),
          dataStreamAdapter: getDataStreamAdapter({ useDataStreamForAlerts }),
          elasticsearchAndSOAvailability$,
+          isServerless: false,
        });
        elasticsearchAndSOAvailability$.next(true);

@ -590,7 +593,7 @@ describe('Task Runner', () => {

        expect(clusterClient.bulk).toHaveBeenCalledWith({
          index: '.alerts-test.alerts-default',
-          refresh: true,
+          refresh: 'wait_for',
          require_alias: !useDataStreamForAlerts,
          body: [
            {
--- a/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner_cancel.test.ts
@ -158,6 +158,7 @@ describe('Task Runner Cancel', () => {
    supportsEphemeralTasks: false,
    uiSettings: uiSettingsService,
    usageCounter: mockUsageCounter,
+    isServerless: false,
  };

  beforeEach(() => {
--- a/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts
+++ b/x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts
@ -129,6 +129,7 @@ describe('Task Runner Factory', () => {
    supportsEphemeralTasks: true,
    uiSettings: uiSettingsService,
    usageCounter: mockUsageCounter,
+    isServerless: false,
  };

  beforeEach(() => {
--- a/x-pack/plugins/alerting/server/task_runner/types.ts
+++ b/x-pack/plugins/alerting/server/task_runner/types.ts
@ -146,6 +146,7 @@ export interface RuleTypeRunnerContext {
  ruleLogPrefix: string;
  ruleRunMetricsStore: RuleRunMetricsStore;
  spaceId: string;
+  isServerless: boolean;
 }

 export interface RuleRunnerErrorStackTraceLog {
@ -180,4 +181,5 @@ export interface TaskRunnerContext {
  supportsEphemeralTasks: boolean;
  uiSettings: UiSettingsServiceStart;
  usageCounter?: UsageCounter;
+  isServerless: boolean;
 }
--- a/x-pack/plugins/alerting/server/types.ts
+++ b/x-pack/plugins/alerting/server/types.ts
@ -133,6 +133,7 @@ export interface RuleExecutorOptions<
  namespace?: string;
  flappingSettings: RulesSettingsFlappingProperties;
  getTimeRange: (timeWindow?: string) => GetTimeRangeResult;
+  isServerless: boolean;
 }

 export interface RuleParamsAndRefs<Params extends RuleTypeParams> {
--- a/x-pack/plugins/observability_solution/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts
+++ b/x-pack/plugins/observability_solution/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts
@ -102,6 +102,7 @@ const mockOptions = {
    const date = STARTED_AT_MOCK_DATE.toISOString();
    return { dateStart: date, dateEnd: date };
  },
+  isServerless: false,
 };

 const setEvaluationResults = (response: Array<Record<string, Evaluation>>) => {
--- a/x-pack/plugins/observability_solution/observability/server/lib/rules/custom_threshold/custom_threshold_executor.test.ts
+++ b/x-pack/plugins/observability_solution/observability/server/lib/rules/custom_threshold/custom_threshold_executor.test.ts
@ -120,6 +120,7 @@ const mockOptions = {
    const date = STARTED_AT_MOCK_DATE.toISOString();
    return { dateStart: date, dateEnd: date };
  },
+  isServerless: false,
 };

 const setEvaluationResults = (response: Array<Record<string, Evaluation>>) => {
--- a/x-pack/plugins/observability_solution/slo/server/lib/rules/slo_burn_rate/executor.test.ts
+++ b/x-pack/plugins/observability_solution/slo/server/lib/rules/slo_burn_rate/executor.test.ts
@ -204,6 +204,7 @@ describe('BurnRateRuleExecutor', () => {
          state: {},
          flappingSettings: DEFAULT_FLAPPING_SETTINGS,
          getTimeRange,
+          isServerless: false,
        })
      ).rejects.toThrowError();
    });
@ -226,6 +227,7 @@ describe('BurnRateRuleExecutor', () => {
        state: {},
        flappingSettings: DEFAULT_FLAPPING_SETTINGS,
        getTimeRange,
+        isServerless: false,
      });

      expect(esClientMock.search).not.toHaveBeenCalled();
@ -276,6 +278,7 @@ describe('BurnRateRuleExecutor', () => {
        state: {},
        flappingSettings: DEFAULT_FLAPPING_SETTINGS,
        getTimeRange,
+        isServerless: false,
      });

      expect(servicesMock.alertsClient?.report).not.toBeCalled();
@ -323,6 +326,7 @@ describe('BurnRateRuleExecutor', () => {
        state: {},
        flappingSettings: DEFAULT_FLAPPING_SETTINGS,
        getTimeRange,
+        isServerless: false,
      });

      expect(servicesMock.alertsClient?.report).not.toBeCalled();
@ -382,6 +386,7 @@ describe('BurnRateRuleExecutor', () => {
        state: {},
        flappingSettings: DEFAULT_FLAPPING_SETTINGS,
        getTimeRange,
+        isServerless: false,
      });

      expect(servicesMock.alertsClient?.report).toBeCalledWith({
@ -531,6 +536,7 @@ describe('BurnRateRuleExecutor', () => {
        state: {},
        flappingSettings: DEFAULT_FLAPPING_SETTINGS,
        getTimeRange,
+        isServerless: false,
      });

      expect(servicesMock.alertsClient?.report).toBeCalledWith({
@ -651,6 +657,7 @@ describe('BurnRateRuleExecutor', () => {
        state: {},
        flappingSettings: DEFAULT_FLAPPING_SETTINGS,
        getTimeRange,
+        isServerless: false,
      });

      expect(servicesMock.alertsClient!.report).toBeCalledWith({
--- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts
+++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts
@ -570,7 +570,9 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper

                const bulkResponse = await ruleDataClientWriter.bulk({
                  body: [...duplicateAlertUpdates, ...mapAlertsToBulkCreate(augmentedAlerts)],
-                  refresh: true,
+                  // On serverless we can force a refresh to we don't wait for the longer refresh interval
+                  // When too many refresh calls are done in a short period of time, they are throttled by stateless Elasticsearch
+                  refresh: options.isServerless ? true : 'wait_for',
                });

                if (bulkResponse == null) {
--- a/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts
+++ b/x-pack/plugins/rule_registry/server/utils/rule_executor.test_helpers.ts
@ -99,4 +99,5 @@ export const createDefaultAlertExecutorOptions = <
    const date = new Date(Date.now()).toISOString();
    return { dateStart: date, dateEnd: date };
  },
+  isServerless: false,
 });
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions_legacy/logic/notifications/legacy_rules_notification_rule_type.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions_legacy/logic/notifications/legacy_rules_notification_rule_type.test.ts
@ -191,6 +191,7 @@ describe('legacyRules_notification_rule_type', () => {
        const date = new Date('2019-12-14T16:40:33.400Z').toISOString();
        return { dateStart: date, dateEnd: date };
      },
+      isServerless: false,
    };

    rule = legacyRulesNotificationRuleType({
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts
@ -83,7 +83,8 @@ export const previewRulesRoute = (
  securityRuleTypeOptions: CreateSecurityRuleTypeWrapperProps,
  previewRuleDataClient: IRuleDataClient,
  getStartServices: StartServicesAccessor<StartPlugins>,
-  logger: Logger
+  logger: Logger,
+  isServerless: boolean
 ) => {
  router.versioned
    .post({
@ -320,6 +321,7 @@ export const previewRulesRoute = (
                  const date = startedAt.toISOString();
                  return { dateStart: date, dateEnd: date };
                },
+                isServerless,
              })) as { state: TState; loggedRequests: RulePreviewLoggedRequest[] });

              const errors = loggedStatusChanges
--- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/register_routes.ts
+++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/register_routes.ts
@ -24,7 +24,8 @@ export const registerRulePreviewRoutes = (
  securityRuleTypeOptions: CreateSecurityRuleTypeWrapperProps,
  previewRuleDataClient: IRuleDataClient,
  getStartServices: StartServicesAccessor<StartPlugins>,
-  logger: Logger
+  logger: Logger,
+  isServerless: boolean
 ) => {
  previewRulesRoute(
    router,
@ -35,6 +36,7 @@ export const registerRulePreviewRoutes = (
    securityRuleTypeOptions,
    previewRuleDataClient,
    getStartServices,
-    logger
+    logger,
+    isServerless
  );
 };
--- a/x-pack/plugins/security_solution/server/plugin.ts
+++ b/x-pack/plugins/security_solution/server/plugin.ts
@ -395,7 +395,8 @@ export class Plugin implements ISecuritySolutionPlugin {
      core.getStartServices,
      securityRuleTypeOptions,
      previewRuleDataClient,
-      this.telemetryReceiver
+      this.telemetryReceiver,
+      this.pluginContext.env.packageInfo.buildFlavor === 'serverless'
    );

    registerEndpointRoutes(router, this.endpointContext);
--- a/x-pack/plugins/security_solution/server/routes/index.ts
+++ b/x-pack/plugins/security_solution/server/routes/index.ts
@ -77,7 +77,8 @@ export const initRoutes = (
  getStartServices: StartServicesAccessor<StartPlugins>,
  securityRuleTypeOptions: CreateSecurityRuleTypeWrapperProps,
  previewRuleDataClient: IRuleDataClient,
-  previewTelemetryReceiver: ITelemetryReceiver
+  previewTelemetryReceiver: ITelemetryReceiver,
+  isServerless: boolean
 ) => {
  registerFleetIntegrationsRoutes(router);
  registerLegacyRuleActionsRoutes(router, logger);
@ -95,7 +96,8 @@ export const initRoutes = (
    securityRuleTypeOptions,
    previewRuleDataClient,
    getStartServices,
-    logger
+    logger,
+    isServerless
  );

  registerResolverRoutes(router, getStartServices, config);
--- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts
+++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.test.ts
@ -957,5 +957,6 @@ async function invokeExecutor({
      const date = new Date(Date.now()).toISOString();
      return { dateStart: date, dateEnd: date };
    },
+    isServerless: false,
  });
 }
--- a/x-pack/plugins/stack_alerts/server/rule_types/index_threshold/rule_type.test.ts
+++ b/x-pack/plugins/stack_alerts/server/rule_types/index_threshold/rule_type.test.ts
@ -231,6 +231,7 @@ describe('ruleType', () => {
      logger,
      flappingSettings: DEFAULT_FLAPPING_SETTINGS,
      getTimeRange,
+      isServerless: false,
    });

    expect(alertServices.alertsClient.report).toHaveBeenCalledWith({
@ -327,6 +328,7 @@ describe('ruleType', () => {
      logger,
      flappingSettings: DEFAULT_FLAPPING_SETTINGS,
      getTimeRange,
+      isServerless: false,
    });

    expect(customAlertServices.alertFactory.create).not.toHaveBeenCalled();
@ -397,6 +399,7 @@ describe('ruleType', () => {
      logger,
      flappingSettings: DEFAULT_FLAPPING_SETTINGS,
      getTimeRange,
+      isServerless: false,
    });

    expect(customAlertServices.alertFactory.create).not.toHaveBeenCalled();
@ -466,6 +469,7 @@ describe('ruleType', () => {
      logger,
      flappingSettings: DEFAULT_FLAPPING_SETTINGS,
      getTimeRange,
+      isServerless: false,
    });

    expect(data.timeSeriesQuery).toHaveBeenCalledWith(