Detect rule param changes for rolling upgrades and rollback assessment (#173936)

In this PR, I'm adding a test in the alerting framework to detect
changes in a rule type's params schema that will require a snapshot to
be updated. This snapshot will provide a centralized place to view
history on alerting rule params in case we need to asses risk for
rolling upgrades or rollbacks of a release (serverless). The only rule
types affected are those running in serverless in any of the three
project types.

When a rule type is used in serverless, it must provide one of the
following configuration to their rule type on top of everything else:

```
// Zod schema
schemas: {
  params: {
    type: 'zod',
    schema: UnifiedQueryRuleParams
  },
},

// config-schema
schemas: {
  params: {
    type: 'config-schema',
    schema: EsQueryRuleParamsSchema,
  },
},
```

We are working on documenting guidelines so engineers and response ops
can ensure a change to rule parameters will work properly in rolling
upgrade and rollback scenarios and be part of the PR review process.

NOTE to rule type owners: I pass the same schema used to validate to the
`schemas.params` attribute in the rule type. It will be important to
keep them in sync. Down the road, we plan to make `validate.params`
optional and use the schema as a starting point so it's easier to have a
single variable passed in.

## To verify
1. Make changes to the params schema of the ES query rule type.
```
diff --git a/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type_params.ts b/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type_params.ts
index 73e8eae32cf..09ec74104ec 100644
--- a/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type_params.ts
+++ b/x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type_params.ts
@@ -39,6 +39,7 @@ export type EsQueryRuleParamsExtractedParams = Omit<EsQueryRuleParams, 'searchCo
 };

 const EsQueryRuleParamsSchemaProperties = {
+  foo: schema.boolean(),
   size: schema.number({ min: 0, max: ES_QUERY_MAX_HITS_PER_EXECUTION }),
   timeWindowSize: schema.number({ min: 1 }),
   excludeHitsFromPreviousRun: schema.boolean({ defaultValue: true }),
```
2. Run the jest integration test to update the snapshot file
```
node scripts/jest_integration.js x-pack/plugins/alerting/server/integration_tests/serverless_upgrade_and_rollback_checks.test.ts -u
```
3. Notice the
`x-pack/plugins/alerting/server/integration_tests/__snapshots__/serverless_upgrade_and_rollback_checks.test.ts.snap`
file got updated
```
    "foo": Object {
      "flags": Object {
        "error": [Function],
      },
      "type": "boolean",
    },
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

package.json

@@ -1683,7 +1683,8 @@
     "xml-crypto": "^5.0.0",
     "xmlbuilder": "13.0.2",
     "yargs": "^15.4.1",
-    "yarn-deduplicate": "^6.0.2"
+    "yarn-deduplicate": "^6.0.2",
+    "zod-to-json-schema": "^3.22.3"
   },
   "packageManager": "yarn@1.22.21"
 }

x-pack/plugins/alerting/server/integration_tests/lib/setup_test_servers.ts

@@ -5,7 +5,6 @@
  * 2.0.
  */
 
-import deepmerge from 'deepmerge';
 import { createTestServers, createRootWithCorePlugins } from '@kbn/core-test-helpers-kbn-server';
 
 export async function setupTestServers(settings = {}) {
@@ -20,25 +19,7 @@ export async function setupTestServers(settings = {}) {
 
   const esServer = await startES();
 
-  const root = createRootWithCorePlugins(
-    deepmerge(
-      {
-        logging: {
-          root: {
-            level: 'warn',
-          },
-          loggers: [
-            {
-              name: 'plugins.taskManager',
-              level: 'all',
-            },
-          ],
-        },
-      },
-      settings
-    ),
-    { oss: false }
-  );
+  const root = createRootWithCorePlugins(settings, { oss: false });
 
   await root.preboot();
   const coreSetup = await root.setup();

x-pack/plugins/alerting/server/integration_tests/serverless_upgrade_and_rollback_checks.test.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  type TestElasticsearchUtils,
+  type TestKibanaUtils,
+} from '@kbn/core-test-helpers-kbn-server';
+import { uniq } from 'lodash';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { setupTestServers } from './lib';
+import type { RuleTypeRegistry } from '../rule_type_registry';
+
+jest.mock('../rule_type_registry', () => {
+  const actual = jest.requireActual('../rule_type_registry');
+  return {
+    ...actual,
+    RuleTypeRegistry: jest.fn().mockImplementation((opts) => {
+      return new actual.RuleTypeRegistry(opts);
+    }),
+  };
+});
+
+/**
+ * These rule types are manually updated.
+ *
+ * TODO: We should spin up three serverless projects and pull the rule types
+ * directly from them to ensure the list below remains up to date. We will still
+ * need a copied list of rule types here because they are needed to write
+ * test scenarios below.
+ */
+const ruleTypesInEsProjects: string[] = [
+  '.index-threshold',
+  '.geo-containment',
+  '.es-query',
+  'transform_health',
+];
+const ruleTypesInObltProjects: string[] = [
+  '.index-threshold',
+  '.geo-containment',
+  '.es-query',
+  'transform_health',
+  'xpack.ml.anomaly_detection_alert',
+  'xpack.ml.anomaly_detection_jobs_health',
+  'slo.rules.burnRate',
+  'observability.rules.custom_threshold',
+  'metrics.alert.inventory.threshold',
+  'apm.error_rate',
+  'apm.transaction_error_rate',
+  'apm.transaction_duration',
+  'apm.anomaly',
+];
+const ruleTypesInSecurityProjects: string[] = [
+  '.index-threshold',
+  '.geo-containment',
+  '.es-query',
+  'transform_health',
+  'xpack.ml.anomaly_detection_alert',
+  'xpack.ml.anomaly_detection_jobs_health',
+  'siem.notifications',
+  'siem.eqlRule',
+  'siem.indicatorRule',
+  'siem.mlRule',
+  'siem.queryRule',
+  'siem.savedQueryRule',
+  'siem.thresholdRule',
+  'siem.newTermsRule',
+];
+
+describe('Serverless upgrade and rollback checks', () => {
+  let esServer: TestElasticsearchUtils;
+  let kibanaServer: TestKibanaUtils;
+  let ruleTypeRegistry: RuleTypeRegistry;
+  const ruleTypesToCheck: string[] = uniq(
+    ruleTypesInEsProjects.concat(ruleTypesInObltProjects).concat(ruleTypesInSecurityProjects)
+  );
+
+  beforeAll(async () => {
+    const setupResult = await setupTestServers();
+    esServer = setupResult.esServer;
+    kibanaServer = setupResult.kibanaServer;
+
+    const mockedRuleTypeRegistry = jest.requireMock('../rule_type_registry');
+    expect(mockedRuleTypeRegistry.RuleTypeRegistry).toHaveBeenCalledTimes(1);
+    ruleTypeRegistry = mockedRuleTypeRegistry.RuleTypeRegistry.mock.results[0].value;
+  });
+
+  afterAll(async () => {
+    if (kibanaServer) {
+      await kibanaServer.stop();
+    }
+    if (esServer) {
+      await esServer.stop();
+    }
+  });
+
+  for (const ruleTypeId of ruleTypesToCheck) {
+    test(`detect param changes to review for: ${ruleTypeId}`, async () => {
+      const ruleType = ruleTypeRegistry.get(ruleTypeId);
+      if (!ruleType?.schemas?.params) {
+        throw new Error('schema.params is required for rule type:' + ruleTypeId);
+      }
+      const schemaType = ruleType.schemas.params.type;
+      if (schemaType === 'config-schema') {
+        // @ts-ignore-next-line getSchema() exists..
+        expect(ruleType.schemas.params.schema.getSchema().describe()).toMatchSnapshot();
+      } else if (schemaType === 'zod') {
+        expect(zodToJsonSchema(ruleType.schemas.params.schema)).toMatchSnapshot();
+      } else {
+        throw new Error(`Support for ${schemaType} missing`);
+      }
+    });
+  }
+});

x-pack/plugins/alerting/server/types.ts

@@ -11,6 +11,7 @@ import type {
   SavedObjectReference,
   IUiSettingsClient,
 } from '@kbn/core/server';
+import z from 'zod';
 import { DataViewsContract } from '@kbn/data-views-plugin/common';
 import { ISearchStartSearchSource } from '@kbn/data-plugin/common';
 import { LicenseType } from '@kbn/licensing-plugin/server';
@@ -20,6 +21,7 @@ import {
   SavedObjectsClientContract,
   Logger,
 } from '@kbn/core/server';
+import type { ObjectType } from '@kbn/config-schema';
 import type { PublicMethodsOf } from '@kbn/utility-types';
 import { SharePluginStart } from '@kbn/share-plugin/server';
 import type { DefaultAlert, FieldMap } from '@kbn/alerts-as-data-utils';
@@ -287,6 +289,17 @@ export interface RuleType<
   validate: {
     params: RuleTypeParamsValidator<Params>;
   };
+  schemas?: {
+    params?:
+      | {
+          type: 'zod';
+          schema: z.ZodObject<z.ZodRawShape> | z.ZodIntersection<z.ZodTypeAny, z.ZodTypeAny>;
+        }
+      | {
+          type: 'config-schema';
+          schema: ObjectType;
+        };
+  };
   actionGroups: Array<ActionGroup<ActionGroupIds>>;
   defaultActionGroupId: ActionGroup<ActionGroupIds>['id'];
   recoveryActionGroup?: ActionGroup<RecoveryActionGroupId>;

x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts

@@ -82,6 +82,12 @@ export function registerAnomalyRuleType({
       actionGroups: ruleTypeConfig.actionGroups,
       defaultActionGroupId: ruleTypeConfig.defaultActionGroupId,
       validate: { params: anomalyParamsSchema },
+      schemas: {
+        params: {
+          type: 'config-schema',
+          schema: anomalyParamsSchema,
+        },
+      },
       actionVariables: {
         context: [
           apmActionVariables.alertDetailsUrl,

x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts

@@ -92,6 +92,12 @@ export function registerErrorCountRuleType({
       actionGroups: ruleTypeConfig.actionGroups,
       defaultActionGroupId: ruleTypeConfig.defaultActionGroupId,
       validate: { params: errorCountParamsSchema },
+      schemas: {
+        params: {
+          type: 'config-schema',
+          schema: errorCountParamsSchema,
+        },
+      },
       actionVariables: {
         context: errorCountActionVariables,
       },

x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts

@@ -104,6 +104,12 @@ export function registerTransactionDurationRuleType({
     actionGroups: ruleTypeConfig.actionGroups,
     defaultActionGroupId: ruleTypeConfig.defaultActionGroupId,
     validate: { params: transactionDurationParamsSchema },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: transactionDurationParamsSchema,
+      },
+    },
     actionVariables: {
       context: transactionDurationActionVariables,
     },

x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts

@@ -101,6 +101,12 @@ export function registerTransactionErrorRateRuleType({
       actionGroups: ruleTypeConfig.actionGroups,
       defaultActionGroupId: ruleTypeConfig.defaultActionGroupId,
       validate: { params: transactionErrorRateParamsSchema },
+      schemas: {
+        params: {
+          type: 'config-schema',
+          schema: transactionErrorRateParamsSchema,
+        },
+      },
       actionVariables: {
         context: transactionErrorRateActionVariables,
       },

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_rule_type.ts

@@ -91,24 +91,32 @@ export async function registerInventoryThresholdRuleType(
     return;
   }
 
+  const paramsSchema = schema.object(
+    {
+      criteria: schema.arrayOf(condition),
+      nodeType: schema.string() as Type<InventoryItemType>,
+      filterQuery: schema.maybe(
+        schema.string({ validate: validateIsStringElasticsearchJSONFilter })
+      ),
+      sourceId: schema.string(),
+      alertOnNoData: schema.maybe(schema.boolean()),
+    },
+    { unknowns: 'allow' }
+  );
+
   alertingPlugin.registerType({
     id: METRIC_INVENTORY_THRESHOLD_ALERT_TYPE_ID,
     name: i18n.translate('xpack.infra.metrics.inventory.alertName', {
       defaultMessage: 'Inventory',
     }),
     validate: {
-      params: schema.object(
-        {
-          criteria: schema.arrayOf(condition),
-          nodeType: schema.string() as Type<InventoryItemType>,
-          filterQuery: schema.maybe(
-            schema.string({ validate: validateIsStringElasticsearchJSONFilter })
-          ),
-          sourceId: schema.string(),
-          alertOnNoData: schema.maybe(schema.boolean()),
-        },
-        { unknowns: 'allow' }
-      ),
+      params: paramsSchema,
+    },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: paramsSchema,
+      },
     },
     defaultActionGroupId: FIRED_ACTIONS_ID,
     doesSetRecoveryContext: true,

x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts

@@ -180,6 +180,12 @@ export function registerAnomalyDetectionAlertType({
     validate: {
       params: mlAnomalyDetectionAlertParams,
     },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: mlAnomalyDetectionAlertParams,
+      },
+    },
     actionVariables: {
       context: [
         {

x-pack/plugins/ml/server/lib/alerts/register_jobs_monitoring_rule_type.ts

@@ -122,6 +122,12 @@ export function registerJobsMonitoringRuleType({
     validate: {
       params: anomalyDetectionJobsHealthRuleParams,
     },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: anomalyDetectionJobsHealthRuleParams,
+      },
+    },
     actionVariables: {
       context: [
         {

x-pack/plugins/observability/server/lib/rules/custom_threshold/register_custom_threshold_rule_type.ts

@@ -105,6 +105,17 @@ export function thresholdRuleType(
     label: schema.maybe(schema.string()),
   });
 
+  const paramsSchema = schema.object(
+    {
+      criteria: schema.arrayOf(customCriterion),
+      groupBy: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
+      alertOnNoData: schema.maybe(schema.boolean()),
+      alertOnGroupDisappear: schema.maybe(schema.boolean()),
+      searchConfiguration: searchConfigurationSchema,
+    },
+    { unknowns: 'allow' }
+  );
+
   return {
     id: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID,
     name: i18n.translate('xpack.observability.threshold.ruleName', {
@@ -112,16 +123,13 @@ export function thresholdRuleType(
     }),
     fieldsForAAD: CUSTOM_THRESHOLD_AAD_FIELDS,
     validate: {
-      params: schema.object(
-        {
-          criteria: schema.arrayOf(customCriterion),
-          groupBy: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
-          alertOnNoData: schema.maybe(schema.boolean()),
-          alertOnGroupDisappear: schema.maybe(schema.boolean()),
-          searchConfiguration: searchConfigurationSchema,
-        },
-        { unknowns: 'allow' }
-      ),
+      params: paramsSchema,
+    },
+    schemas: {
+      params: {
+        type: 'config-schema' as const,
+        schema: paramsSchema,
+      },
     },
     defaultActionGroupId: FIRED_ACTION.id,
     actionGroups: [FIRED_ACTION, NO_DATA_ACTION],

x-pack/plugins/observability/server/lib/rules/slo_burn_rate/register.ts

@@ -50,6 +50,10 @@ export function sloBurnRateRuleType(
   basePath: IBasePath,
   alertsLocator?: LocatorPublic<AlertsLocatorParams>
 ) {
+  const paramsSchema = schema.object({
+    sloId: schema.string(),
+    windows: schema.arrayOf(windowSchema),
+  });
   return {
     id: SLO_BURN_RATE_RULE_TYPE_ID,
     name: i18n.translate('xpack.observability.slo.rules.burnRate.name', {
@@ -57,10 +61,13 @@ export function sloBurnRateRuleType(
     }),
     fieldsForAAD: SLO_BURN_RATE_AAD_FIELDS,
     validate: {
-      params: schema.object({
-        sloId: schema.string(),
-        windows: schema.arrayOf(windowSchema),
-      }),
+      params: paramsSchema,
+    },
+    schemas: {
+      params: {
+        type: 'config-schema' as const,
+        schema: paramsSchema,
+      },
     },
     defaultActionGroupId: ALERT_ACTION.id,
     actionGroups: [ALERT_ACTION, HIGH_PRIORITY_ACTION, MEDIUM_PRIORITY_ACTION, LOW_PRIORITY_ACTION],

x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions_legacy/logic/notifications/legacy_rules_notification_rule_type.ts

@@ -47,6 +47,12 @@ export const legacyRulesNotificationRuleType = ({
   validate: {
     params: legacyRulesNotificationParams,
   },
+  schemas: {
+    params: {
+      type: 'config-schema',
+      schema: legacyRulesNotificationParams,
+    },
+  },
   useSavedObjectReferences: {
     extractReferences: (params) => legacyExtractReferences({ logger, params }),
     injectReferences: (params, savedObjectReferences) =>

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/eql/create_eql_alert_type.ts

@@ -39,6 +39,9 @@ export const createEqlAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: EqlRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts

@@ -27,6 +27,9 @@ export const createEsqlAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: EsqlRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts

@@ -43,6 +43,9 @@ export const createIndicatorMatchAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: ThreatRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts

@@ -28,6 +28,9 @@ export const createMlAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: MachineLearningRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/create_new_terms_alert_type.ts

@@ -69,6 +69,9 @@ export const createNewTermsAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: NewTermsRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts

@@ -52,6 +52,9 @@ export const createQueryAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: UnifiedQueryRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/threshold/create_threshold_alert_type.ts

@@ -41,6 +41,9 @@ export const createThresholdAlertType = (
         },
       },
     },
+    schemas: {
+      params: { type: 'zod', schema: ThresholdRuleParams },
+    },
     actionGroups: [
       {
         id: 'default',

x-pack/plugins/stack_alerts/server/rule_types/es_query/rule_type.ts

@@ -154,6 +154,12 @@ export function getRuleType(
     validate: {
       params: EsQueryRuleParamsSchema,
     },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: EsQueryRuleParamsSchema,
+      },
+    },
     actionVariables: {
       context: [
         { name: 'message', description: actionVariableContextMessageLabel },

x-pack/plugins/stack_alerts/server/rule_types/geo_containment/rule_type.ts

@@ -185,6 +185,12 @@ export function getRuleType(): GeoContainmentRuleType {
     validate: {
       params: ParamsSchema,
     },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: ParamsSchema,
+      },
+    },
     actionVariables,
     minimumLicenseRequired: 'gold',
     isExportable: true,

x-pack/plugins/stack_alerts/server/rule_types/index_threshold/rule_type.ts

@@ -179,6 +179,12 @@ export function getRuleType(
     validate: {
       params: ParamsSchema,
     },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: ParamsSchema,
+      },
+    },
     actionVariables: {
       context: [
         { name: 'message', description: actionVariableContextMessageLabel },

x-pack/plugins/transform/server/lib/alerting/transform_health_rule_type/register_transform_health_rule_type.ts

@@ -88,6 +88,12 @@ export function getTransformHealthRuleType(
     actionGroups: [TRANSFORM_ISSUE_DETECTED],
     defaultActionGroupId: TRANSFORM_ISSUE,
     validate: { params: transformHealthRuleParams },
+    schemas: {
+      params: {
+        type: 'config-schema',
+        schema: transformHealthRuleParams,
+      },
+    },
     actionVariables: {
       context: [
         {

yarn.lock

@@ -32071,10 +32071,10 @@ zip-stream@^4.1.0:
     compress-commons "^4.1.0"
     readable-stream "^3.6.0"
 
-zod-to-json-schema@^3.20.4:
-  version "3.21.4"
-  resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.21.4.tgz#de97c5b6d4a25e9d444618486cb55c0c7fb949fd"
-  integrity sha512-fjUZh4nQ1s6HMccgIeE0VP4QG/YRGPmyjO9sAh890aQKPEk3nqbfUXhMFaC+Dr5KvYBm8BCyvfpZf2jY9aGSsw==
+zod-to-json-schema@^3.20.4, zod-to-json-schema@^3.22.3:
+  version "3.22.3"
+  resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.22.3.tgz#1c71f9fa23f80b2f3b5eed537afa8a13a66a5200"
+  integrity sha512-9isG8SqRe07p+Aio2ruBZmLm2Q6Sq4EqmXOiNpDxp+7f0LV6Q/LX65fs5Nn+FV/CzfF3NLBoksXbS2jNYIfpKw==
 
 zod@^3.22.3:
   version "3.22.3"