Add alerting_test_data package and a script to create rules for manual testing (#168493)

Closes https://github.com/elastic/actionable-observability/issues/140

## Summary

Add alerting_test_data package to create custom threshold and APM rules
for manual testing.

More information about the usage:
https://github.com/elastic/actionable-observability/pull/156

---------

Co-authored-by: almudenasanz <almudena.sanz@elastic.co>
Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -541,6 +541,7 @@ x-pack/plugins/notifications @elastic/appex-sharedux
 packages/kbn-object-versioning @elastic/appex-sharedux
 x-pack/plugins/observability_ai_assistant @elastic/obs-ai-assistant
 x-pack/packages/observability/alert_details @elastic/actionable-observability
+x-pack/packages/observability/alerting_test_data @elastic/actionable-observability
 x-pack/test/cases_api_integration/common/plugins/observability @elastic/response-ops
 x-pack/plugins/observability_log_explorer @elastic/infra-monitoring-ui
 x-pack/plugins/observability_onboarding @elastic/apm-ui

package.json

@@ -560,6 +560,7 @@
     "@kbn/object-versioning": "link:packages/kbn-object-versioning",
     "@kbn/observability-ai-assistant-plugin": "link:x-pack/plugins/observability_ai_assistant",
     "@kbn/observability-alert-details": "link:x-pack/packages/observability/alert_details",
+    "@kbn/observability-alerting-test-data": "link:x-pack/packages/observability/alerting_test_data",
     "@kbn/observability-fixtures-plugin": "link:x-pack/test/cases_api_integration/common/plugins/observability",
     "@kbn/observability-log-explorer-plugin": "link:x-pack/plugins/observability_log_explorer",
     "@kbn/observability-onboarding-plugin": "link:x-pack/plugins/observability_onboarding",

scripts/create_observability_rules.js

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('@babel/register')({
+  extensions: ['.ts', '.js'],
+  presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'],
+});
+
+var run = require('@kbn/observability-alerting-test-data').run;
+
+run();

tsconfig.base.json

@@ -1076,6 +1076,8 @@
       "@kbn/observability-ai-assistant-plugin/*": ["x-pack/plugins/observability_ai_assistant/*"],
       "@kbn/observability-alert-details": ["x-pack/packages/observability/alert_details"],
       "@kbn/observability-alert-details/*": ["x-pack/packages/observability/alert_details/*"],
+      "@kbn/observability-alerting-test-data": ["x-pack/packages/observability/alerting_test_data"],
+      "@kbn/observability-alerting-test-data/*": ["x-pack/packages/observability/alerting_test_data/*"],
       "@kbn/observability-fixtures-plugin": ["x-pack/test/cases_api_integration/common/plugins/observability"],
       "@kbn/observability-fixtures-plugin/*": ["x-pack/test/cases_api_integration/common/plugins/observability/*"],
       "@kbn/observability-log-explorer-plugin": ["x-pack/plugins/observability_log_explorer"],

x-pack/packages/observability/alerting_test_data/README.md

@@ -0,0 +1,3 @@
+# @kbn/observability-alerting-test-data
+
+Provides utilities to generate alerting test data

x-pack/packages/observability/alerting_test_data/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { createApmErrorCountRule } from './src/create_apm_error_count_threshold_rule';
+export { createApmFailedTransactionRateRule } from './src/create_apm_failed_transaction_rate_rule';
+export { createCustomThresholdRule } from './src/create_custom_threshold_rule';
+export { createDataView } from './src/create_data_view';
+export { createIndexConnector } from './src/create_index_connector';
+export { createRule } from './src/create_rule';
+export { run } from './src/run';
+
+export * from './src/scenarios';

x-pack/packages/observability/alerting_test_data/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../..',
+  roots: ['<rootDir>/x-pack/packages/observability/alerting_test_data'],
+};

x-pack/packages/observability/alerting_test_data/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/observability-alerting-test-data",
+  "owner": "@elastic/actionable-observability"
+}

x-pack/packages/observability/alerting_test_data/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@kbn/observability-alerting-test-data",
+  "descriptio": "Utils to generate observability alerting test data",
+  "author": "Actionable Observability",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/observability/alerting_test_data/src/constants.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const DATA_VIEW_ID = 'data-view-id';
+export const FIRED_ACTIONS_ID = 'custom_threshold.fired';
+export const ALERT_ACTION_INDEX = 'test-alert-action-index';
+
+export const KIBANA_DEFAULT_URL = 'http://127.0.0.1:5601';
+export const USERNAME = 'elastic';
+export const PASSWORD = 'changeme';
+
+export const HEADERS = {
+  'kbn-xsrf': 'true',
+  'x-elastic-internal-origin': 'foo',
+};

x-pack/packages/observability/alerting_test_data/src/create_apm_error_count_threshold_rule.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createRule } from './create_rule';
+
+export const createApmErrorCountRule = async (actionId: string) => {
+  const apmErrorRateRuleParams = {
+    tags: ['apm'],
+    consumer: 'apm',
+    name: 'apm_error_count_threshold',
+    rule_type_id: 'apm.error_rate',
+    params: {
+      threshold: 5,
+      windowSize: 5,
+      windowUnit: 'm',
+      transactionType: undefined,
+      serviceName: undefined,
+      environment: 'ENVIRONMENT_ALL',
+      searchConfiguration: {
+        query: {
+          query: 'service.environment: "rule-test"',
+          language: 'kuery',
+        },
+      },
+      groupBy: ['service.name', 'service.environment'],
+      useKqlFilter: true,
+    },
+    actions: [
+      {
+        group: 'threshold_met',
+        id: actionId,
+        params: {
+          documents: [
+            {
+              ruleName: '{{rule.name}}',
+              alertDetailsUrl: '{{context.alertDetailsUrl}}',
+              environment: '{{context.environment}}',
+              interval: '{{context.interval}}',
+              reason: '{{context.reason}}',
+              serviceName: '{{context.serviceName}}',
+              threshold: '{{context.threshold}}',
+              transactionName: '{{context.transactionName}}',
+              transactionType: '{{context.transactionType}}',
+              triggerValue: '{{context.triggerValue}}',
+              viewInAppUrl: '{{context.viewInAppUrl}}',
+            },
+          ],
+        },
+        frequency: {
+          notify_when: 'onActionGroupChange',
+          throttle: null,
+          summary: false,
+        },
+      },
+    ],
+    schedule: {
+      interval: '1m',
+    },
+  };
+
+  return createRule(apmErrorRateRuleParams);
+};

x-pack/packages/observability/alerting_test_data/src/create_apm_failed_transaction_rate_rule.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createRule } from './create_rule';
+
+export const createApmFailedTransactionRateRule = async (actionId: string) => {
+  const apmErrorRateRuleParams = {
+    tags: ['apm'],
+    consumer: 'apm',
+    name: 'apm_failed_transaction_rate_threshold',
+    rule_type_id: 'apm.transaction_error_rate',
+    params: {
+      threshold: 30,
+      windowSize: 5,
+      windowUnit: 'm',
+      transactionType: undefined,
+      serviceName: undefined,
+      environment: 'ENVIRONMENT_ALL',
+      searchConfiguration: {
+        query: {
+          query: 'service.environment: "rule-test"',
+          language: 'kuery',
+        },
+      },
+      groupBy: ['service.name', 'service.environment', 'transaction.type'],
+      useKqlFilter: true,
+    },
+    actions: [
+      {
+        group: 'threshold_met',
+        id: actionId,
+        params: {
+          documents: [
+            {
+              ruleName: '{{rule.name}}',
+              alertDetailsUrl: '{{context.alertDetailsUrl}}',
+              environment: '{{context.environment}}',
+              interval: '{{context.interval}}',
+              reason: '{{context.reason}}',
+              serviceName: '{{context.serviceName}}',
+              threshold: '{{context.threshold}}',
+              transactionName: '{{context.transactionName}}',
+              transactionType: '{{context.transactionType}}',
+              triggerValue: '{{context.triggerValue}}',
+              viewInAppUrl: '{{context.viewInAppUrl}}',
+            },
+          ],
+        },
+        frequency: {
+          notify_when: 'onActionGroupChange',
+          throttle: null,
+          summary: false,
+        },
+      },
+    ],
+    schedule: {
+      interval: '1m',
+    },
+  };
+
+  return createRule(apmErrorRateRuleParams);
+};

x-pack/packages/observability/alerting_test_data/src/create_custom_threshold_rule.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
+import { FIRED_ACTIONS_ID } from './constants';
+import { createRule } from './create_rule';
+
+export const createCustomThresholdRule = async (
+  actionId: string,
+  dataViewId: string,
+  ruleParams: {
+    consumer?: string;
+    name?: string;
+    params?: {
+      criteria: any[];
+      groupBy?: string[];
+      searchConfiguration: {
+        query: {
+          query?: string;
+        };
+      };
+    };
+  }
+) => {
+  const customThresholdRuleParams = {
+    tags: ['observability'],
+    consumer: ruleParams.consumer || 'logs',
+    name: ruleParams.name || 'Default custom threshold rule name',
+    rule_type_id: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID,
+    params: {
+      criteria: ruleParams.params?.criteria || [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.GT,
+          threshold: [1],
+          timeSize: 1,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', filter: '', aggType: Aggregators.COUNT }],
+        },
+      ],
+      groupBy: ruleParams.params?.groupBy,
+      alertOnNoData: true,
+      alertOnGroupDisappear: true,
+      searchConfiguration: {
+        query: {
+          query: ruleParams.params?.searchConfiguration.query.query || '',
+          language: 'kuery',
+        },
+        index: dataViewId,
+      },
+    },
+    actions: [
+      {
+        group: FIRED_ACTIONS_ID,
+        id: actionId,
+        params: {
+          documents: [
+            {
+              ruleName: '{{rule.name}}',
+              ruleType: '{{rule.type}}',
+              alertDetailsUrl: '{{context.alertDetailsUrl}}',
+              reason: '{{context.reason}}',
+              value: '{{context.value}}',
+              host: '{{context.host}}',
+            },
+          ],
+        },
+        frequency: {
+          notify_when: 'onActionGroupChange',
+          throttle: null,
+          summary: false,
+        },
+      },
+    ],
+    schedule: {
+      interval: '1m',
+    },
+  };
+
+  return createRule(customThresholdRuleParams);
+};

x-pack/packages/observability/alerting_test_data/src/create_data_view.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import axios from 'axios';
+import { HEADERS, PASSWORD, USERNAME } from './constants';
+import { getKibanaUrl } from './get_kibana_url';
+
+export const createDataView = async ({
+  indexPattern,
+  id,
+}: {
+  indexPattern: string;
+  id: string;
+}) => {
+  const DATA_VIEW_CREATION_API = `${await getKibanaUrl()}/api/content_management/rpc/create`;
+  const dataViewParams = {
+    contentTypeId: 'index-pattern',
+    data: {
+      fieldAttrs: '{}',
+      title: indexPattern,
+      timeFieldName: '@timestamp',
+      sourceFilters: '[]',
+      fields: '[]',
+      fieldFormatMap: '{}',
+      typeMeta: '{}',
+      runtimeFieldMap: '{}',
+      name: indexPattern,
+    },
+    options: { id },
+    version: 1,
+  };
+
+  return axios.post(DATA_VIEW_CREATION_API, dataViewParams, {
+    headers: HEADERS,
+    auth: {
+      username: USERNAME,
+      password: PASSWORD,
+    },
+  });
+};

x-pack/packages/observability/alerting_test_data/src/create_index_connector.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import axios from 'axios';
+import { ALERT_ACTION_INDEX, HEADERS, PASSWORD, USERNAME } from './constants';
+import { getKibanaUrl } from './get_kibana_url';
+
+export const createIndexConnector = async () => {
+  const INDEX_CONNECTOR_API = `${await getKibanaUrl()}/api/actions/connector`;
+  const indexConnectorParams = {
+    name: 'Test Index Connector',
+    config: {
+      index: ALERT_ACTION_INDEX,
+      refresh: true,
+    },
+    connector_type_id: '.index',
+  };
+
+  return axios.post(INDEX_CONNECTOR_API, indexConnectorParams, {
+    headers: HEADERS,
+    auth: {
+      username: USERNAME,
+      password: PASSWORD,
+    },
+  });
+};

x-pack/packages/observability/alerting_test_data/src/create_rule.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import axios from 'axios';
+import { HEADERS, PASSWORD, USERNAME } from './constants';
+import { getKibanaUrl } from './get_kibana_url';
+
+export const createRule = async (ruleParams: any) => {
+  const RULE_CREATION_API = `${await getKibanaUrl()}/api/alerting/rule`;
+  return axios.post(RULE_CREATION_API, ruleParams, {
+    headers: HEADERS,
+    auth: {
+      username: USERNAME,
+      password: PASSWORD,
+    },
+  });
+};

x-pack/packages/observability/alerting_test_data/src/get_kibana_url.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import fetch from 'node-fetch';
+import { format, parse } from 'url';
+import { KIBANA_DEFAULT_URL } from './constants';
+
+let kibanaUrl: string;
+
+export async function getKibanaUrl() {
+  if (kibanaUrl) {
+    return kibanaUrl;
+  }
+
+  try {
+    const unredirectedResponse = await fetch(KIBANA_DEFAULT_URL, {
+      method: 'HEAD',
+      follow: 1,
+      redirect: 'manual',
+    });
+
+    const discoveredKibanaUrl =
+      unredirectedResponse.headers
+        .get('location')
+        ?.replace('/spaces/enter', '')
+        ?.replace('spaces/space_selector', '') || KIBANA_DEFAULT_URL;
+
+    const parsedTarget = parse(KIBANA_DEFAULT_URL);
+
+    const parsedDiscoveredUrl = parse(discoveredKibanaUrl);
+
+    const discoveredKibanaUrlWithAuth = format({
+      ...parsedDiscoveredUrl,
+      auth: parsedTarget.auth,
+    });
+
+    const redirectedResponse = await fetch(discoveredKibanaUrlWithAuth, {
+      method: 'HEAD',
+    });
+
+    if (redirectedResponse.status !== 200) {
+      throw new Error(
+        `Expected HTTP 200 from ${discoveredKibanaUrlWithAuth}, got ${redirectedResponse.status}`
+      );
+    }
+
+    // eslint-disable-next-line no-console
+    console.log(`Discovered kibana running at: ${discoveredKibanaUrlWithAuth}`);
+
+    kibanaUrl = discoveredKibanaUrlWithAuth.replace(/\/$/, '');
+    return kibanaUrl;
+  } catch (error) {
+    throw new Error(`Could not connect to Kibana: ` + error.message);
+  }
+}

x-pack/packages/observability/alerting_test_data/src/run.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createApmErrorCountRule } from './create_apm_error_count_threshold_rule';
+import { createApmFailedTransactionRateRule } from './create_apm_failed_transaction_rate_rule';
+import { createCustomThresholdRule } from './create_custom_threshold_rule';
+import { createDataView } from './create_data_view';
+import { createIndexConnector } from './create_index_connector';
+
+import { scenario1, scenario2, scenario3, scenario4, scenario5, scenario6 } from './scenarios';
+
+const scenarios = [
+  // Logs use-cases
+  scenario1,
+  scenario2,
+  scenario3,
+  // Metrics use-cases
+  scenario4,
+  scenario5,
+  scenario6,
+];
+
+/* eslint-disable no-console */
+export async function run() {
+  console.log('Creating index connector - start');
+  const response = await createIndexConnector();
+  const actionId = await response.data.id;
+  console.log('Creating index connector - finished - actionId: ', actionId);
+  for (const scenario of scenarios) {
+    if (scenario.dataView.shouldCreate) {
+      console.log('Creating data view - start - id: ', scenario.dataView.id);
+      await createDataView(scenario.dataView);
+      console.log('Creating data view - finished - id: ', scenario.dataView.id);
+    }
+    console.log('Creating Custom threshold rule - start - name: ', scenario.ruleParams.name);
+    await createCustomThresholdRule(actionId, scenario.dataView.id, scenario.ruleParams);
+    console.log('Creating Custom threshold rule - finished - name: ', scenario.ruleParams.name);
+  }
+
+  console.log('Creating APM error count rule - start');
+  await createApmErrorCountRule(actionId);
+  console.log('Creating APM error count rule - finished');
+
+  console.log('Creating APM failed transaction rate rule - start');
+  await createApmFailedTransactionRateRule(actionId);
+  console.log('Creating APM failed transaction rate rule - finished');
+}

x-pack/packages/observability/alerting_test_data/src/scenarios/custom_threshold_log_count.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+
+export const scenario1 = {
+  dataView: {
+    indexPattern: 'high-cardinality-data-fake_hosts.fake_hosts-*',
+    id: 'data-view-id',
+    shouldCreate: true,
+  },
+  ruleParams: {
+    consumer: 'logs',
+    name: 'custom_threshold_log_count',
+    params: {
+      criteria: [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.LT,
+          threshold: [100],
+          timeSize: 1,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', filter: '', aggType: Aggregators.COUNT }],
+        },
+      ],
+      searchConfiguration: {
+        query: {
+          query: 'labels.scenario: custom_threshold_log_count',
+        },
+      },
+    },
+  },
+};

x-pack/packages/observability/alerting_test_data/src/scenarios/custom_threshold_log_count_groupby.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+
+export const scenario2 = {
+  dataView: {
+    indexPattern: 'high-cardinality-data-fake_hosts.fake_hosts-*',
+    id: 'data-view-id',
+    shouldCreate: false,
+  },
+  ruleParams: {
+    consumer: 'logs',
+    name: 'custom_threshold_log_count_groupby',
+    params: {
+      criteria: [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.LT,
+          threshold: [40],
+          timeSize: 1,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', filter: '', aggType: Aggregators.COUNT }],
+        },
+      ],
+      groupBy: ['event.dataset'],
+      searchConfiguration: {
+        query: {
+          query: 'labels.scenario: custom_threshold_log_count_groupby',
+        },
+      },
+    },
+  },
+};

x-pack/packages/observability/alerting_test_data/src/scenarios/custom_threshold_log_count_nodata.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+
+export const scenario3 = {
+  dataView: {
+    indexPattern: 'high-cardinality-data-fake_hosts.fake_hosts-*',
+    id: 'data-view-id',
+    shouldCreate: false,
+  },
+  ruleParams: {
+    consumer: 'logs',
+    name: 'custom_threshold_log_count_nodata',
+    params: {
+      criteria: [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.LT,
+          threshold: [5],
+          timeSize: 1,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', filter: '', aggType: Aggregators.COUNT }],
+        },
+      ],
+      searchConfiguration: {
+        query: {
+          query: 'labels.scenario: custom_threshold_log_count_nodata',
+        },
+      },
+    },
+  },
+};

x-pack/packages/observability/alerting_test_data/src/scenarios/custom_threshold_metric_avg.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+
+export const scenario4 = {
+  dataView: {
+    indexPattern: 'high-cardinality-data-fake_hosts.fake_hosts-*',
+    id: 'data-view-id',
+    shouldCreate: false,
+  },
+  ruleParams: {
+    consumer: 'logs',
+    name: 'custom_threshold_metric_avg',
+    params: {
+      criteria: [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.GT,
+          threshold: [80],
+          timeSize: 1,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', field: 'system.cpu.user.pct', aggType: Aggregators.AVERAGE }],
+        },
+      ],
+      searchConfiguration: {
+        query: {
+          query: 'labels.scenario: custom_threshold_metric_avg',
+        },
+      },
+    },
+  },
+};

x-pack/packages/observability/alerting_test_data/src/scenarios/custom_threshold_metric_avg_groupby.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+
+export const scenario5 = {
+  dataView: {
+    indexPattern: 'high-cardinality-data-fake_hosts.fake_hosts-*',
+    id: 'data-view-id',
+    shouldCreate: false,
+  },
+  ruleParams: {
+    consumer: 'logs',
+    name: 'custom_threshold_metric_avg_groupby',
+    params: {
+      criteria: [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.GT,
+          threshold: [80],
+          timeSize: 5,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', field: 'system.cpu.user.pct', aggType: Aggregators.AVERAGE }],
+        },
+      ],
+      groupBy: ['host.name'],
+      searchConfiguration: {
+        query: {
+          query: 'labels.scenario: custom_threshold_metric_avg_groupby',
+        },
+      },
+    },
+  },
+};

x-pack/packages/observability/alerting_test_data/src/scenarios/custom_threshold_metric_avg_nodata.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+} from '@kbn/observability-plugin/common/custom_threshold_rule/types';
+
+export const scenario6 = {
+  dataView: {
+    indexPattern: 'high-cardinality-data-fake_hosts.fake_hosts-*',
+    id: 'data-view-id',
+    shouldCreate: false,
+  },
+  ruleParams: {
+    consumer: 'logs',
+    name: 'custom_threshold_metric_avg_nodata',
+    params: {
+      criteria: [
+        {
+          aggType: Aggregators.CUSTOM,
+          comparator: Comparator.LT,
+          threshold: [1],
+          timeSize: 1,
+          timeUnit: 'm',
+          metrics: [{ name: 'A', field: 'system.cpu.user.pct', aggType: Aggregators.AVERAGE }],
+        },
+      ],
+      searchConfiguration: {
+        query: {
+          query: 'labels.scenario: custom_threshold_metric_avg_nodata',
+        },
+      },
+    },
+  },
+};

x-pack/packages/observability/alerting_test_data/src/scenarios/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { scenario1 } from './custom_threshold_log_count';
+export { scenario2 } from './custom_threshold_log_count_groupby';
+export { scenario3 } from './custom_threshold_log_count_nodata';
+export { scenario4 } from './custom_threshold_metric_avg';
+export { scenario5 } from './custom_threshold_metric_avg_groupby';
+export { scenario6 } from './custom_threshold_metric_avg_nodata';

x-pack/packages/observability/alerting_test_data/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/observability-plugin",
+    "@kbn/rule-data-utils",
+  ]
+}

yarn.lock

@@ -5090,6 +5090,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/observability-alerting-test-data@link:x-pack/packages/observability/alerting_test_data":
+  version "0.0.0"
+  uid ""
+
 "@kbn/observability-fixtures-plugin@link:x-pack/test/cases_api_integration/common/plugins/observability":
   version "0.0.0"
   uid ""