[AO] Add metric threshold integration test (#157489)

## Summary

Closes #157189

This PR adds a metric threshold integration test. This is the first step
in adding more test coverage for observability rules.

**Steps during the test**
1. Generating fake host data by using a similar implementation as
https://github.com/elastic/high-cardinality-cluster
    - Data is generated for the last 15 mins
- Implementation was simplified only to cover fake hosts and was
converted to typescript
2. Creating an action using an index connector
3. Creating a metric threshold rule containing step number 2 action
4. Checking the status of the rule to be active
5. Checking the triggered action to have the correct parameters
6. Checking the generated alert to have the correct information
7. Clean up

**How to run locally**
- Run server
```
node scripts/functional_tests_server --config x-pack/test/api_integration/apis/metrics_ui/config.ts
```
- Then run the test
```
node scripts/functional_tests__runner --include-pack/test/api_integration/apis/metrics_ui/cometric_threshold_rule.ts --config x-pack/test/api_integration/apis/metrics_ui/config.ts
```

**Reference**
I created https://github.com/elastic/integrations/issues/6168 to find a
better way to generate data and make sure that data matches what
metricbeats generates

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -413,6 +413,7 @@ packages/kbn-import-resolver @elastic/kibana-operations
 x-pack/plugins/index_lifecycle_management @elastic/platform-deployment-management
 x-pack/plugins/index_management @elastic/platform-deployment-management
 test/plugin_functional/plugins/index_patterns @elastic/kibana-app-services
+x-pack/packages/kbn-infra-forge @elastic/actionable-observability
 x-pack/plugins/infra @elastic/infra-monitoring-ui
 x-pack/plugins/ingest_pipelines @elastic/platform-deployment-management
 src/plugins/input_control_vis @elastic/kibana-presentation

package.json

@@ -436,6 +436,7 @@
     "@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
     "@kbn/index-management-plugin": "link:x-pack/plugins/index_management",
     "@kbn/index-patterns-test-plugin": "link:test/plugin_functional/plugins/index_patterns",
+    "@kbn/infra-forge": "link:x-pack/packages/kbn-infra-forge",
     "@kbn/infra-plugin": "link:x-pack/plugins/infra",
     "@kbn/ingest-pipelines-plugin": "link:x-pack/plugins/ingest_pipelines",
     "@kbn/input-control-vis-plugin": "link:src/plugins/input_control_vis",

tsconfig.base.json

@@ -820,6 +820,8 @@
       "@kbn/index-management-plugin/*": ["x-pack/plugins/index_management/*"],
       "@kbn/index-patterns-test-plugin": ["test/plugin_functional/plugins/index_patterns"],
       "@kbn/index-patterns-test-plugin/*": ["test/plugin_functional/plugins/index_patterns/*"],
+      "@kbn/infra-forge": ["x-pack/packages/kbn-infra-forge"],
+      "@kbn/infra-forge/*": ["x-pack/packages/kbn-infra-forge/*"],
       "@kbn/infra-plugin": ["x-pack/plugins/infra"],
       "@kbn/infra-plugin/*": ["x-pack/plugins/infra/*"],
       "@kbn/ingest-pipelines-plugin": ["x-pack/plugins/ingest_pipelines"],

x-pack/packages/kbn-infra-forge/README.md

@@ -0,0 +1,6 @@
+# @kbn/infra-forge
+
+`@kbn/data-forge` is a tool to generate infra data for integration testing.
+
+**Datasets**
+- fake hosts

x-pack/packages/kbn-infra-forge/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { cleanup, generate } from './src/run';

x-pack/packages/kbn-infra-forge/jest.config.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test/jest_node',
+  rootDir: '../../..',
+  roots: ['<rootDir>/x-pack/packages/kbn-infra-forge'],
+};

x-pack/packages/kbn-infra-forge/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/infra-forge",
+  "owner": "@elastic/actionable-observability"
+}

x-pack/packages/kbn-infra-forge/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/infra-forge",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0"
+}

x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index.ts

@@ -0,0 +1,140 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import lodash from 'lodash';
+import type { Moment } from 'moment';
+
+export { template } from './template';
+
+const createGroupIndex = (index: number) => Math.floor(index / 1000) * 1000;
+
+const randomBetween = (start = 0, end = 1, step = 0.1) =>
+  lodash.sample(lodash.range(start, end, step));
+
+let networkDataCount = 0;
+const generateNetworkData = lodash.memoize(() => {
+  networkDataCount += 10000;
+  return networkDataCount;
+});
+
+export const generateEvent = (index: number, timestamp: Moment, interval: number) => {
+  const groupIndex = createGroupIndex(index);
+  return [
+    {
+      '@timestamp': timestamp.toISOString(),
+      tags: [`group-${groupIndex}`, `event-${index}`],
+      host: {
+        name: `host-${index}`,
+        mac: ['00-00-5E-00-53-23', '00-00-5E-00-53-24'],
+        network: {
+          name: `network-${index}`,
+        },
+      },
+      event: {
+        module: 'system',
+        dataset: 'system.cpu',
+      },
+      labels: {
+        groupId: `group-${groupIndex}`,
+        eventId: `event-${index}`,
+      },
+      system: {
+        cpu: {
+          cores: 4,
+          total: {
+            norm: {
+              pct: randomBetween(),
+            },
+          },
+          user: {
+            pct: randomBetween(1, 4),
+          },
+          system: {
+            pct: randomBetween(1, 4),
+          },
+        },
+      },
+      metricset: {
+        period: interval,
+      },
+      container: {
+        id: `container-${index}`,
+        name: 'container-name',
+      },
+    },
+    {
+      '@timestamp': timestamp.toISOString(),
+      host: {
+        name: `host-${index}`,
+        network: {
+          name: `network-${index}`,
+        },
+      },
+      event: {
+        module: 'system',
+        dataset: 'system.network',
+      },
+      labels: {
+        groupId: `group-${groupIndex}`,
+        eventId: `event-${index}`,
+      },
+      system: {
+        network: {
+          name: 'eth0',
+          in: {
+            bytes: generateNetworkData(),
+          },
+          out: {
+            bytes: generateNetworkData(),
+          },
+        },
+      },
+      metricset: {
+        period: interval,
+      },
+      container: {
+        id: `container-${index}`,
+        name: 'container-name',
+      },
+    },
+    {
+      '@timestamp': timestamp.toISOString(),
+      host: {
+        name: `host-${index}`,
+        network: {
+          name: `network-${index}`,
+        },
+      },
+      event: {
+        module: 'system',
+        dataset: 'system.network',
+      },
+      labels: {
+        groupId: `group-${groupIndex}`,
+        eventId: `event-${index}`,
+      },
+      system: {
+        network: {
+          name: 'eth1',
+          in: {
+            bytes: generateNetworkData(),
+          },
+          out: {
+            bytes: generateNetworkData(),
+          },
+        },
+      },
+      metricset: {
+        period: interval,
+      },
+      container: {
+        id: `container-${index}`,
+        name: 'container-name',
+      },
+    },
+  ];
+};

x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/template.ts

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const template = {
+  order: 1,
+  index_patterns: ['kbn-data-forge*'],
+  settings: {
+    index: {
+      mapping: {
+        total_fields: {
+          limit: '10000',
+        },
+      },
+      number_of_shards: '1',
+      number_of_replicas: '0',
+      query: {
+        default_field: ['message', 'labels.*', 'event.*'],
+      },
+    },
+  },
+  mappings: {
+    dynamic_templates: [
+      {
+        labels: {
+          path_match: 'labels.*',
+          mapping: {
+            type: 'keyword',
+          },
+          match_mapping_type: 'string',
+        },
+      },
+      {
+        strings_as_keyword: {
+          mapping: {
+            ignore_above: 1024,
+            type: 'keyword',
+          },
+          match_mapping_type: 'string',
+        },
+      },
+    ],
+    date_detection: false,
+    properties: {
+      '@timestamp': {
+        type: 'date',
+      },
+      tags: {
+        type: 'keyword',
+      },
+      metricset: {
+        properties: {
+          period: {
+            type: 'long',
+          },
+        },
+      },
+      host: {
+        properties: {
+          name: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+          network: {
+            properties: {
+              name: {
+                type: 'keyword',
+                ignore_above: 256,
+              },
+            },
+          },
+        },
+      },
+      event: {
+        properties: {
+          dataset: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+          module: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+        },
+      },
+      system: {
+        properties: {
+          cpu: {
+            properties: {
+              cores: {
+                type: 'long',
+              },
+              total: {
+                properties: {
+                  norm: {
+                    properties: {
+                      pct: {
+                        scaling_factor: 1000,
+                        type: 'scaled_float',
+                      },
+                    },
+                  },
+                },
+              },
+              user: {
+                properties: {
+                  pct: {
+                    scaling_factor: 1000,
+                    type: 'scaled_float',
+                  },
+                  norm: {
+                    properties: {
+                      pct: {
+                        scaling_factor: 1000,
+                        type: 'scaled_float',
+                      },
+                    },
+                  },
+                },
+              },
+              system: {
+                properties: {
+                  pct: {
+                    scaling_factor: 1000,
+                    type: 'scaled_float',
+                  },
+                },
+              },
+            },
+          },
+          network: {
+            properties: {
+              name: {
+                type: 'keyword',
+                ignore_above: 256,
+              },
+              in: {
+                properties: {
+                  bytes: {
+                    type: 'long',
+                  },
+                },
+              },
+              out: {
+                properties: {
+                  bytes: {
+                    type: 'long',
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      container: {
+        properties: {
+          id: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+          name: {
+            type: 'keyword',
+            ignore_above: 256,
+          },
+        },
+      },
+    },
+  },
+  aliases: {
+    'metrics-fake_hosts': {},
+  },
+};

x-pack/packages/kbn-infra-forge/src/lib/install_template.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Client } from '@elastic/elasticsearch';
+import type { ToolingLog } from '@kbn/tooling-log';
+
+export function installTemplate(
+  client: Client,
+  template: object,
+  namespace: string,
+  logger: ToolingLog
+) {
+  logger.debug(`installTemplate > template name: kbn-data-forge-${namespace}`);
+  return client.indices
+    .putTemplate({ name: `kbn-data-forge-${namespace}`, body: template })
+    .catch((error: any) => logger.error(`installTemplate > ${JSON.stringify(error)}`));
+}
+
+export function deleteTemplate(client: Client, namespace: string, logger: ToolingLog) {
+  logger.debug(`deleteTemplate > template name: kbn-data-forge-${namespace}`);
+  return client.indices
+    .deleteTemplate({ name: `kbn-data-forge-${namespace}` })
+    .catch((error: any) => logger.error(`deleteTemplate > ${JSON.stringify(error)}`));
+}

x-pack/packages/kbn-infra-forge/src/lib/queue.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import async from 'async';
+import { omit } from 'lodash';
+
+import type { Client } from '@elastic/elasticsearch';
+import type { ToolingLog } from '@kbn/tooling-log';
+
+export const getIndexName = (namespace: string) => `kbn-data-forge-${namespace}`;
+
+export const createQueue = (
+  esClient: Client,
+  namespace: string,
+  payloadSize = 1000,
+  concurrency = 5,
+  logger: ToolingLog
+) => {
+  const indexName = getIndexName(namespace);
+  logger.debug(`createQueue > index name: ${indexName}`);
+  return async.cargoQueue(
+    (docs: object[], callback) => {
+      const body: any[] = [];
+      docs.forEach((doc) => {
+        body.push({
+          create: {
+            _index: indexName,
+          },
+        });
+        body.push(omit(doc, 'namespace'));
+      });
+      esClient
+        .bulk({ body })
+        .then((resp) => {
+          if (resp.errors) {
+            logger.error(
+              `createQueue > Failed to index document to ${indexName} index: ${JSON.stringify(
+                resp.errors
+              )}`
+            );
+            return callback(new Error('Failed to index'));
+          }
+          return callback();
+        })
+        .catch((error) => {
+          logger.error(
+            `createQueue > Error while indexing document to ${indexName} index: ${error}`
+          );
+          callback(error);
+        });
+    },
+    concurrency,
+    payloadSize
+  );
+};

x-pack/packages/kbn-infra-forge/src/run.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { range } from 'lodash';
+import moment from 'moment';
+import datemath from '@kbn/datemath';
+import type { Moment } from 'moment';
+import type { ToolingLog } from '@kbn/tooling-log';
+import type { Client } from '@elastic/elasticsearch';
+import { createQueue, getIndexName } from './lib/queue';
+import { deleteTemplate, installTemplate } from './lib/install_template';
+import * as fakeHosts from './data_sources/fake_hosts';
+
+const generateEventsFns = {
+  fake_hosts: fakeHosts.generateEvent,
+};
+
+const templates = {
+  fake_hosts: fakeHosts.template,
+};
+
+const EVENTS_PER_CYCLE = 1;
+const PAYLOAD_SIZE = 10000;
+const CONCURRENCY = 5;
+const INDEX_INTERVAL = 60000;
+const DATASET = 'fake_hosts';
+
+const createEvents = (size: number, timestamp: Moment) =>
+  range(size)
+    .map((i) => {
+      const generateEvent = generateEventsFns[DATASET];
+      return generateEvent(i, timestamp, INDEX_INTERVAL);
+    })
+    .flat();
+
+function indexHistory(lookback: string, queue: any, logger: ToolingLog, nextTimestamp?: Moment) {
+  const now = moment();
+  const startTs = datemath.parse(lookback, { momentInstance: moment });
+  const timestamp = nextTimestamp || moment(startTs);
+  createEvents(EVENTS_PER_CYCLE, timestamp).forEach((event) => queue.push(event));
+  return queue.drain().then(() => {
+    if (timestamp.isBefore(now)) {
+      return indexHistory(lookback, queue, logger, timestamp.add(INDEX_INTERVAL, 'ms'));
+    }
+    return Promise.resolve();
+  });
+}
+
+export const generate = async ({
+  esClient,
+  lookback,
+  logger,
+}: {
+  esClient: Client;
+  lookback: string;
+  logger: ToolingLog;
+}) => {
+  const queue = createQueue(esClient, DATASET, PAYLOAD_SIZE, CONCURRENCY, logger);
+  const template = templates[DATASET];
+  await installTemplate(esClient, template, DATASET, logger).then(() =>
+    indexHistory(lookback, queue, logger)
+  );
+  return getIndexName(DATASET);
+};
+
+export const cleanup = async ({ esClient, logger }: { esClient: Client; logger: ToolingLog }) => {
+  await deleteTemplate(esClient, DATASET, logger);
+};

x-pack/packages/kbn-infra-forge/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/tooling-log",
+    "@kbn/datemath",
+  ]
+}

x-pack/plugins/infra/common/alerting/metrics/types.ts

@@ -14,6 +14,18 @@ export const METRIC_THRESHOLD_ALERT_TYPE_ID = 'metrics.alert.threshold';
 export const METRIC_INVENTORY_THRESHOLD_ALERT_TYPE_ID = 'metrics.alert.inventory.threshold';
 export const METRIC_ANOMALY_ALERT_TYPE_ID = 'metrics.alert.anomaly';
 
+export enum InfraRuleType {
+  MetricThreshold = 'metrics.alert.threshold',
+  InventoryThreshold = 'metrics.alert.inventory.threshold',
+  Anomaly = 'metrics.alert.anomaly',
+}
+
+export interface InfraRuleTypeParams {
+  [InfraRuleType.MetricThreshold]: MetricThresholdParams;
+  [InfraRuleType.InventoryThreshold]: InventoryMetricConditions;
+  [InfraRuleType.Anomaly]: MetricAnomalyParams;
+}
+
 export enum Comparator {
   GT = '>',
   LT = '<',
@@ -88,6 +100,15 @@ export interface InventoryMetricThresholdParams {
   alertOnNoData?: boolean;
 }
 
+export interface MetricThresholdParams {
+  criteria: MetricExpressionParams[];
+  filterQuery?: string;
+  filterQueryText?: string;
+  sourceId?: string;
+  alertOnNoData?: boolean;
+  alertOnGroupDisappear?: boolean;
+}
+
 interface BaseMetricExpressionParams {
   timeSize: number;
   timeUnit: TimeUnitChar;

x-pack/test/api_integration/apis/metrics_ui/helpers/alerting_api_helper.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { InfraRuleType, InfraRuleTypeParams } from '@kbn/infra-plugin/common/alerting/metrics';
+import type { SuperTest, Test } from 'supertest';
+
+export async function createIndexConnector({
+  supertest,
+  name,
+  indexName,
+}: {
+  supertest: SuperTest<Test>;
+  name: string;
+  indexName: string;
+}) {
+  const { body } = await supertest
+    .post(`/api/actions/connector`)
+    .set('kbn-xsrf', 'foo')
+    .send({
+      name,
+      config: {
+        index: indexName,
+        refresh: true,
+      },
+      connector_type_id: '.index',
+    });
+  return body.id as string;
+}
+
+export async function createMetricThresholdRule<T extends InfraRuleType>({
+  supertest,
+  name,
+  ruleTypeId,
+  params,
+  actions = [],
+  schedule,
+}: {
+  supertest: SuperTest<Test>;
+  ruleTypeId: T;
+  name: string;
+  params: InfraRuleTypeParams[T];
+  actions?: any[];
+  schedule?: { interval: string };
+}) {
+  const { body } = await supertest
+    .post(`/api/alerting/rule`)
+    .set('kbn-xsrf', 'foo')
+    .send({
+      params,
+      consumer: 'infrastructure',
+      schedule: schedule || {
+        interval: '5m',
+      },
+      tags: ['infrastructure'],
+      name,
+      rule_type_id: ruleTypeId,
+      actions,
+    });
+  return body;
+}

x-pack/test/api_integration/apis/metrics_ui/helpers/alerting_wait_for_helpers.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import pRetry from 'p-retry';
+
+import type SuperTest from 'supertest';
+import type { Client } from '@elastic/elasticsearch';
+import type {
+  AggregationsAggregate,
+  SearchResponse,
+} from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+export async function waitForRuleStatus({
+  id,
+  expectedStatus,
+  supertest,
+}: {
+  id: string;
+  expectedStatus: string;
+  supertest: SuperTest.SuperTest<SuperTest.Test>;
+}): Promise<Record<string, any>> {
+  return pRetry(
+    async () => {
+      const response = await supertest.get(`/api/alerting/rule/${id}`);
+      const { execution_status: executionStatus } = response.body || {};
+      const { status } = executionStatus || {};
+      if (status !== expectedStatus) {
+        throw new Error(`waitForStatus(${expectedStatus}): got ${status}`);
+      }
+      return executionStatus;
+    },
+    { retries: 10 }
+  );
+}
+
+export async function waitForDocumentInIndex<T>({
+  esClient,
+  indexName,
+}: {
+  esClient: Client;
+  indexName: string;
+}): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
+  return pRetry(
+    async () => {
+      const response = await esClient.search<T>({ index: indexName });
+      if (response.hits.hits.length === 0) {
+        throw new Error('No hits found');
+      }
+      return response;
+    },
+    { retries: 10 }
+  );
+}
+
+export async function waitForAlertInIndex<T>({
+  esClient,
+  indexName,
+  ruleId,
+}: {
+  esClient: Client;
+  indexName: string;
+  ruleId: string;
+}): Promise<SearchResponse<T, Record<string, AggregationsAggregate>>> {
+  return pRetry(
+    async () => {
+      const response = await esClient.search<T>({
+        index: indexName,
+        body: {
+          query: {
+            term: {
+              'kibana.alert.rule.uuid': ruleId,
+            },
+          },
+        },
+      });
+      if (response.hits.hits.length === 0) {
+        throw new Error('No hits found');
+      }
+      return response;
+    },
+    { retries: 10 }
+  );
+}

x-pack/test/api_integration/apis/metrics_ui/index.js

@@ -18,6 +18,7 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./ip_to_hostname'));
     loadTestFile(require.resolve('./http_source'));
     loadTestFile(require.resolve('./metric_threshold_alert'));
+    loadTestFile(require.resolve('./metric_threshold_rule'));
     loadTestFile(require.resolve('./metrics_overview_top'));
     loadTestFile(require.resolve('./metrics_process_list'));
     loadTestFile(require.resolve('./metrics_process_list_chart'));

x-pack/test/api_integration/apis/metrics_ui/metric_threshold_rule.ts

@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { cleanup, generate } from '@kbn/infra-forge';
+import { Aggregators, Comparator, InfraRuleType } from '@kbn/infra-plugin/common/alerting/metrics';
+import {
+  waitForDocumentInIndex,
+  waitForAlertInIndex,
+  waitForRuleStatus,
+} from './helpers/alerting_wait_for_helpers';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import { createIndexConnector, createMetricThresholdRule } from './helpers/alerting_api_helper';
+
+export default function ({ getService }: FtrProviderContext) {
+  const esClient = getService('es');
+  const esDeleteAllIndices = getService('esDeleteAllIndices');
+  const supertest = getService('supertest');
+  const logger = getService('log');
+
+  describe('Metric threshold rule >', () => {
+    let ruleId: string;
+    let actionId: string | undefined;
+    let infraDataIndex: string;
+
+    const METRICS_ALERTS_INDEX = '.alerts-observability.metrics.alerts-default';
+    const ALERT_ACTION_INDEX = 'alert-action-metric-threshold';
+
+    describe('alert and action creation', () => {
+      before(async () => {
+        infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger });
+        actionId = await createIndexConnector({
+          supertest,
+          name: 'Index Connector: Metric threshold API test',
+          indexName: ALERT_ACTION_INDEX,
+        });
+        const createdRule = await createMetricThresholdRule({
+          supertest,
+          ruleTypeId: InfraRuleType.MetricThreshold,
+          name: 'Metric threshold rule',
+          params: {
+            criteria: [
+              {
+                aggType: Aggregators.AVERAGE,
+                comparator: Comparator.GT,
+                threshold: [0.5],
+                timeSize: 5,
+                timeUnit: 'm',
+                metric: 'system.cpu.user.pct',
+              },
+            ],
+            sourceId: 'default',
+            alertOnNoData: true,
+            alertOnGroupDisappear: true,
+          },
+          actions: [
+            {
+              group: 'metrics.threshold.fired',
+              id: actionId,
+              params: {
+                documents: [
+                  {
+                    ruleType: '{{rule.type}}',
+                  },
+                ],
+              },
+              frequency: {
+                notify_when: 'onActionGroupChange',
+                throttle: null,
+                summary: false,
+              },
+            },
+          ],
+          schedule: {
+            interval: '1m',
+          },
+        });
+        ruleId = createdRule.id;
+      });
+
+      after(async () => {
+        await supertest.delete(`/api/alerting/rule/${ruleId}`).set('kbn-xsrf', 'foo');
+        await supertest.delete(`/api/actions/connector/${actionId}`).set('kbn-xsrf', 'foo');
+        await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
+        await esClient.deleteByQuery({
+          index: METRICS_ALERTS_INDEX,
+          query: { term: { 'kibana.alert.rule.uuid': ruleId } },
+        });
+        await esClient.deleteByQuery({
+          index: '.kibana-event-log-*',
+          query: { term: { 'kibana.alert.rule.consumer': 'infrastructure' } },
+        });
+        await cleanup({ esClient, logger });
+      });
+
+      it('rule should be active', async () => {
+        const executionStatus = await waitForRuleStatus({
+          id: ruleId,
+          expectedStatus: 'active',
+          supertest,
+        });
+        expect(executionStatus.status).to.be('active');
+      });
+
+      it('should set correct action parameter: ruleType', async () => {
+        const resp = await waitForDocumentInIndex<{ ruleType: string }>({
+          esClient,
+          indexName: ALERT_ACTION_INDEX,
+        });
+
+        expect(resp.hits.hits[0]._source?.ruleType).eql('metrics.alert.threshold');
+      });
+
+      it('should set correct information in the alert document', async () => {
+        const resp = await waitForAlertInIndex({
+          esClient,
+          indexName: METRICS_ALERTS_INDEX,
+          ruleId,
+        });
+        expect(resp.hits.hits[0]._source).property(
+          'kibana.alert.rule.category',
+          'Metric threshold'
+        );
+        expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'infrastructure');
+        expect(resp.hits.hits[0]._source).property(
+          'kibana.alert.rule.name',
+          'Metric threshold rule'
+        );
+        expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'infrastructure');
+        expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0);
+        expect(resp.hits.hits[0]._source).property(
+          'kibana.alert.rule.rule_type_id',
+          'metrics.alert.threshold'
+        );
+        expect(resp.hits.hits[0]._source).property('kibana.alert.rule.uuid', ruleId);
+        expect(resp.hits.hits[0]._source).property('kibana.space_ids').contain('default');
+        expect(resp.hits.hits[0]._source)
+          .property('kibana.alert.rule.tags')
+          .contain('infrastructure');
+        expect(resp.hits.hits[0]._source).property(
+          'kibana.alert.action_group',
+          'metrics.threshold.fired'
+        );
+        expect(resp.hits.hits[0]._source).property('tags').contain('infrastructure');
+        expect(resp.hits.hits[0]._source).property('kibana.alert.instance.id', '*');
+        expect(resp.hits.hits[0]._source).property('kibana.alert.workflow_status', 'open');
+        expect(resp.hits.hits[0]._source).property('event.kind', 'signal');
+        expect(resp.hits.hits[0]._source).property('event.action', 'open');
+
+        expect(resp.hits.hits[0]._source)
+          .property('kibana.alert.rule.parameters')
+          .eql({
+            criteria: [
+              {
+                aggType: 'avg',
+                comparator: '>',
+                threshold: [0.5],
+                timeSize: 5,
+                timeUnit: 'm',
+                metric: 'system.cpu.user.pct',
+              },
+            ],
+            sourceId: 'default',
+            alertOnNoData: true,
+            alertOnGroupDisappear: true,
+          });
+      });
+    });
+  });
+}

x-pack/test/tsconfig.json

@@ -125,5 +125,6 @@
     "@kbn/guided-onboarding-plugin",
     "@kbn/field-formats-plugin",
     "@kbn/ml-anomaly-utils",
+    "@kbn/infra-forge",
   ]
 }

yarn.lock

@@ -4378,6 +4378,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/infra-forge@link:x-pack/packages/kbn-infra-forge":
+  version "0.0.0"
+  uid ""
+
 "@kbn/infra-plugin@link:x-pack/plugins/infra":
   version "0.0.0"
   uid ""