Copy group and action variable tests to serverless and skip a test (#179136)

Closes #167897
Closes #167518
Closes #175499

## Summary

I also added a refresh index after generating data-forge data to
decrease the time of running tests and making sure data is available
when the rule is executed, which in one instance, the time of the tests
decreased from `30.0s` to `15.7s` 🎉

|Before|After|
|---|---|

|![image](e6bedd5c-c920-4bc5-ab76-91f3e420aa1e)|

|![image](75b398d1-53d9-44b0-bbdf-b8826d25de69)|

[200] Flaky test runner:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5539
✅

Commands to run test locally:
```
// Server
node scripts/functional_tests_server.js --config x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts

// One test (Remove --include to run all tests)
node scripts/functional_test_runner --config=x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts --include=x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/p99_pct_fired.ts
```

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/kbn-data-forge/src/run.ts

@@ -23,5 +23,6 @@ export async function run(config: Config, client: Client, logger: ToolingLog) {
   await indexSchedule(config, client, logger);
   const indicesCreated = [...indices];
   indices.clear();
+  await client.indices.refresh({ index: indicesCreated });
   return indicesCreated;
 }

x-pack/test/alerting_api_integration/observability/custom_threshold_rule/p99_pct_fired.ts

@@ -36,18 +36,18 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Custom Threshold rule - P99 - PCT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
-    const DATE_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    const DATA_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*';
-    const DATE_VIEW_NAME = 'ad-hoc-data-view-name';
+    const DATA_VIEW_NAME = 'ad-hoc-data-view-name';
     const DATA_VIEW_ID = 'data-view-id';
     const MOCKED_AD_HOC_DATA_VIEW = {
       id: DATA_VIEW_ID,
-      title: DATE_VIEW_TITLE,
+      title: DATA_VIEW_TITLE,
       timeFieldName: '@timestamp',
       sourceFilters: [],
       fieldFormats: {},
       runtimeFieldMap: {},
       allowNoIndex: false,
-      name: DATE_VIEW_NAME,
+      name: DATA_VIEW_NAME,
       allowHidden: false,
     };
     let dataForgeConfig: PartialConfig;
@@ -77,7 +77,7 @@ export default function ({ getService }: FtrProviderContext) {
       logger.info(JSON.stringify(dataForgeIndices.join(',')));
       await waitForDocumentInIndex({
         esClient,
-        indexName: DATE_VIEW_TITLE,
+        indexName: DATA_VIEW_TITLE,
         docCountTarget: 270,
         retryService,
         logger,
@@ -246,7 +246,7 @@ export default function ({ getService }: FtrProviderContext) {
           `https://localhost:5601/app/observability/alerts/${alertId}`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `99th percentile of system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATE_VIEW_NAME})`
+          `99th percentile of system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATA_VIEW_NAME})`
         );
         expect(resp.hits.hits[0]._source?.value).eql('250%');
 
@@ -256,7 +256,7 @@ export default function ({ getService }: FtrProviderContext) {
 
         expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOGS_EXPLORER_LOCATOR');
         expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({
-          dataset: DATE_VIEW_TITLE,
+          dataset: DATA_VIEW_TITLE,
           timeRange: { to: 'now' },
           query: { query: '', language: 'kuery' },
           filters: [],

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts

@@ -13,7 +13,11 @@ import {
 import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/custom_threshold/constants';
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
+import { parseSearchParams } from '@kbn/share-plugin/common/url_service';
+import { omit } from 'lodash';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { ISO_DATE_REGEX } from './constants';
+import { ActionDocument, LogsExplorerLocatorParsedParams } from './typings';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
@@ -25,13 +29,15 @@ export default function ({ getService }: FtrProviderContext) {
 
   describe('Custom Threshold rule - AVG - PCT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
-    const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
+    const DATA_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    const DATA_VIEW_NAME = 'data-view-name';
     const DATA_VIEW_ID = 'data-view-id';
     let dataForgeConfig: PartialConfig;
     let dataForgeIndices: string[];
     let actionId: string;
     let ruleId: string;
+    let alertId: string;
 
     before(async () => {
       dataForgeConfig = {
@@ -54,11 +60,11 @@ export default function ({ getService }: FtrProviderContext) {
         },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
-      await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 360 });
+      await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW_TITLE, docCountTarget: 360 });
       await dataViewApi.create({
-        name: DATA_VIEW,
+        name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
-        title: DATA_VIEW,
+        title: DATA_VIEW_TITLE,
       });
     });
 
@@ -130,6 +136,11 @@ export default function ({ getService }: FtrProviderContext) {
                 documents: [
                   {
                     ruleType: '{{rule.type}}',
+                    alertDetailsUrl: '{{context.alertDetailsUrl}}',
+                    reason: '{{context.reason}}',
+                    value: '{{context.value}}',
+                    host: '{{context.host}}',
+                    viewInAppUrl: '{{context.viewInAppUrl}}',
                   },
                 ],
               },
@@ -164,6 +175,7 @@ export default function ({ getService }: FtrProviderContext) {
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });
+        alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
         expect(resp.hits.hits[0]._source).property(
           'kibana.alert.rule.category',
@@ -209,6 +221,35 @@ export default function ({ getService }: FtrProviderContext) {
             searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
           });
       });
+
+      it('should set correct action variables', async () => {
+        const resp = await alertingApi.waitForDocumentInIndex<ActionDocument>({
+          indexName: ALERT_ACTION_INDEX,
+          docCountTarget: 1,
+        });
+
+        expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
+        expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
+          `http://localhost:5620/app/observability/alerts/${alertId}`
+        );
+        expect(resp.hits.hits[0]._source?.reason).eql(
+          `Average system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATA_VIEW_NAME})`
+        );
+        expect(resp.hits.hits[0]._source?.value).eql('250%');
+
+        const parsedViewInAppUrl = parseSearchParams<LogsExplorerLocatorParsedParams>(
+          new URL(resp.hits.hits[0]._source?.viewInAppUrl || '').search
+        );
+
+        expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOGS_EXPLORER_LOCATOR');
+        expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({
+          dataset: DATA_VIEW_ID,
+          timeRange: { to: 'now' },
+          query: { query: '', language: 'kuery' },
+          filters: [],
+        });
+        expect(parsedViewInAppUrl.params.timeRange.from).match(ISO_DATE_REGEX);
+      });
     });
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_no_data.ts

@@ -9,29 +9,37 @@ import {
   Aggregators,
   Comparator,
 } from '@kbn/observability-plugin/common/custom_threshold_rule/types';
-import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/custom_threshold/constants';
+import { NO_DATA_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/custom_threshold/constants';
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
+import { parseSearchParams } from '@kbn/share-plugin/common/url_service';
+import { omit } from 'lodash';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { ISO_DATE_REGEX } from './constants';
+import { ActionDocument, LogsExplorerLocatorParsedParams } from './typings';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const alertingApi = getService('alertingApi');
   const dataViewApi = getService('dataViewApi');
+  const esDeleteAllIndices = getService('esDeleteAllIndices');
 
   describe('Custom Threshold rule - AVG - PCT - NoData', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
+    const DATA_VIEW_NAME = 'no-data-pattern-name';
     const DATA_VIEW_ID = 'data-view-id-no-data';
+    const DATA_VIEW_TITLE = 'no-data-pattern-title';
     let actionId: string;
     let ruleId: string;
+    let alertId: string;
 
     before(async () => {
       await dataViewApi.create({
-        name: 'no-data-pattern',
+        name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
-        title: 'no-data-pattern',
+        title: DATA_VIEW_TITLE,
       });
     });
 
@@ -57,6 +65,7 @@ export default function ({ getService }: FtrProviderContext) {
       await dataViewApi.delete({
         id: DATA_VIEW_ID,
       });
+      await esDeleteAllIndices([ALERT_ACTION_INDEX]);
     });
 
     describe('Rule creation', () => {
@@ -95,12 +104,16 @@ export default function ({ getService }: FtrProviderContext) {
           },
           actions: [
             {
-              group: FIRED_ACTIONS_ID,
+              group: NO_DATA_ACTIONS_ID,
               id: actionId,
               params: {
                 documents: [
                   {
                     ruleType: '{{rule.type}}',
+                    alertDetailsUrl: '{{context.alertDetailsUrl}}',
+                    reason: '{{context.reason}}',
+                    value: '{{context.value}}',
+                    viewInAppUrl: '{{context.viewInAppUrl}}',
                   },
                 ],
               },
@@ -135,6 +148,7 @@ export default function ({ getService }: FtrProviderContext) {
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });
+        alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
         expect(resp.hits.hits[0]._source).property(
           'kibana.alert.rule.category',
@@ -183,6 +197,35 @@ export default function ({ getService }: FtrProviderContext) {
             },
           });
       });
+
+      it('should set correct action variables', async () => {
+        const resp = await alertingApi.waitForDocumentInIndex<ActionDocument>({
+          indexName: ALERT_ACTION_INDEX,
+          docCountTarget: 1,
+        });
+
+        expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
+        expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
+          `http://localhost:5620/app/observability/alerts/${alertId}`
+        );
+        expect(resp.hits.hits[0]._source?.reason).eql(
+          'Average system.cpu.user.pct reported no data in the last 5m'
+        );
+        expect(resp.hits.hits[0]._source?.value).eql('[NO DATA]');
+
+        const parsedViewInAppUrl = parseSearchParams<LogsExplorerLocatorParsedParams>(
+          new URL(resp.hits.hits[0]._source?.viewInAppUrl || '').search
+        );
+
+        expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOGS_EXPLORER_LOCATOR');
+        expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({
+          dataset: DATA_VIEW_ID,
+          timeRange: { to: 'now' },
+          query: { query: '', language: 'kuery' },
+          filters: [],
+        });
+        expect(parsedViewInAppUrl.params.timeRange.from).match(ISO_DATE_REGEX);
+      });
     });
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts

@@ -20,6 +20,7 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/cus
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { ActionDocument } from './typings';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
@@ -38,6 +39,7 @@ export default function ({ getService }: FtrProviderContext) {
     let dataForgeIndices: string[];
     let actionId: string;
     let ruleId: string;
+    let alertId: string;
 
     before(async () => {
       dataForgeConfig = {
@@ -139,6 +141,9 @@ export default function ({ getService }: FtrProviderContext) {
                 documents: [
                   {
                     ruleType: '{{rule.type}}',
+                    alertDetailsUrl: '{{context.alertDetailsUrl}}',
+                    reason: '{{context.reason}}',
+                    value: '{{context.value}}',
                   },
                 ],
               },
@@ -173,6 +178,7 @@ export default function ({ getService }: FtrProviderContext) {
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });
+        alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
         expect(resp.hits.hits[0]._source).property(
           'kibana.alert.rule.category',
@@ -222,6 +228,22 @@ export default function ({ getService }: FtrProviderContext) {
             searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
           });
       });
+
+      it('should set correct action variables', async () => {
+        const resp = await alertingApi.waitForDocumentInIndex<ActionDocument>({
+          indexName: ALERT_ACTION_INDEX,
+          docCountTarget: 1,
+        });
+
+        expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
+        expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
+          `http://localhost:5620/app/observability/alerts/${alertId}`
+        );
+        expect(resp.hits.hits[0]._source?.reason).eql(
+          `Custom equation is 1 B, above the threshold of 0.9 B. (duration: 1 min, data view: ${DATA_VIEW})`
+        );
+        expect(resp.hits.hits[0]._source?.value).eql('1 B');
+      });
     });
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts

@@ -13,7 +13,11 @@ import {
 import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/custom_threshold/constants';
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
+import { parseSearchParams } from '@kbn/share-plugin/common/url_service';
+import { omit } from 'lodash';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { ISO_DATE_REGEX } from './constants';
+import { ActionDocument, LogsExplorerLocatorParsedParams } from './typings';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
@@ -25,20 +29,22 @@ export default function ({ getService }: FtrProviderContext) {
 
   describe('Custom Threshold rule - DOCUMENTS_COUNT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
-    const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
+    const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const DATA_VIEW_ID = 'data-view-id';
+    const DATA_VIEW_NAME = 'data-view-name';
     let dataForgeConfig: PartialConfig;
     let dataForgeIndices: string[];
     let actionId: string;
     let ruleId: string;
+    let alertId: string;
 
     before(async () => {
       dataForgeConfig = {
         schedule: [
           {
             template: 'good',
-            start: 'now-15m',
+            start: 'now-10m',
             end: 'now+5m',
             metrics: [
               { name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 },
@@ -55,9 +61,12 @@ export default function ({ getService }: FtrProviderContext) {
         },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
-      await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 60 });
+      await alertingApi.waitForDocumentInIndex({
+        indexName: dataForgeIndices.join(','),
+        docCountTarget: 45,
+      });
       await dataViewApi.create({
-        name: DATA_VIEW,
+        name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
         title: DATA_VIEW,
       });
@@ -89,8 +98,7 @@ export default function ({ getService }: FtrProviderContext) {
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/175499
+    describe('Rule creation', () => {
-    describe.skip('Rule creation', () => {
       it('creates rule successfully', async () => {
         actionId = await alertingApi.createIndexConnector({
           name: 'Index Connector: Threshold API test',
@@ -109,14 +117,14 @@ export default function ({ getService }: FtrProviderContext) {
                 threshold: [1, 2],
                 timeSize: 1,
                 timeUnit: 'm',
-                metrics: [{ name: 'A', filter: '', aggType: Aggregators.COUNT }],
+                metrics: [{ name: 'A', filter: 'container.id:*', aggType: Aggregators.COUNT }],
               },
             ],
             alertOnNoData: true,
             alertOnGroupDisappear: true,
             searchConfiguration: {
               query: {
-                query: '',
+                query: 'host.name:*',
                 language: 'kuery',
               },
               index: DATA_VIEW_ID,
@@ -130,6 +138,10 @@ export default function ({ getService }: FtrProviderContext) {
                 documents: [
                   {
                     ruleType: '{{rule.type}}',
+                    alertDetailsUrl: '{{context.alertDetailsUrl}}',
+                    reason: '{{context.reason}}',
+                    value: '{{context.value}}',
+                    viewInAppUrl: '{{context.viewInAppUrl}}',
                   },
                 ],
               },
@@ -164,6 +176,7 @@ export default function ({ getService }: FtrProviderContext) {
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });
+        alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
         expect(resp.hits.hits[0]._source).property(
           'kibana.alert.rule.category',
@@ -191,6 +204,7 @@ export default function ({ getService }: FtrProviderContext) {
         expect(resp.hits.hits[0]._source).property('kibana.alert.workflow_status', 'open');
         expect(resp.hits.hits[0]._source).property('event.kind', 'signal');
         expect(resp.hits.hits[0]._source).property('event.action', 'open');
+        expect(resp.hits.hits[0]._source).not.have.property('kibana.alert.group');
         expect(resp.hits.hits[0]._source).not.have.property('kibana.alert.evaluation.threshold');
         expect(resp.hits.hits[0]._source)
           .property('kibana.alert.rule.parameters')
@@ -201,14 +215,47 @@ export default function ({ getService }: FtrProviderContext) {
                 threshold: [1, 2],
                 timeSize: 1,
                 timeUnit: 'm',
-                metrics: [{ name: 'A', filter: '', aggType: 'count' }],
+                metrics: [{ name: 'A', filter: 'container.id:*', aggType: 'count' }],
               },
             ],
             alertOnNoData: true,
             alertOnGroupDisappear: true,
-            searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
+            searchConfiguration: {
+              index: 'data-view-id',
+              query: { query: 'host.name:*', language: 'kuery' },
+            },
           });
       });
+
+      it('should set correct action variables', async () => {
+        const resp = await alertingApi.waitForDocumentInIndex<ActionDocument>({
+          indexName: ALERT_ACTION_INDEX,
+          docCountTarget: 1,
+        });
+
+        expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
+        expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
+          `http://localhost:5620/app/observability/alerts/${alertId}`
+        );
+
+        expect(resp.hits.hits[0]._source?.reason).eql(
+          `Document count is 3, not between the threshold of 1 and 2. (duration: 1 min, data view: ${DATA_VIEW_NAME})`
+        );
+        expect(resp.hits.hits[0]._source?.value).eql('3');
+
+        const parsedViewInAppUrl = parseSearchParams<LogsExplorerLocatorParsedParams>(
+          new URL(resp.hits.hits[0]._source?.viewInAppUrl || '').search
+        );
+
+        expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOGS_EXPLORER_LOCATOR');
+        expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({
+          dataset: DATA_VIEW_ID,
+          timeRange: { to: 'now' },
+          query: { query: 'host.name:* and container.id:*', language: 'kuery' },
+          filters: [],
+        });
+        expect(parsedViewInAppUrl.params.timeRange.from).match(ISO_DATE_REGEX);
+      });
     });
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts

@@ -21,6 +21,7 @@ import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/cus
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { ActionDocument } from './typings';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
@@ -130,7 +131,7 @@ export default function ({ getService }: FtrProviderContext) {
               },
               index: DATA_VIEW_ID,
             },
-            groupBy: ['host.name'],
+            groupBy: ['host.name', 'container.id'],
           },
           actions: [
             {
@@ -144,6 +145,7 @@ export default function ({ getService }: FtrProviderContext) {
                     reason: '{{context.reason}}',
                     value: '{{context.value}}',
                     host: '{{context.host}}',
+                    group: '{{context.group}}',
                   },
                 ],
               },
@@ -202,7 +204,10 @@ export default function ({ getService }: FtrProviderContext) {
           'custom_threshold.fired'
         );
         expect(resp.hits.hits[0]._source).property('tags').contain('observability');
-        expect(resp.hits.hits[0]._source).property('kibana.alert.instance.id', 'host-0');
+        expect(resp.hits.hits[0]._source).property(
+          'kibana.alert.instance.id',
+          'host-0,container-0'
+        );
         expect(resp.hits.hits[0]._source).property('kibana.alert.workflow_status', 'open');
         expect(resp.hits.hits[0]._source).property('event.kind', 'signal');
         expect(resp.hits.hits[0]._source).property('event.action', 'open');
@@ -214,6 +219,18 @@ export default function ({ getService }: FtrProviderContext) {
         expect(resp.hits.hits[0]._source).property('container.id', 'container-0');
         expect(resp.hits.hits[0]._source).property('container.name', 'container-name');
         expect(resp.hits.hits[0]._source).not.property('container.cpu');
+        expect(resp.hits.hits[0]._source)
+          .property('kibana.alert.group')
+          .eql([
+            {
+              field: 'host.name',
+              value: 'host-0',
+            },
+            {
+              field: 'container.id',
+              value: 'container-0',
+            },
+          ]);
         expect(resp.hits.hits[0]._source).property('kibana.alert.evaluation.threshold').eql([0.2]);
         expect(resp.hits.hits[0]._source)
           .property('kibana.alert.rule.parameters')
@@ -230,18 +247,12 @@ export default function ({ getService }: FtrProviderContext) {
             alertOnNoData: true,
             alertOnGroupDisappear: true,
             searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
-            groupBy: ['host.name'],
+            groupBy: ['host.name', 'container.id'],
           });
       });
 
       it('should set correct action variables', async () => {
-        const resp = await alertingApi.waitForDocumentInIndex<{
+        const resp = await alertingApi.waitForDocumentInIndex<ActionDocument>({
-          ruleType: string;
-          alertDetailsUrl: string;
-          reason: string;
-          value: string;
-          host: string;
-        }>({
           indexName: ALERT_ACTION_INDEX,
         });
         const { protocol, hostname, port } = kbnTestConfig.getUrlParts();
@@ -251,12 +262,15 @@ export default function ({ getService }: FtrProviderContext) {
           `${protocol}://${hostname}:${port}/app/observability/alerts/${alertId}`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Average system.cpu.total.norm.pct is 80%, above or equal the threshold of 20%. (duration: 1 min, data view: ${DATA_VIEW}, group: host-0)`
+          `Average system.cpu.total.norm.pct is 80%, above or equal the threshold of 20%. (duration: 1 min, data view: ${DATA_VIEW}, group: host-0,container-0)`
         );
         expect(resp.hits.hits[0]._source?.value).eql('80%');
         expect(resp.hits.hits[0]._source?.host).eql(
           '{"name":"host-0","mac":["00-00-5E-00-53-23","00-00-5E-00-53-24"]}'
         );
+        expect(resp.hits.hits[0]._source?.group).eql(
+          '{"field":"host.name","value":"host-0"},{"field":"container.id","value":"container-0"}'
+        );
       });
     });
   });

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/index.ts

@@ -14,5 +14,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./documents_count_fired'));
     loadTestFile(require.resolve('./custom_eq_avg_bytes_fired'));
     loadTestFile(require.resolve('./group_by_fired'));
+    loadTestFile(require.resolve('./p99_pct_fired'));
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/p99_pct_fired.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { cleanup, generate } from '@kbn/infra-forge';
+import { cleanup, generate, Dataset, PartialConfig } from '@kbn/data-forge';
 import {
   Aggregators,
   Comparator,
@@ -13,37 +13,64 @@ import {
 import { FIRED_ACTIONS_ID } from '@kbn/observability-plugin/server/lib/rules/custom_threshold/constants';
 import expect from '@kbn/expect';
 import { OBSERVABILITY_THRESHOLD_RULE_TYPE_ID } from '@kbn/rule-data-utils';
+import { parseSearchParams } from '@kbn/share-plugin/common/url_service';
+import { omit } from 'lodash';
 import { FtrProviderContext } from '../../../ftr_provider_context';
+import { ISO_DATE_REGEX } from './constants';
+import { ActionDocument, LogsExplorerLocatorParsedParams } from './typings';
 
 export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
   const esDeleteAllIndices = getService('esDeleteAllIndices');
   const alertingApi = getService('alertingApi');
-  const dataViewApi = getService('dataViewApi');
   const logger = getService('log');
 
-  describe('Custom Threshold rule - P99 - BYTES - FIRED', () => {
+  describe('Custom Threshold rule - P99 - PCT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
-    // DATE_VIEW should match the index template:
-    // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json
-    const DATE_VIEW = 'kbn-data-forge-fake_hosts';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
+    const DATA_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    const DATA_VIEW_NAME = 'ad-hoc-data-view-name';
     const DATA_VIEW_ID = 'data-view-id';
-    let infraDataIndex: string;
+    const MOCKED_AD_HOC_DATA_VIEW = {
+      id: DATA_VIEW_ID,
+      title: DATA_VIEW_TITLE,
+      timeFieldName: '@timestamp',
+      sourceFilters: [],
+      fieldFormats: {},
+      runtimeFieldMap: {},
+      allowNoIndex: false,
+      name: DATA_VIEW_NAME,
+      allowHidden: false,
+    };
+    let dataForgeConfig: PartialConfig;
+    let dataForgeIndices: string[];
     let actionId: string;
     let ruleId: string;
+    let alertId: string;
 
     before(async () => {
-      infraDataIndex = await generate({
+      dataForgeConfig = {
-        esClient,
+        schedule: [
-        lookback: 'now-15m',
+          {
-        logger,
+            template: 'good',
-      });
+            start: 'now-10m',
-      await dataViewApi.create({
+            end: 'now+5m',
-        name: DATE_VIEW,
+            metrics: [{ name: 'system.cpu.user.pct', method: 'linear', start: 2.5, end: 2.5 }],
-        id: DATA_VIEW_ID,
+          },
-        title: DATE_VIEW,
+        ],
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 10000,
+          alignEventsToInterval: true,
+        },
+      };
+      dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
+      logger.info(JSON.stringify(dataForgeIndices.join(',')));
+      await alertingApi.waitForDocumentInIndex({
+        indexName: DATA_VIEW_TITLE,
+        docCountTarget: 270,
       });
     });
 
@@ -63,14 +90,11 @@ export default function ({ getService }: FtrProviderContext) {
       });
       await esClient.deleteByQuery({
         index: '.kibana-event-log-*',
-        query: { term: { 'rule.id': ruleId } },
+        query: { term: { 'kibana.alert.rule.consumer': 'logs' } },
         conflicts: 'proceed',
       });
-      await dataViewApi.delete({
+      await esDeleteAllIndices([ALERT_ACTION_INDEX, ...dataForgeIndices]);
-        id: DATA_VIEW_ID,
+      await cleanup({ client: esClient, config: dataForgeConfig, logger });
-      });
-      await esDeleteAllIndices([ALERT_ACTION_INDEX, infraDataIndex]);
-      await cleanup({ esClient, logger });
     });
 
     describe('Rule creation', () => {
@@ -89,12 +113,10 @@ export default function ({ getService }: FtrProviderContext) {
             criteria: [
               {
                 comparator: Comparator.GT,
-                threshold: [1],
+                threshold: [0.5],
                 timeSize: 5,
                 timeUnit: 'm',
-                metrics: [
+                metrics: [{ name: 'A', field: 'system.cpu.user.pct', aggType: Aggregators.P99 }],
-                  { name: 'A', field: 'system.network.in.bytes', aggType: Aggregators.P99 },
-                ],
               },
             ],
             alertOnNoData: true,
@@ -104,7 +126,7 @@ export default function ({ getService }: FtrProviderContext) {
                 query: '',
                 language: 'kuery',
               },
-              index: DATA_VIEW_ID,
+              index: MOCKED_AD_HOC_DATA_VIEW,
             },
           },
           actions: [
@@ -115,6 +137,11 @@ export default function ({ getService }: FtrProviderContext) {
                 documents: [
                   {
                     ruleType: '{{rule.type}}',
+                    alertDetailsUrl: '{{context.alertDetailsUrl}}',
+                    reason: '{{context.reason}}',
+                    value: '{{context.value}}',
+                    host: '{{context.host}}',
+                    viewInAppUrl: '{{context.viewInAppUrl}}',
                   },
                 ],
               },
@@ -149,6 +176,7 @@ export default function ({ getService }: FtrProviderContext) {
           indexName: CUSTOM_THRESHOLD_RULE_ALERT_INDEX,
           ruleId,
         });
+        alertId = (resp.hits.hits[0]._source as any)['kibana.alert.uuid'];
 
         expect(resp.hits.hits[0]._source).property(
           'kibana.alert.rule.category',
@@ -183,17 +211,49 @@ export default function ({ getService }: FtrProviderContext) {
             criteria: [
               {
                 comparator: '>',
-                threshold: [1],
+                threshold: [0.5],
                 timeSize: 5,
                 timeUnit: 'm',
-                metrics: [{ name: 'A', field: 'system.network.in.bytes', aggType: 'p99' }],
+                metrics: [{ name: 'A', field: 'system.cpu.user.pct', aggType: 'p99' }],
               },
             ],
             alertOnNoData: true,
             alertOnGroupDisappear: true,
-            searchConfiguration: { index: 'data-view-id', query: { query: '', language: 'kuery' } },
+            searchConfiguration: {
+              index: MOCKED_AD_HOC_DATA_VIEW,
+              query: { query: '', language: 'kuery' },
+            },
           });
       });
+
+      it('should set correct action variables', async () => {
+        const resp = await alertingApi.waitForDocumentInIndex<ActionDocument>({
+          indexName: ALERT_ACTION_INDEX,
+          docCountTarget: 1,
+        });
+
+        expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold');
+        expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
+          `http://localhost:5620/app/observability/alerts/${alertId}`
+        );
+        expect(resp.hits.hits[0]._source?.reason).eql(
+          `99th percentile of system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATA_VIEW_NAME})`
+        );
+        expect(resp.hits.hits[0]._source?.value).eql('250%');
+
+        const parsedViewInAppUrl = parseSearchParams<LogsExplorerLocatorParsedParams>(
+          new URL(resp.hits.hits[0]._source?.viewInAppUrl || '').search
+        );
+
+        expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOGS_EXPLORER_LOCATOR');
+        expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({
+          dataset: DATA_VIEW_TITLE,
+          timeRange: { to: 'now' },
+          query: { query: '', language: 'kuery' },
+          filters: [],
+        });
+        expect(parsedViewInAppUrl.params.timeRange.from).match(ISO_DATE_REGEX);
+      });
     });
   });
 }

x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/typings.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Query, TimeRange } from '@kbn/es-query';
+import { SerializableRecord } from '@kbn/utility-types';
+
+export interface ActionDocument {
+  ruleType: string;
+  alertDetailsUrl: string;
+  reason: string;
+  value: string;
+  viewInAppUrl: string;
+  host?: string;
+  group?: string;
+}
+
+export interface LogsExplorerLocatorParsedParams extends SerializableRecord {
+  dataset: string;
+  timeRange: TimeRange;
+  query: Query;
+}

x-pack/test_serverless/tsconfig.json

@@ -93,5 +93,8 @@
     "@kbn/infra-forge",
     "@kbn/reporting-common",
     "@kbn/slo-plugin",
+    "@kbn/share-plugin",
+    "@kbn/es-query",
+    "@kbn/utility-types",
   ]
 }