[APM] Query transaction metrics for alerts (#108167)

Closes #78122.

x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts

@@ -19,6 +19,7 @@ import {
   ALERT_REASON as ALERT_REASON_NON_TYPED,
   // @ts-expect-error
 } from '@kbn/rule-data-utils/target_node/technical_field_names';
+import { SearchAggregatedTransactionSetting } from '../../../common/aggregated_transactions';
 import { asDuration } from '../../../../observability/common/utils/formatters';
 import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server';
 import {
@@ -44,6 +45,7 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
 import { apmActionVariables } from './action_variables';
 import { alertingEsClient } from './alerting_es_client';
 import { RegisterRuleDependencies } from './register_apm_alerts';
+import { getDocumentTypeFilterForAggregatedTransactions } from '../helpers/aggregated_transactions';
 
 const ALERT_EVALUATION_THRESHOLD: typeof ALERT_EVALUATION_THRESHOLD_TYPED = ALERT_EVALUATION_THRESHOLD_NON_TYPED;
 const ALERT_EVALUATION_VALUE: typeof ALERT_EVALUATION_VALUE_TYPED = ALERT_EVALUATION_VALUE_NON_TYPED;
@@ -105,8 +107,19 @@ export function registerTransactionDurationAlertType({
         savedObjectsClient: services.savedObjectsClient,
       });
 
+      // only query transaction events when set to 'never',
+      // to prevent (likely) unnecessary blocking request
+      // in rule execution
+      const searchAggregatedTransactions =
+        config['xpack.apm.searchAggregatedTransactions'] !==
+        SearchAggregatedTransactionSetting.never;
+
+      const index = searchAggregatedTransactions
+        ? indices['apm_oss.metricsIndices']
+        : indices['apm_oss.transactionIndices'];
+
       const searchParams = {
-        index: indices['apm_oss.transactionIndices'],
+        index,
         body: {
           size: 0,
           query: {
@@ -119,9 +132,9 @@ export function registerTransactionDurationAlertType({
                     },
                   },
                 },
-                {
-                  term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction },
-                },
+                ...getDocumentTypeFilterForAggregatedTransactions(
+                  searchAggregatedTransactions
+                ),
                 { term: { [SERVICE_NAME]: alertParams.serviceName } },
                 {
                   term: {

x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts

@@ -45,6 +45,8 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices';
 import { apmActionVariables } from './action_variables';
 import { alertingEsClient } from './alerting_es_client';
 import { RegisterRuleDependencies } from './register_apm_alerts';
+import { SearchAggregatedTransactionSetting } from '../../../common/aggregated_transactions';
+import { getDocumentTypeFilterForAggregatedTransactions } from '../helpers/aggregated_transactions';
 import { asPercent } from '../../../../observability/common/utils/formatters';
 
 const ALERT_EVALUATION_THRESHOLD: typeof ALERT_EVALUATION_THRESHOLD_TYPED = ALERT_EVALUATION_THRESHOLD_NON_TYPED;
@@ -102,9 +104,20 @@ export function registerTransactionErrorRateAlertType({
           savedObjectsClient: services.savedObjectsClient,
         });
 
+        // only query transaction events when set to 'never',
+        // to prevent (likely) unnecessary blocking request
+        // in rule execution
+        const searchAggregatedTransactions =
+          config['xpack.apm.searchAggregatedTransactions'] !==
+          SearchAggregatedTransactionSetting.never;
+
+        const index = searchAggregatedTransactions
+          ? indices['apm_oss.metricsIndices']
+          : indices['apm_oss.transactionIndices'];
+
         const searchParams = {
-          index: indices['apm_oss.transactionIndices'],
-          size: 1,
+          index,
+          size: 0,
           body: {
             query: {
               bool: {
@@ -116,7 +129,9 @@ export function registerTransactionErrorRateAlertType({
                       },
                     },
                   },
-                  { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } },
+                  ...getDocumentTypeFilterForAggregatedTransactions(
+                    searchAggregatedTransactions
+                  ),
                   {
                     terms: {
                       [EVENT_OUTCOME]: [

x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts

@@ -43,9 +43,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
   const INDEXING_DELAY = 5000;
 
   const ALERTS_INDEX_TARGET = '.kibana-alerts-observability.apm.alerts*';
-  const APM_TRANSACTION_INDEX_NAME = 'apm-8.0.0-transaction';
+  const APM_METRIC_INDEX_NAME = 'apm-8.0.0-transaction';
 
-  const createTransactionEvent = (override: Record<string, any>) => {
+  const createTransactionMetric = (override: Record<string, any>) => {
     const now = Date.now();
 
     const time = now - INDEXING_DELAY;
@@ -61,12 +61,15 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         },
         transaction: {
           duration: {
-            us: 1000000,
+            histogram: {
+              values: [1000000],
+              counts: [1],
+            },
           },
           type: 'request',
         },
         processor: {
-          event: 'transaction',
+          event: 'metric',
         },
         observer: {
           version_major: 7,
@@ -141,7 +144,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
       before(async () => {
         await es.indices.create({
-          index: APM_TRANSACTION_INDEX_NAME,
+          index: APM_METRIC_INDEX_NAME,
           body: {
             mappings: {
               dynamic: 'strict',
@@ -184,8 +187,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
                     },
                     duration: {
                       properties: {
-                        us: {
-                          type: 'long',
+                        histogram: {
+                          type: 'histogram',
                         },
                       },
                     },
@@ -252,7 +255,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         });
 
         await es.indices.delete({
-          index: APM_TRANSACTION_INDEX_NAME,
+          index: APM_METRIC_INDEX_NAME,
         });
       });
 
@@ -281,8 +284,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         expect(beforeDataResponse.body.hits.hits.length).to.be(0);
 
         await es.index({
-          index: APM_TRANSACTION_INDEX_NAME,
-          body: createTransactionEvent({
+          index: APM_METRIC_INDEX_NAME,
+          body: createTransactionMetric({
             event: {
               outcome: 'success',
             },
@@ -310,8 +313,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         expect(afterInitialDataResponse.body.hits.hits.length).to.be(0);
 
         await es.index({
-          index: APM_TRANSACTION_INDEX_NAME,
-          body: createTransactionEvent({
+          index: APM_METRIC_INDEX_NAME,
+          body: createTransactionMetric({
             event: {
               outcome: 'failure',
             },
@@ -410,16 +413,16 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         `);
 
         await es.bulk({
-          index: APM_TRANSACTION_INDEX_NAME,
+          index: APM_METRIC_INDEX_NAME,
           body: [
             { index: {} },
-            createTransactionEvent({
+            createTransactionMetric({
               event: {
                 outcome: 'success',
               },
             }),
             { index: {} },
-            createTransactionEvent({
+            createTransactionMetric({
               event: {
                 outcome: 'success',
               },