Adds additional context to recovered alerts of Infrastructure rules (#144683)

## Summary

Closes https://github.com/elastic/kibana/issues/143725,
https://github.com/elastic/kibana/issues/143726

This PR reads the context variables indexed in Alerts-As-Data for
`Infrastructure Rules` and adds it to context when alerts are recovered.

Also, the context variables are now flattened before being indexed in
AAD for both of the `Infrastructure Rules`.

## The context newly added for recovered alerts with this PR
- `cloud.*`
- `host.*`
   - Excluding:
      - `host.cpu.*`
      - `host.disk.*`
      - `host.network.*`
- `orchestrator.*`
- `container.*`
- `labels.*`
- `tags`

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/infra/server/lib/alerting/common/utils.ts

@@ -14,6 +14,7 @@ import { ObservabilityConfig } from '@kbn/observability-plugin/server';
 import { ALERT_RULE_PARAMETERS, TIMESTAMP } from '@kbn/rule-data-utils';
 import { parseTechnicalFields } from '@kbn/rule-registry-plugin/common/parse_technical_fields';
 import { ES_FIELD_TYPES } from '@kbn/field-types';
+import { set } from '@kbn/safer-lodash-set';
 import { LINK_TO_METRICS_EXPLORER } from '../../../../common/alerting/metrics';
 import { getInventoryViewInAppUrl } from '../../../../common/alerting/metrics/alert_link';
 import {
@@ -21,6 +22,18 @@ import {
   InventoryMetricConditions,
 } from '../../../../common/alerting/metrics/types';
 
+const ALERT_CONTEXT_CONTAINER = 'container';
+const ALERT_CONTEXT_ORCHESTRATOR = 'orchestrator';
+const ALERT_CONTEXT_CLOUD = 'cloud';
+const ALERT_CONTEXT_HOST = 'host';
+const ALERT_CONTEXT_LABELS = 'labels';
+const ALERT_CONTEXT_TAGS = 'tags';
+
+const HOST_NAME = 'host.name';
+const HOST_HOSTNAME = 'host.hostname';
+const HOST_ID = 'host.id';
+const CONTAINER_ID = 'container.id';
+
 const SUPPORTED_ES_FIELD_TYPES = [
   ES_FIELD_TYPES.KEYWORD,
   ES_FIELD_TYPES.IP,
@@ -142,11 +155,6 @@ export const getAlertDetailsUrl = (
   alertUuid: string | null
 ) => addSpaceIdToPath(basePath.publicBaseUrl, spaceId, `/app/observability/alerts/${alertUuid}`);
 
-const HOST_NAME = 'host.name';
-const HOST_HOSTNAME = 'host.hostname';
-const HOST_ID = 'host.id';
-const CONTAINER_ID = 'container.id';
-
 export const KUBERNETES_POD_UID = 'kubernetes.pod.uid';
 export const NUMBER_OF_DOCUMENTS = 10;
 export const termsAggField: Record<string, string> = { [KUBERNETES_POD_UID]: CONTAINER_ID };
@@ -210,3 +218,79 @@ export const shouldTermsAggOnContainer = (groupBy: string | string[] | undefined
     ? groupBy.includes(KUBERNETES_POD_UID)
     : groupBy === KUBERNETES_POD_UID;
 };
+
+export const flattenAdditionalContext = (
+  additionalContext: AdditionalContext | undefined | null
+): AdditionalContext => {
+  let flattenedContext: AdditionalContext = {};
+  if (additionalContext) {
+    Object.keys(additionalContext).forEach((context: string) => {
+      if (additionalContext[context]) {
+        flattenedContext = {
+          ...flattenedContext,
+          ...flattenObject(additionalContext[context], [context + '.']),
+        };
+      }
+    });
+  }
+  return flattenedContext;
+};
+
+export const getContextForRecoveredAlerts = (
+  alertHits: AdditionalContext | undefined | null
+): AdditionalContext => {
+  const alertHitsSource =
+    alertHits && alertHits.length > 0 ? unflattenObject(alertHits[0]._source) : undefined;
+
+  return {
+    cloud: alertHitsSource?.[ALERT_CONTEXT_CLOUD],
+    host: alertHitsSource?.[ALERT_CONTEXT_HOST],
+    orchestrator: alertHitsSource?.[ALERT_CONTEXT_ORCHESTRATOR],
+    container: alertHitsSource?.[ALERT_CONTEXT_CONTAINER],
+    labels: alertHitsSource?.[ALERT_CONTEXT_LABELS],
+    tags: alertHitsSource?.[ALERT_CONTEXT_TAGS],
+  };
+};
+
+export const unflattenObject = <T extends object = AdditionalContext>(object: object): T =>
+  Object.entries(object).reduce((acc, [key, value]) => {
+    set(acc, key, value);
+    return acc;
+  }, {} as T);
+
+/**
+ * Wrap the key with [] if it is a key from an Array
+ * @param key The object key
+ * @param isArrayItem Flag to indicate if it is the key of an Array
+ */
+const renderKey = (key: string, isArrayItem: boolean): string => (isArrayItem ? `[${key}]` : key);
+
+export const flattenObject = (
+  obj: AdditionalContext,
+  prefix: string[] = [],
+  isArrayItem = false
+): AdditionalContext =>
+  Object.keys(obj).reduce<AdditionalContext>((acc, k) => {
+    const nextValue = obj[k];
+
+    if (typeof nextValue === 'object' && nextValue !== null) {
+      const isNextValueArray = Array.isArray(nextValue);
+      const dotSuffix = isNextValueArray ? '' : '.';
+
+      if (Object.keys(nextValue).length > 0) {
+        return {
+          ...acc,
+          ...flattenObject(
+            nextValue,
+            [...prefix, `${renderKey(k, isArrayItem)}${dotSuffix}`],
+            isNextValueArray
+          ),
+        };
+      }
+    }
+
+    const fullPath = `${prefix.join('')}${renderKey(k, isArrayItem)}`;
+    acc[fullPath] = nextValue;
+
+    return acc;
+  }, {});

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/inventory_metric_threshold_executor.ts

@@ -33,7 +33,9 @@ import {
 import {
   AdditionalContext,
   createScopedLogger,
+  flattenAdditionalContext,
   getAlertDetailsUrl,
+  getContextForRecoveredAlerts,
   getViewInInventoryAppUrl,
   UNGROUPED_FACTORY_KEY,
 } from '../common/utils';
@@ -81,14 +83,20 @@ export const createInventoryMetricThresholdExecutor = (libs: InfraBackendLibs) =
 
     const esClient = services.scopedClusterClient.asCurrentUser;
 
-    const { alertWithLifecycle, savedObjectsClient, getAlertStartedDate, getAlertUuid } = services;
+    const {
+      alertWithLifecycle,
+      savedObjectsClient,
+      getAlertStartedDate,
+      getAlertUuid,
+      getAlertByAlertUuid,
+    } = services;
     const alertFactory: InventoryMetricThresholdAlertFactory = (id, reason, additionalContext) =>
       alertWithLifecycle({
         id,
         fields: {
           [ALERT_REASON]: reason,
           [ALERT_RULE_PARAMETERS]: params as any, // the type assumes the object is already flattened when writing the same way as when reading https://github.com/elastic/kibana/blob/main/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts#L60
-          ...additionalContext,
+          ...flattenAdditionalContext(additionalContext),
         },
       });
 
@@ -246,6 +254,8 @@ export const createInventoryMetricThresholdExecutor = (libs: InfraBackendLibs) =
       const recoveredAlertId = alert.getId();
       const indexedStartedDate = getAlertStartedDate(recoveredAlertId) ?? startedAt.toISOString();
       const alertUuid = getAlertUuid(recoveredAlertId);
+      const alertHits = alertUuid ? await getAlertByAlertUuid(alertUuid) : undefined;
+      const additionalContext = getContextForRecoveredAlerts(alertHits);
 
       alert.setContext({
         alertDetailsUrl: getAlertDetailsUrl(libs.basePath, spaceId, alertUuid),
@@ -261,6 +271,7 @@ export const createInventoryMetricThresholdExecutor = (libs: InfraBackendLibs) =
           timestamp: indexedStartedDate,
           spaceId,
         }),
+        ...additionalContext,
       });
     }
 

x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.ts

@@ -30,10 +30,12 @@ import {
   createScopedLogger,
   AdditionalContext,
   getAlertDetailsUrl,
+  getContextForRecoveredAlerts,
   getViewInMetricsAppUrl,
   UNGROUPED_FACTORY_KEY,
   hasAdditionalContext,
   validGroupByForContext,
+  flattenAdditionalContext,
 } from '../common/utils';
 
 import { EvaluatedRuleParams, evaluateRule } from './lib/evaluate_rule';
@@ -96,14 +98,14 @@ export const createMetricThresholdExecutor = (libs: InfraBackendLibs) =>
       executionId,
     });
 
-    const { alertWithLifecycle, savedObjectsClient, getAlertUuid } = services;
+    const { alertWithLifecycle, savedObjectsClient, getAlertUuid, getAlertByAlertUuid } = services;
 
     const alertFactory: MetricThresholdAlertFactory = (id, reason, additionalContext) =>
       alertWithLifecycle({
         id,
         fields: {
           [ALERT_REASON]: reason,
-          ...additionalContext,
+          ...flattenAdditionalContext(additionalContext),
         },
       });
 
@@ -299,6 +301,9 @@ export const createMetricThresholdExecutor = (libs: InfraBackendLibs) =>
       const recoveredAlertId = alert.getId();
       const alertUuid = getAlertUuid(recoveredAlertId);
 
+      const alertHits = alertUuid ? await getAlertByAlertUuid(alertUuid) : undefined;
+      const additionalContext = getContextForRecoveredAlerts(alertHits);
+
       alert.setContext({
         alertDetailsUrl: getAlertDetailsUrl(libs.basePath, spaceId, alertUuid),
         alertState: stateToAlertMessage[AlertStates.OK],
@@ -307,6 +312,7 @@ export const createMetricThresholdExecutor = (libs: InfraBackendLibs) =>
         timestamp: startedAt.toISOString(),
         threshold: mapToConditionsLookup(criteria, (c) => c.threshold),
         viewInAppUrl: getViewInMetricsAppUrl(libs.basePath, spaceId),
+        ...additionalContext,
       });
     }
 

x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts

@@ -44,6 +44,7 @@ import { IRuleDataClient } from '../rule_data_client';
 import { AlertExecutorOptionsWithExtraServices } from '../types';
 import { fetchExistingAlerts } from './fetch_existing_alerts';
 import { getCommonAlertFields } from './get_common_alert_fields';
+import { fetchAlertByAlertUUID } from './fetch_alert_by_uuid';
 
 type ImplicitTechnicalFieldName = CommonAlertFieldNameLatest | CommonAlertIdFieldNameLatest;
 
@@ -71,6 +72,7 @@ export interface LifecycleAlertServices<
   alertWithLifecycle: LifecycleAlertService<InstanceState, InstanceContext, ActionGroupIds>;
   getAlertStartedDate: (alertInstanceId: string) => string | null;
   getAlertUuid: (alertInstanceId: string) => string | null;
+  getAlertByAlertUuid: (alertUuid: string) => { [x: string]: any } | null;
 }
 
 export type LifecycleRuleExecutor<
@@ -182,6 +184,13 @@ export const createLifecycleExecutor =
 
         return state.trackedAlerts[alertId].alertUuid;
       },
+      getAlertByAlertUuid: async (alertUuid: string) => {
+        try {
+          return await fetchAlertByAlertUUID(ruleDataClient, alertUuid);
+        } catch (err) {
+          return null;
+        }
+      },
     };
 
     const nextWrappedState = await wrappedExecutor({

x-pack/plugins/rule_registry/server/utils/fetch_alert_by_uuid.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PublicContract } from '@kbn/utility-types';
+import { IRuleDataClient } from '../rule_data_client';
+import { ALERT_UUID } from '../../common/technical_rule_data_field_names';
+
+type RuleDataClient = PublicContract<IRuleDataClient>;
+
+export const fetchAlertByAlertUUID = async (ruleDataClient: RuleDataClient, alertUuid: string) => {
+  const request = {
+    body: {
+      query: {
+        bool: {
+          filter: [
+            {
+              term: {
+                [ALERT_UUID]: alertUuid,
+              },
+            },
+          ],
+        },
+      },
+      size: 1,
+    },
+    allow_no_indices: true,
+  };
+  const { hits } = await ruleDataClient.getReader().search(request);
+  return hits?.hits;
+};

x-pack/plugins/rule_registry/server/utils/lifecycle_alert_services.mock.ts

@@ -37,4 +37,5 @@ export const createLifecycleAlertServicesMock = <
   alertWithLifecycle: ({ id }) => alertServices.alertFactory.create(id),
   getAlertStartedDate: jest.fn((id: string) => null),
   getAlertUuid: jest.fn((id: string) => null),
+  getAlertByAlertUuid: jest.fn((id: string) => null),
 });