feat(pending): pending configurations are returned

x-pack/solutions/observability/plugins/synthetics/common/runtime_types/alert_rules/common.ts

@@ -76,6 +76,7 @@ export const AlertStatusCodec = t.interface({
 
 export type StaleDownConfig = t.TypeOf<typeof StaleAlertStatusMetaDataCodec>;
 export type AlertStatusMetaData = t.TypeOf<typeof AlertStatusMetaDataCodec>;
+type AlertPendingStatusMetaData = t.TypeOf<typeof AlertPendingStatusMetaDataCodec>;
 export type AlertOverviewStatus = t.TypeOf<typeof AlertStatusCodec>;
 export type StatusRuleInspect = AlertOverviewStatus & {
   monitors: Array<{
@@ -86,3 +87,4 @@ export type StatusRuleInspect = AlertOverviewStatus & {
 };
 export type TLSRuleInspect = StatusRuleInspect;
 export type AlertStatusConfigs = Record<string, AlertStatusMetaData>;
+export type AlertPendingStatusConfigs = Record<string, AlertPendingStatusMetaData>;

x-pack/solutions/observability/plugins/synthetics/server/alert_rules/status_rule/queries/get_search_ping_params.ts

@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  AggregationsTopHitsAggregation,
+  QueryDslQueryContainer,
+} from '@elastic/elasticsearch/lib/api/types';
+import { createEsParams } from '../../../lib';
+import {
+  FINAL_SUMMARY_FILTER,
+  SUMMARY_FILTER,
+  getRangeFilter,
+} from '../../../../common/constants/client_defaults';
+
+export const getSearchPingsParams = ({
+  idsToQuery,
+  idSize,
+  monitorLocationIds,
+  range,
+  numberOfChecks,
+  includeRetests,
+}: {
+  idsToQuery: string[];
+  idSize: number;
+  monitorLocationIds: string[];
+  range: { from: string; to: string };
+  numberOfChecks: number;
+  includeRetests: boolean;
+}) => {
+  // Filters for pings
+  const queryFilters: QueryDslQueryContainer[] = [
+    ...(includeRetests ? [SUMMARY_FILTER] : [FINAL_SUMMARY_FILTER]),
+    getRangeFilter({ from: range.from, to: range.to }),
+    {
+      terms: {
+        'monitor.id': idsToQuery,
+      },
+    },
+  ];
+
+  // For each ping we want to get the monitor id as an aggregation and the location as a sub-aggregation
+  // The location aggregation will have a filter for down checks and a top hits aggregation to get the latest ping
+
+  // Total checks per location per monitor
+  const totalChecks: AggregationsTopHitsAggregation = {
+    size: numberOfChecks,
+    sort: [
+      {
+        '@timestamp': {
+          order: 'desc',
+        },
+      },
+    ],
+    _source: {
+      includes: [
+        '@timestamp',
+        'summary',
+        'monitor',
+        'observer',
+        'config_id',
+        'error',
+        'agent',
+        'url',
+        'state',
+        'tags',
+        'service',
+        'labels',
+      ],
+    },
+  };
+
+  // Down checks per location per monitor
+  const downChecks: QueryDslQueryContainer = {
+    range: {
+      'summary.down': {
+        gte: '1',
+      },
+    },
+  };
+
+  const locationAggs = {
+    downChecks: {
+      filter: downChecks,
+    },
+    totalChecks: {
+      top_hits: totalChecks,
+    },
+  };
+
+  const idAggs = {
+    location: {
+      terms: {
+        field: 'observer.name',
+        size: monitorLocationIds.length || 100,
+      },
+      aggs: locationAggs,
+    },
+  };
+
+  const pingAggs = {
+    id: {
+      terms: {
+        field: 'monitor.id',
+        size: idSize,
+      },
+      aggs: idAggs,
+    },
+  };
+
+  const params = createEsParams({
+    size: 0,
+    query: {
+      bool: {
+        filter: queryFilters,
+      },
+    },
+    aggs: pingAggs,
+  });
+
+  if (monitorLocationIds.length > 0) {
+    params.query.bool.filter.push({
+      terms: {
+        'observer.name': monitorLocationIds,
+      },
+    });
+  }
+
+  return params;
+};

x-pack/solutions/observability/plugins/synthetics/server/alert_rules/status_rule/queries/query_monitor_status_alert.ts

@@ -7,41 +7,105 @@
 
 import pMap from 'p-map';
 import { times } from 'lodash';
-import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import { intersection } from 'lodash';
-import { AlertStatusMetaData } from '../../../../common/runtime_types/alert_rules/common';
+import moment from 'moment';
 import {
-  FINAL_SUMMARY_FILTER,
+  AlertStatusConfigs,
-  getRangeFilter,
+  AlertStatusMetaData,
-  SUMMARY_FILTER,
+  AlertPendingStatusConfigs,
-} from '../../../../common/constants/client_defaults';
+} from '../../../../common/runtime_types/alert_rules/common';
+
 import { OverviewPing } from '../../../../common/runtime_types';
-import { createEsParams, SyntheticsEsClient } from '../../../lib';
+import { SyntheticsEsClient } from '../../../lib';
+import { getSearchPingsParams } from './get_search_ping_params';
 
 const DEFAULT_MAX_ES_BUCKET_SIZE = 10000;
 
-const fields = [
-  '@timestamp',
-  'summary',
-  'monitor',
-  'observer',
-  'config_id',
-  'error',
-  'agent',
-  'url',
-  'state',
-  'tags',
-  'service',
-  'labels',
-];
-type StatusConfigs = Record<string, AlertStatusMetaData>;
-
 export interface AlertStatusResponse {
-  upConfigs: StatusConfigs;
+  upConfigs: AlertStatusConfigs;
-  downConfigs: StatusConfigs;
+  downConfigs: AlertStatusConfigs;
+  pendingConfigs: AlertPendingStatusConfigs;
   enabledMonitorQueryIds: string[];
 }
 
+const getPendingConfigs = async ({
+  monitorQueryIds,
+  monitorLocationIds,
+  esClient,
+  includeRetests,
+  upConfigs,
+  downConfigs,
+  monitorLocationsMap,
+}: {
+  monitorQueryIds: string[];
+  monitorLocationIds: string[];
+  esClient: SyntheticsEsClient;
+  includeRetests: boolean;
+  upConfigs: AlertStatusConfigs;
+  downConfigs: AlertStatusConfigs;
+  monitorLocationsMap: Record<string, string[]>;
+}) => {
+  // Check if a config is missing, if it is it means that the monitor is pending
+  const pendingConfigs: AlertPendingStatusConfigs = {};
+  const idsToQuery: Set<string> = new Set();
+  const locationsToQuery: Set<string> = new Set();
+
+  for (const monitorQueryId of monitorQueryIds) {
+    for (const locationId of monitorLocationIds) {
+      const configWithLocationId = `${monitorQueryId}-${locationId}`;
+
+      const isConfigMissing =
+        !upConfigs[configWithLocationId] &&
+        !downConfigs[configWithLocationId] &&
+        monitorLocationsMap[monitorQueryId]?.includes(locationId);
+
+      if (isConfigMissing) {
+        // Add the monitor and location ids to fetch the latest ping
+        // for the pending config
+        idsToQuery.add(monitorQueryId);
+        locationsToQuery.add(locationId);
+        // Add a temporary pending config, this will be updated if a ping is found
+        // If a monitor has no pings the config will not be updated
+        pendingConfigs[configWithLocationId] = {
+          status: 'pending',
+          configId: monitorQueryId,
+          monitorQueryId,
+          locationId,
+        };
+      }
+    }
+  }
+
+  // Get the last ping for the pending configs in the last month
+  const params = getSearchPingsParams({
+    idSize: Array.from(idsToQuery).length,
+    idsToQuery: Array.from(idsToQuery),
+    monitorLocationIds: Array.from(locationsToQuery),
+    numberOfChecks: 1,
+    includeRetests,
+    range: { from: moment().subtract(1, 'M').toISOString(), to: 'now' },
+  });
+
+  const {
+    body: { aggregations },
+  } = await esClient.search<OverviewPing, typeof params>(params);
+
+  aggregations?.id.buckets.forEach(({ location, key: monitorQueryId }) => {
+    location.buckets.forEach(({ key: locationId, totalChecks }) => {
+      const latestPing = totalChecks.hits.hits[0]._source;
+      const configWithLocationId = `${monitorQueryId}-${locationId}`;
+
+      pendingConfigs[configWithLocationId] = {
+        ...pendingConfigs[configWithLocationId],
+        ping: latestPing,
+        timestamp: latestPing['@timestamp'],
+      };
+    });
+  });
+
+  return pendingConfigs;
+};
+
 export async function queryMonitorStatusAlert({
   esClient,
   monitorLocationIds,
@@ -50,6 +114,8 @@ export async function queryMonitorStatusAlert({
   monitorLocationsMap,
   numberOfChecks,
   includeRetests = true,
+  scheduleInMsMap,
+  waitSecondsBeforeIsPending = 60,
 }: {
   esClient: SyntheticsEsClient;
   monitorLocationIds: string[];
@@ -58,82 +124,27 @@ export async function queryMonitorStatusAlert({
   monitorLocationsMap: Record<string, string[]>;
   numberOfChecks: number;
   includeRetests?: boolean;
+  scheduleInMsMap: Record<string, number>;
+  waitSecondsBeforeIsPending?: number;
 }): Promise<AlertStatusResponse> {
   const idSize = Math.trunc(DEFAULT_MAX_ES_BUCKET_SIZE / monitorLocationIds.length || 1);
   const pageCount = Math.ceil(monitorQueryIds.length / idSize);
-  const upConfigs: StatusConfigs = {};
+  const upConfigs: AlertStatusConfigs = {};
-  const downConfigs: StatusConfigs = {};
+  const downConfigs: AlertStatusConfigs = {};
 
   await pMap(
     times(pageCount),
     async (i) => {
       const idsToQuery = (monitorQueryIds as string[]).slice(i * idSize, i * idSize + idSize);
-      const params = createEsParams({
-        size: 0,
-        query: {
-          bool: {
-            filter: [
-              ...(includeRetests ? [SUMMARY_FILTER] : [FINAL_SUMMARY_FILTER]),
-              getRangeFilter({ from: range.from, to: range.to }),
-              {
-                terms: {
-                  'monitor.id': idsToQuery,
-                },
-              },
-            ] as QueryDslQueryContainer[],
-          },
-        },
-        aggs: {
-          id: {
-            terms: {
-              field: 'monitor.id',
-              size: idSize,
-            },
-            aggs: {
-              location: {
-                terms: {
-                  field: 'observer.name',
-                  size: monitorLocationIds.length || 100,
-                },
-                aggs: {
-                  downChecks: {
-                    filter: {
-                      range: {
-                        'summary.down': {
-                          gte: '1',
-                        },
-                      },
-                    },
-                  },
-                  totalChecks: {
-                    top_hits: {
-                      size: numberOfChecks,
-                      sort: [
-                        {
-                          '@timestamp': {
-                            order: 'desc',
-                          },
-                        },
-                      ],
-                      _source: {
-                        includes: fields,
-                      },
-                    },
-                  },
-                },
-              },
-            },
-          },
-        },
-      });
 
-      if (monitorLocationIds.length > 0) {
+      const params = getSearchPingsParams({
-        params.query.bool.filter.push({
+        idSize,
-          terms: {
+        idsToQuery,
-            'observer.name': monitorLocationIds,
+        monitorLocationIds,
-          },
+        range,
-        });
+        numberOfChecks,
-      }
+        includeRetests,
+      });
 
       const { body: result } = await esClient.search<OverviewPing, typeof params>(
         params,
@@ -164,6 +175,16 @@ export async function queryMonitorStatusAlert({
             const configId = latestPing.config_id;
             const monitorQueryId = latestPing.monitor.id;
 
+            const msSinceLastPing =
+              new Date().getTime() - new Date(latestPing['@timestamp']).getTime();
+            const msBeforeIsPending =
+              scheduleInMsMap[monitorQueryId] +
+              moment.duration(waitSecondsBeforeIsPending, 'seconds').asMilliseconds();
+
+            // Example: if a monitor has a schedule of 5m and the waitSecondsBeforeIsPending is 1m the last valid ping can be at (5+1)m
+            // If it's greater than that it means the monitor is pending
+            const isValidPing = msBeforeIsPending - msSinceLastPing > 0;
+
             const meta: AlertStatusMetaData = {
               ping: latestPing,
               configId,
@@ -180,13 +201,13 @@ export async function queryMonitorStatusAlert({
               status: 'up',
             };
 
-            if (downCount > 0) {
+            if (isValidPing && downCount > 0) {
               downConfigs[`${configId}-${monLocationId}`] = {
                 ...meta,
                 status: 'down',
               };
             }
-            if (isLatestPingUp) {
+            if (isValidPing && isLatestPingUp) {
               upConfigs[`${configId}-${monLocationId}`] = {
                 ...meta,
                 status: 'up',
@@ -199,9 +220,20 @@ export async function queryMonitorStatusAlert({
     { concurrency: 5 }
   );
 
+  const pendingConfigs = await getPendingConfigs({
+    monitorQueryIds,
+    monitorLocationIds,
+    esClient,
+    includeRetests,
+    upConfigs,
+    downConfigs,
+    monitorLocationsMap,
+  });
+
   return {
     upConfigs,
     downConfigs,
+    pendingConfigs,
     enabledMonitorQueryIds: monitorQueryIds,
   };
 }

x-pack/solutions/observability/plugins/synthetics/server/alert_rules/status_rule/status_rule_executor.ts

@@ -150,8 +150,13 @@ export class StatusRuleExecutor {
   }
 
   async getDownChecks(prevDownConfigs: AlertStatusConfigs = {}): Promise<AlertOverviewStatus> {
-    const { enabledMonitorQueryIds, maxPeriod, monitorLocationIds, monitorLocationsMap } =
+    const {
-      await this.init();
+      enabledMonitorQueryIds,
+      maxPeriod,
+      monitorLocationIds,
+      monitorLocationsMap,
+      scheduleInMsMap,
+    } = await this.init();
 
     const range = this.getRange(maxPeriod);
 
@@ -185,14 +190,15 @@ export class StatusRuleExecutor {
       numberOfChecks,
       monitorLocationsMap,
       includeRetests: this.params.condition?.includeRetests,
+      scheduleInMsMap,
     });
 
-    const { downConfigs, upConfigs } = currentStatus;
+    const { downConfigs, upConfigs, pendingConfigs } = currentStatus;
 
     this.debug(
-      `Found ${Object.keys(downConfigs).length} down configs and ${
+      `Found ${Object.keys(downConfigs).length} down configs, ${
         Object.keys(upConfigs).length
-      } up configs`
+      } up configs and ${Object.keys(pendingConfigs).length} pending configs`
     );
 
     const downConfigsById = getConfigsByIds(downConfigs);
@@ -218,7 +224,6 @@ export class StatusRuleExecutor {
     return {
       ...currentStatus,
       staleDownConfigs,
-      pendingConfigs: {},
       maxPeriod,
     };
   }

x-pack/solutions/observability/plugins/synthetics/server/saved_objects/synthetics_monitor/get_all_monitors.ts

@@ -28,7 +28,7 @@ export const processMonitors = (
   const disabledMonitorQueryIds: string[] = [];
   let disabledCount = 0;
   let disabledMonitorsCount = 0;
-  let maxPeriod = 0;
+  const scheduleInMsMap: Record<string, number> = {};
   let projectMonitorsCount = 0;
   const allIds: string[] = [];
   let listOfLocationsSet = new Set<string>();
@@ -63,12 +63,12 @@ export const processMonitors = (
         : monitorLocIds;
       listOfLocationsSet = new Set([...listOfLocationsSet, ...monitorLocIds]);
 
-      maxPeriod = Math.max(maxPeriod, periodToMs(attrs[ConfigKey.SCHEDULE]));
+      scheduleInMsMap[attrs[ConfigKey.MONITOR_QUERY_ID]] = periodToMs(attrs[ConfigKey.SCHEDULE]);
     }
   }
 
   return {
-    maxPeriod,
+    maxPeriod: Math.max(...Object.values(scheduleInMsMap)),
     allIds,
     enabledMonitorQueryIds,
     disabledMonitorQueryIds,
@@ -78,5 +78,6 @@ export const processMonitors = (
     projectMonitorsCount,
     monitorLocationIds: [...listOfLocationsSet],
     monitorQueryIdToConfigIdMap,
+    scheduleInMsMap,
   };
 };