[Security Solution] Add Active Endpoint Count to Usage Collector (#145024)

## Summary

Added active endpoint count to usage collector. Endpoint count is
technically already being counted via the daily usage counter; however,
it is counted during the execution of the endpoint task which could
potentially stall/timeout or even fail leading to inconsistent reporting
of active endpoint counts(thanks to @pjhampton for bringing this up and
suggesting to add this to the usage collector)


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/security_solution/common/constants.ts

@@ -56,6 +56,7 @@ export const SCROLLING_DISABLED_CLASS_NAME = 'scrolling-disabled' as const;
 export const FULL_SCREEN_TOGGLED_CLASS_NAME = 'fullScreenToggled' as const;
 export const NO_ALERT_INDEX = 'no-alert-index-049FC71A-4C2C-446F-9901-37XMC5024C51' as const;
 export const ENDPOINT_METADATA_INDEX = 'metrics-endpoint.metadata-*' as const;
+export const ENDPOINT_METRICS_INDEX = '.ds-metrics-endpoint.metrics-*' as const;
 export const DEFAULT_RULE_REFRESH_INTERVAL_ON = true as const;
 export const DEFAULT_RULE_REFRESH_INTERVAL_VALUE = 60000 as const; // ms
 export const DEFAULT_RULE_NOTIFICATION_QUERY_SIZE = 100 as const;

x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -58,6 +58,7 @@ import type {
   ValueListIndicatorMatchResponseAggregation,
 } from './types';
 import { telemetryConfiguration } from './configuration';
+import { ENDPOINT_METRICS_INDEX } from '../../../common/constants';
 
 export interface ITelemetryReceiver {
   start(
@@ -277,7 +278,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
 
     const query: SearchRequest = {
       expand_wildcards: ['open' as const, 'hidden' as const],
-      index: `.ds-metrics-endpoint.metrics-*`,
+      index: ENDPOINT_METRICS_INDEX,
       ignore_unavailable: false,
       body: {
         size: 0, // no query results required - only aggregation quantity

x-pack/plugins/security_solution/server/usage/collector.ts

@@ -9,11 +9,13 @@ import type { CollectorFetchContext } from '@kbn/usage-collection-plugin/server'
 import type { CollectorDependencies } from './types';
 import { getDetectionsMetrics } from './detections/get_metrics';
 import { getInternalSavedObjectsClient } from './get_internal_saved_objects_client';
+import { getEndpointMetrics } from './endpoint/get_metrics';
 
 export type RegisterCollector = (deps: CollectorDependencies) => void;
 
 export interface UsageData {
   detectionMetrics: {};
+  endpointMetrics: {};
 }
 
 export const registerCollector: RegisterCollector = ({
@@ -2397,20 +2399,30 @@ export const registerCollector: RegisterCollector = ({
           },
         },
       },
+      endpointMetrics: {
+        unique_endpoint_count: {
+          type: 'long',
+          _meta: { description: 'Number of active unique endpoints in last 24 hours' },
+        },
+      },
     },
     isReady: () => true,
     fetch: async ({ esClient }: CollectorFetchContext): Promise<UsageData> => {
       const savedObjectsClient = await getInternalSavedObjectsClient(core);
-      const detectionMetrics = await getDetectionsMetrics({
-        eventLogIndex,
-        signalsIndex,
-        esClient,
-        savedObjectsClient,
-        logger,
-        mlClient: ml,
-      });
+      const [detectionMetrics, endpointMetrics] = await Promise.allSettled([
+        getDetectionsMetrics({
+          eventLogIndex,
+          signalsIndex,
+          esClient,
+          savedObjectsClient,
+          logger,
+          mlClient: ml,
+        }),
+        getEndpointMetrics({ esClient, logger }),
+      ]);
       return {
-        detectionMetrics: detectionMetrics || {},
+        detectionMetrics: detectionMetrics.status === 'fulfilled' ? detectionMetrics.value : {},
+        endpointMetrics: endpointMetrics.status === 'fulfilled' ? endpointMetrics.value : {},
       };
     },
   });

x-pack/plugins/security_solution/server/usage/endpoint/get_metrics.mocks.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types';
+import type { AggregationsAggregate } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+export const getUniqueEndpointCountMock = (): SearchResponse<
+  unknown,
+  Record<string, AggregationsAggregate>
+> => ({
+  took: 495,
+  timed_out: false,
+  _shards: {
+    total: 1,
+    successful: 1,
+    skipped: 0,
+    failed: 0,
+  },
+  hits: {
+    max_score: null,
+    hits: [],
+  },
+  aggregations: {
+    endpoint_count: { value: 3 },
+  },
+});

x-pack/plugins/security_solution/server/usage/endpoint/get_metrics.test.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
+import { getEndpointMetrics, getUniqueEndpointCount } from './get_metrics';
+import { getUniqueEndpointCountMock } from './get_metrics.mocks';
+import type { EndpointMetrics } from './types';
+
+describe('Endpoint Metrics', () => {
+  let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
+  let logger: ReturnType<typeof loggingSystemMock.createLogger>;
+
+  describe('getEndpointMetrics()', () => {
+    beforeEach(() => {
+      esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+      logger = loggingSystemMock.createLogger();
+    });
+
+    it('returns accurate active unique endpoint count', async () => {
+      esClient.search.mockResponseOnce(getUniqueEndpointCountMock());
+      const result = await getEndpointMetrics({
+        esClient,
+        logger,
+      });
+      expect(result).toEqual<EndpointMetrics>({
+        unique_endpoint_count: 3,
+      });
+    });
+  });
+  describe('getUniqueEndpointCount()', () => {
+    beforeEach(() => {
+      esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
+      logger = loggingSystemMock.createLogger();
+    });
+
+    it('returns unique endpoint count', async () => {
+      esClient.search.mockResponseOnce(getUniqueEndpointCountMock());
+      const result = await getUniqueEndpointCount(esClient, logger);
+      expect(esClient.search).toHaveBeenCalled();
+      expect(result).toEqual(3);
+    });
+
+    it('returns 0 on error', async () => {
+      esClient.search.mockRejectedValueOnce(new Error('Connection Error'));
+      const result = await getUniqueEndpointCount(esClient, logger);
+      expect(esClient.search).toHaveBeenCalled();
+      expect(result).toEqual(0);
+    });
+  });
+});

x-pack/plugins/security_solution/server/usage/endpoint/get_metrics.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient, Logger } from '@kbn/core/server';
+import type { SearchRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { EndpointMetrics, UniqueEndpointCountResponse } from './types';
+import { ENDPOINT_METRICS_INDEX } from '../../../common/constants';
+import { tlog } from '../../lib/telemetry/helpers';
+
+export interface GetEndpointMetricsOptions {
+  esClient: ElasticsearchClient;
+  logger: Logger;
+}
+
+export const getEndpointMetrics = async ({
+  esClient,
+  logger,
+}: GetEndpointMetricsOptions): Promise<EndpointMetrics> => {
+  return {
+    unique_endpoint_count: await getUniqueEndpointCount(esClient, logger),
+  };
+};
+
+export const getUniqueEndpointCount = async (
+  esClient: ElasticsearchClient,
+  logger: Logger
+): Promise<number> => {
+  try {
+    const query: SearchRequest = {
+      expand_wildcards: ['open' as const, 'hidden' as const],
+      index: ENDPOINT_METRICS_INDEX,
+      ignore_unavailable: false,
+      body: {
+        size: 0, // no query results required - only aggregation quantity
+        query: {
+          range: {
+            '@timestamp': {
+              gte: 'now-24h',
+              lt: 'now',
+            },
+          },
+        },
+        aggs: {
+          endpoint_count: {
+            cardinality: {
+              field: 'agent.id',
+            },
+          },
+        },
+      },
+    };
+
+    const response = await esClient.search(query);
+    const { aggregations: endpointCountResponse } =
+      response as unknown as UniqueEndpointCountResponse;
+    return endpointCountResponse?.endpoint_count?.value ?? 0;
+  } catch (e) {
+    tlog(logger, `Failed to get active endpoint count due to: ${e.message}`);
+    return 0;
+  }
+};

x-pack/plugins/security_solution/server/usage/endpoint/types.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface EndpointMetrics {
+  unique_endpoint_count: number;
+}
+
+export interface UniqueEndpointCountResponse {
+  aggregations: {
+    endpoint_count: { value: number };
+  };
+}

x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

@@ -13010,6 +13010,14 @@
               }
             }
           }
+        },
+        "endpointMetrics": {
+          "properties": {
+            "unique_endpoint_count": {
+              "type": "long",
+              "_meta": { "description": "Number of active unique endpoints in last 24 hours" }
+            }
+          }
         }
       }
     },