[8.7] [Security Solution] Enrich Value List Telemetry (#149621) (#151076)

# Backport This will backport the following commits from `main` to `8.7`: - [[Security Solution] Enrich Value List Telemetry (#149621)](https://github.com/elastic/kibana/pull/149621)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: JD Kurma <JDKurma@gmail.com>
2025-04-24 17:59:23 -04:00 · 2023-02-13 17:51:47 -05:00 · 2023-02-13 17:51:47 -05:00 · f2ea20f6e0
commit f2ea20f6e0
parent 489abdc160
6 changed files with 72 additions and 45 deletions
--- a/x-pack/plugins/security_solution/server/lib/telemetry/mocks/index.ts
+++ b/x-pack/plugins/security_solution/server/lib/telemetry/mocks/index.ts
@ -15,6 +15,7 @@ import type { PackagePolicy } from '@kbn/fleet-plugin/common/types/models/packag
 import { stubEndpointAlertResponse, stubProcessTree, stubFetchTimelineEvents } from './timeline';
 import { stubEndpointMetricsResponse } from './metrics';
 import { prebuiltRuleAlertsResponse } from './prebuilt_rule_alerts';
+import type { ESClusterInfo, ESLicense } from '../types';

 export const createMockTelemetryEventsSender = (
  enableTelemetry?: boolean,
@ -37,8 +38,7 @@ export const createMockTelemetryEventsSender = (
  } as unknown as jest.Mocked<TelemetryEventsSender>;
 };

-const stubClusterInfo = {
-  name: 'Stub-MacBook-Pro.local',
+export const stubClusterInfo: ESClusterInfo = {
  cluster_name: 'elasticsearch',
  cluster_uuid: '5Pr5PXRQQpGJUTn0czAvKQ',
  version: {
@ -46,24 +46,23 @@ const stubClusterInfo = {
    build_type: 'tar',
    build_hash: '38537ab4a726b42ce8f034aad78d8fca4d4f3e51',
    build_date: moment().toISOString(),
+    build_flavor: 'DEFAULT',
    build_snapshot: true,
    lucene_version: '9.2.0',
    minimum_wire_compatibility_version: '7.17.0',
    minimum_index_compatibility_version: '7.0.0',
  },
-  tagline: 'You Know, for Search',
 };

-const stubLicenseInfo = {
+export const stubLicenseInfo: ESLicense = {
  status: 'active',
  uid: '4a7dde08-e5f8-4e50-80f8-bc85b72b4934',
  type: 'trial',
  issue_date: moment().toISOString(),
  issue_date_in_millis: 1653299879146,
  expiry_date: moment().toISOString(),
-  expiry_date_in_millis: 1655891879146,
+  expirty_date_in_millis: 1655891879146,
  max_nodes: 1000,
-  max_resource_units: null,
  issued_to: 'elasticsearch',
  issuer: 'elasticsearch',
  start_date_in_millis: -1,
--- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts
+++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.test.ts
@ -6,7 +6,7 @@
 */

 import moment from 'moment';
-import { createMockPackagePolicy } from './__mocks__';
+import { createMockPackagePolicy, stubClusterInfo, stubLicenseInfo } from './__mocks__';
 import {
  LIST_DETECTION_RULE_EXCEPTION,
  LIST_ENDPOINT_EXCEPTION,
@ -21,7 +21,7 @@ import {
  isPackagePolicyList,
  templateExceptionList,
  addDefaultAdvancedPolicyConfigSettings,
-  metricsResponseToValueListMetaData,
+  formatValueListMetaData,
  tlog,
  setIsElasticCloudDeployment,
  createTaskMetric,
@ -805,10 +805,11 @@ describe('test advanced policy config overlap ', () => {

 describe('test metrics response to value list meta data', () => {
  test('can succeed when metrics response is fully populated', async () => {
+    jest.useFakeTimers().setSystemTime(new Date('2023-01-30'));
    const stubMetricResponses = {
      listMetricsResponse: {
        aggregations: {
-          total_value_list_count: 5,
+          total_value_list_count: { value: 5 },
          type_breakdown: {
            buckets: [
              {
@ -858,8 +859,12 @@ describe('test metrics response to value list meta data', () => {
        },
      },
    };
-    const response = metricsResponseToValueListMetaData(stubMetricResponses);
+    const response = formatValueListMetaData(stubMetricResponses, stubClusterInfo, stubLicenseInfo);
    expect(response).toEqual({
+      '@timestamp': '2023-01-30T00:00:00.000Z',
+      cluster_uuid: '5Pr5PXRQQpGJUTn0czAvKQ',
+      cluster_name: 'elasticsearch',
+      license_id: '4a7dde08-e5f8-4e50-80f8-bc85b72b4934',
      total_list_count: 5,
      types: [
        {
@ -901,8 +906,12 @@ describe('test metrics response to value list meta data', () => {
      indicatorMatchMetricsResponse: {},
    };
    // @ts-ignore
-    const response = metricsResponseToValueListMetaData(stubMetricResponses);
+    const response = formatValueListMetaData(stubMetricResponses, stubClusterInfo, stubLicenseInfo);
    expect(response).toEqual({
+      '@timestamp': '2023-01-30T00:00:00.000Z',
+      cluster_uuid: '5Pr5PXRQQpGJUTn0czAvKQ',
+      cluster_name: 'elasticsearch',
+      license_id: '4a7dde08-e5f8-4e50-80f8-bc85b72b4934',
      total_list_count: 0,
      types: [],
      lists: [],
--- a/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts
+++ b/x-pack/plugins/security_solution/server/lib/telemetry/helpers.ts
@ -18,10 +18,7 @@ import type {
  ESLicense,
  ListTemplate,
  TelemetryEvent,
-  ValueListResponseAggregation,
-  ValueListExceptionListResponseAggregation,
-  ValueListItemsResponseAggregation,
-  ValueListIndicatorMatchResponseAggregation,
+  ValueListResponse,
  TaskMetric,
 } from './types';
 import {
@ -241,32 +238,37 @@ export const addDefaultAdvancedPolicyConfigSettings = (policyConfig: PolicyConfi
  return merge(DEFAULT_ADVANCED_POLICY_CONFIG_SETTINGS, policyConfig);
 };

-export const metricsResponseToValueListMetaData = ({
-  listMetricsResponse,
-  itemMetricsResponse,
-  exceptionListMetricsResponse,
-  indicatorMatchMetricsResponse,
-}: {
-  listMetricsResponse: ValueListResponseAggregation;
-  itemMetricsResponse: ValueListItemsResponseAggregation;
-  exceptionListMetricsResponse: ValueListExceptionListResponseAggregation;
-  indicatorMatchMetricsResponse: ValueListIndicatorMatchResponseAggregation;
-}) => ({
-  total_list_count: listMetricsResponse?.aggregations?.total_value_list_count ?? 0,
+export const formatValueListMetaData = (
+  valueListResponse: ValueListResponse,
+  clusterInfo: ESClusterInfo,
+  licenseInfo: ESLicense | undefined
+) => ({
+  '@timestamp': moment().toISOString(),
+  cluster_uuid: clusterInfo.cluster_uuid,
+  cluster_name: clusterInfo.cluster_name,
+  license_id: licenseInfo?.uid,
+  total_list_count:
+    valueListResponse.listMetricsResponse?.aggregations?.total_value_list_count?.value ?? 0,
  types:
-    listMetricsResponse?.aggregations?.type_breakdown?.buckets.map((breakdown) => ({
-      type: breakdown.key,
-      count: breakdown.doc_count,
-    })) ?? [],
+    valueListResponse.listMetricsResponse?.aggregations?.type_breakdown?.buckets.map(
+      (breakdown) => ({
+        type: breakdown.key,
+        count: breakdown.doc_count,
+      })
+    ) ?? [],
  lists:
-    itemMetricsResponse?.aggregations?.value_list_item_count?.buckets.map((itemCount) => ({
-      id: itemCount.key,
-      count: itemCount.doc_count,
-    })) ?? [],
+    valueListResponse.itemMetricsResponse?.aggregations?.value_list_item_count?.buckets.map(
+      (itemCount) => ({
+        id: itemCount.key,
+        count: itemCount.doc_count,
+      })
+    ) ?? [],
  included_in_exception_lists_count:
-    exceptionListMetricsResponse?.aggregations?.vl_included_in_exception_lists_count?.value ?? 0,
+    valueListResponse.exceptionListMetricsResponse?.aggregations
+      ?.vl_included_in_exception_lists_count?.value ?? 0,
  used_in_indicator_match_rule_count:
-    indicatorMatchMetricsResponse?.aggregations?.vl_used_in_indicator_match_rule_count?.value ?? 0,
+    valueListResponse.indicatorMatchMetricsResponse?.aggregations
+      ?.vl_used_in_indicator_match_rule_count?.value ?? 0,
 });

 export let isElasticCloudDeployment = false;
--- a/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts
+++ b/x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts
@ -41,7 +41,6 @@ import {
  exceptionListItemToTelemetryEntry,
  trustedApplicationToTelemetryEntry,
  ruleExceptionListItemToTelemetryEvent,
-  metricsResponseToValueListMetaData,
  tlog,
 } from './helpers';
 import { Fetcher } from '../../endpoint/routes/resolver/tree/utils/fetch';
@ -55,7 +54,7 @@ import type {
  GetEndpointListResponse,
  RuleSearchResult,
  ExceptionListItem,
-  ValueListMetaData,
+  ValueListResponse,
  ValueListResponseAggregation,
  ValueListItemsResponseAggregation,
  ValueListExceptionListResponseAggregation,
@ -172,7 +171,7 @@ export interface ITelemetryReceiver {
    nodeIds: string[]
  ): Promise<SearchResponse<SafeEndpointEvent, Record<string, AggregationsAggregate>>>;

-  fetchValueListMetaData(interval: number): Promise<ValueListMetaData>;
+  fetchValueListMetaData(interval: number): Promise<ValueListResponse>;
 }

 export class TelemetryReceiver implements ITelemetryReceiver {
@ -924,12 +923,12 @@ export class TelemetryReceiver implements ITelemetryReceiver {
      exceptionListMetrics as unknown as ValueListExceptionListResponseAggregation;
    const indicatorMatchMetricsResponse =
      indicatorMatchMetrics as unknown as ValueListIndicatorMatchResponseAggregation;
-    return metricsResponseToValueListMetaData({
+    return {
      listMetricsResponse,
      itemMetricsResponse,
      exceptionListMetricsResponse,
      indicatorMatchMetricsResponse,
-    });
+    };
  }

  public async fetchClusterInfo(): Promise<ESClusterInfo> {
--- a/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts
+++ b/x-pack/plugins/security_solution/server/lib/telemetry/tasks/security_lists.ts
@ -18,7 +18,13 @@ import {
  TASK_METRICS_CHANNEL,
 } from '../constants';
 import type { ESClusterInfo, ESLicense } from '../types';
-import { batchTelemetryRecords, templateExceptionList, tlog, createTaskMetric } from '../helpers';
+import {
+  batchTelemetryRecords,
+  templateExceptionList,
+  tlog,
+  createTaskMetric,
+  formatValueListMetaData,
+} from '../helpers';
 import type { ITelemetryEventsSender } from '../sender';
 import type { ITelemetryReceiver } from '../receiver';
 import type { TaskExecutionPeriod } from '../task';
@ -114,9 +120,14 @@ export function createTelemetrySecurityListTaskConfig(maxTelemetryBatch: number)
        }

        // Value list meta data
-        const valueListMetaData = await receiver.fetchValueListMetaData(
+        const valueListResponse = await receiver.fetchValueListMetaData(
          FETCH_VALUE_LIST_META_DATA_INTERVAL_IN_HOURS
        );
+        const valueListMetaData = formatValueListMetaData(
+          valueListResponse,
+          clusterInfo,
+          licenseInfo
+        );
        tlog(logger, `Value List Meta Data: ${JSON.stringify(valueListMetaData)}`);
        if (valueListMetaData?.total_list_count) {
          await sender.sendOnDemand(TELEMETRY_CHANNEL_LISTS, [valueListMetaData]);
--- a/x-pack/plugins/security_solution/server/lib/telemetry/types.ts
+++ b/x-pack/plugins/security_solution/server/lib/telemetry/types.ts
@ -382,7 +382,7 @@ export interface ValueListMetaData {

 export interface ValueListResponseAggregation {
  aggregations: {
-    total_value_list_count: number;
+    total_value_list_count: { value: number };
    type_breakdown: {
      buckets: Array<{
        key: string;
@ -437,3 +437,10 @@ export interface TelemetryFilterListArtifact {
  exception_lists: AllowlistFields;
  prebuilt_rules_alerts: AllowlistFields;
 }
+
+export interface ValueListResponse {
+  listMetricsResponse: ValueListResponseAggregation;
+  itemMetricsResponse: ValueListItemsResponseAggregation;
+  exceptionListMetricsResponse: ValueListExceptionListResponseAggregation;
+  indicatorMatchMetricsResponse: ValueListIndicatorMatchResponseAggregation;
+}