[Security Solution] Telemetry Artifacts (#140652)

* kbn artifact

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* unzip manifest

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* remove staging

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* logs + address merge conflicts

* adm-zip package

* fix return type

* [CI] Auto-commit changed files from 'node scripts/precommit_hook.js --ref HEAD~1..HEAD --fix'

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* reset default on task errors

* optional check

* [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix'

* add adm-zip package

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

package.json

@@ -422,9 +422,11 @@
     "@turf/distance": "6.0.1",
     "@turf/helpers": "6.0.1",
     "@turf/length": "^6.0.2",
+    "@types/adm-zip": "^0.5.0",
     "@types/byte-size": "^8.1.0",
     "JSONStream": "1.3.5",
     "abort-controller": "^3.0.0",
+    "adm-zip": "^0.5.9",
     "antlr4ts": "^0.5.0-alpha.3",
     "archiver": "^5.3.1",
     "axios": "^0.27.2",

x-pack/plugins/security_solution/server/lib/telemetry/artifact.test.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createMockTelemetryReceiver } from './__mocks__';
+import { artifactService } from './artifact';
+
+describe('telemetry artifact test', () => {
+  test('diagnostics telemetry task should query and enqueue events', async () => {
+    const mockTelemetryReceiver = createMockTelemetryReceiver();
+    await artifactService.start(mockTelemetryReceiver);
+    expect(mockTelemetryReceiver.fetchClusterInfo).toHaveBeenCalled();
+  });
+});

x-pack/plugins/security_solution/server/lib/telemetry/artifact.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import axios from 'axios';
+import AdmZip from 'adm-zip';
+import type { ITelemetryReceiver } from './receiver';
+import type { ESClusterInfo } from './types';
+
+export interface IArtifact {
+  start(receiver: ITelemetryReceiver): Promise<void>;
+  getArtifact(name: string): Promise<unknown>;
+}
+
+class Artifact implements IArtifact {
+  private manifestUrl?: string;
+  private readonly CDN_URL = 'https://artifacts.security.elastic.co';
+  private readonly AXIOS_TIMEOUT_MS = 10_000;
+  private receiver?: ITelemetryReceiver;
+  private esClusterInfo?: ESClusterInfo;
+
+  public async start(receiver: ITelemetryReceiver) {
+    this.receiver = receiver;
+    this.esClusterInfo = await this.receiver.fetchClusterInfo();
+    const version = this.esClusterInfo?.version?.number;
+    this.manifestUrl = `${this.CDN_URL}/downloads/endpoint/manifest/artifacts-${version}.zip`;
+  }
+
+  public async getArtifact(name: string): Promise<unknown> {
+    if (this.manifestUrl) {
+      const response = await axios.get(this.manifestUrl, {
+        timeout: this.AXIOS_TIMEOUT_MS,
+        responseType: 'arraybuffer',
+      });
+      const zip = new AdmZip(response.data);
+      const entries = zip.getEntries();
+      const manifest = JSON.parse(entries[0].getData().toString());
+      const relativeUrl = manifest.artifacts[name]?.relative_url;
+      if (relativeUrl) {
+        const url = `${this.CDN_URL}${relativeUrl}`;
+        const artifactResponse = await axios.get(url, { timeout: this.AXIOS_TIMEOUT_MS });
+        return artifactResponse.data;
+      } else {
+        throw Error(`No artifact for name ${name}`);
+      }
+    } else {
+      throw Error('No manifest url');
+    }
+  }
+}
+
+export const artifactService = new Artifact();

x-pack/plugins/security_solution/server/lib/telemetry/configuration.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+class TelemetryConfiguration {
+  private readonly DEFAULT_TELEMETRY_MAX_BUFFER_SIZE = 100;
+  private readonly DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH = 100;
+  private readonly DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH = 300;
+  private readonly DEFAULT_MAX_DETECTION_RULE_TELEMETRY_BATCH = 1_000;
+  private readonly DEFAULT_MAX_DETECTION_ALERTS_BATCH = 50;
+  private _telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE;
+  private _max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH;
+  private _max_endpoint_telemetry_batch = this.DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH;
+  private _max_detection_rule_telemetry_batch = this.DEFAULT_MAX_DETECTION_RULE_TELEMETRY_BATCH;
+  private _max_detection_alerts_batch = this.DEFAULT_MAX_DETECTION_ALERTS_BATCH;
+
+  public get telemetry_max_buffer_size(): number {
+    return this._telemetry_max_buffer_size;
+  }
+
+  public set telemetry_max_buffer_size(num: number) {
+    this._telemetry_max_buffer_size = num;
+  }
+
+  public get max_security_list_telemetry_batch(): number {
+    return this._max_security_list_telemetry_batch;
+  }
+
+  public set max_security_list_telemetry_batch(num: number) {
+    this._max_security_list_telemetry_batch = num;
+  }
+
+  public get max_endpoint_telemetry_batch(): number {
+    return this._max_endpoint_telemetry_batch;
+  }
+
+  public set max_endpoint_telemetry_batch(num: number) {
+    this._max_endpoint_telemetry_batch = num;
+  }
+
+  public get max_detection_rule_telemetry_batch(): number {
+    return this._max_detection_rule_telemetry_batch;
+  }
+
+  public set max_detection_rule_telemetry_batch(num: number) {
+    this._max_detection_rule_telemetry_batch = num;
+  }
+
+  public get max_detection_alerts_batch(): number {
+    return this._max_detection_alerts_batch;
+  }
+
+  public set max_detection_alerts_batch(num: number) {
+    this._max_detection_alerts_batch = num;
+  }
+
+  public resetAllToDefault() {
+    this._telemetry_max_buffer_size = this.DEFAULT_TELEMETRY_MAX_BUFFER_SIZE;
+    this._max_security_list_telemetry_batch = this.DEFAULT_MAX_SECURITY_LIST_TELEMETRY_BATCH;
+    this._max_endpoint_telemetry_batch = this.DEFAULT_MAX_ENDPOINT_TELEMETRY_BATCH;
+    this._max_detection_rule_telemetry_batch = this.DEFAULT_MAX_DETECTION_RULE_TELEMETRY_BATCH;
+    this._max_detection_alerts_batch = this.DEFAULT_MAX_DETECTION_ALERTS_BATCH;
+  }
+}
+
+export const telemetryConfiguration = new TelemetryConfiguration();

x-pack/plugins/security_solution/server/lib/telemetry/constants.ts

@@ -5,16 +5,6 @@
  * 2.0.
  */
 
-export const TELEMETRY_MAX_BUFFER_SIZE = 100;
-
-export const MAX_SECURITY_LIST_TELEMETRY_BATCH = 100;
-
-export const MAX_ENDPOINT_TELEMETRY_BATCH = 300;
-
-export const MAX_DETECTION_RULE_TELEMETRY_BATCH = 1_000;
-
-export const MAX_DETECTION_ALERTS_BATCH = 50;
-
 export const TELEMETRY_CHANNEL_LISTS = 'security-lists-v2';
 
 export const TELEMETRY_CHANNEL_ENDPOINT_META = 'endpoint-metadata';

x-pack/plugins/security_solution/server/lib/telemetry/receiver.ts

@@ -33,7 +33,6 @@ import type { Agent, AgentPolicy } from '@kbn/fleet-plugin/common';
 import type { AgentClient, AgentPolicyServiceInterface } from '@kbn/fleet-plugin/server';
 import type { ExceptionListClient } from '@kbn/lists-plugin/server';
 import type { EndpointAppContextService } from '../../endpoint/endpoint_app_context_services';
-import { TELEMETRY_MAX_BUFFER_SIZE } from './constants';
 import {
   exceptionListItemToTelemetryEntry,
   trustedApplicationToTelemetryEntry,
@@ -58,6 +57,7 @@ import type {
   ValueListExceptionListResponseAggregation,
   ValueListIndicatorMatchResponseAggregation,
 } from './types';
+import { telemetryConfiguration } from './configuration';
 
 export interface ITelemetryReceiver {
   start(
@@ -378,7 +378,7 @@ export class TelemetryReceiver implements ITelemetryReceiver {
       expand_wildcards: ['open' as const, 'hidden' as const],
       index: '.logs-endpoint.diagnostic.collection-*',
       ignore_unavailable: true,
-      size: TELEMETRY_MAX_BUFFER_SIZE,
+      size: telemetryConfiguration.telemetry_max_buffer_size,
       body: {
         query: {
           range: {

x-pack/plugins/security_solution/server/lib/telemetry/sender.ts

@@ -23,9 +23,9 @@ import { copyAllowlistedFields, endpointAllowlistFields } from './filterlists';
 import { createTelemetryTaskConfigs } from './tasks';
 import { createUsageCounterLabel, tlog } from './helpers';
 import type { TelemetryEvent } from './types';
-import { TELEMETRY_MAX_BUFFER_SIZE } from './constants';
 import type { SecurityTelemetryTaskConfig } from './task';
 import { SecurityTelemetryTask } from './task';
+import { telemetryConfiguration } from './configuration';
 
 const usageLabelPrefix: string[] = ['security_telemetry', 'sender'];
 
@@ -60,7 +60,7 @@ export class TelemetryEventsSender implements ITelemetryEventsSender {
   private readonly initialCheckDelayMs = 10 * 1000;
   private readonly checkIntervalMs = 60 * 1000;
   private readonly logger: Logger;
-  private maxQueueSize = TELEMETRY_MAX_BUFFER_SIZE;
+  private maxQueueSize = telemetryConfiguration.telemetry_max_buffer_size;
   private telemetryStart?: TelemetryPluginStart;
   private telemetrySetup?: TelemetryPluginSetup;
   private intervalId?: NodeJS.Timeout;

x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.test.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { loggingSystemMock } from '@kbn/core/server/mocks';
+import { createTelemetryConfigurationTaskConfig } from './configuration';
+import { createMockTelemetryEventsSender, createMockTelemetryReceiver } from '../__mocks__';
+
+describe('telemetry configuration task test', () => {
+  let logger: ReturnType<typeof loggingSystemMock.createLogger>;
+
+  beforeEach(() => {
+    logger = loggingSystemMock.createLogger();
+  });
+
+  test('diagnostics telemetry task should query and enqueue events', async () => {
+    const testTaskExecutionPeriod = {
+      last: new Date().toISOString(),
+      current: new Date().toISOString(),
+    };
+    const mockTelemetryReceiver = createMockTelemetryReceiver();
+    const mockTelemetryEventsSender = createMockTelemetryEventsSender();
+    const telemetryDiagnoticsTaskConfig = createTelemetryConfigurationTaskConfig();
+
+    await telemetryDiagnoticsTaskConfig.runTask(
+      'test-id',
+      logger,
+      mockTelemetryReceiver,
+      mockTelemetryEventsSender,
+      testTaskExecutionPeriod
+    );
+
+    // TODO: Add tests
+  });
+});

x-pack/plugins/security_solution/server/lib/telemetry/tasks/configuration.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Logger } from '@kbn/core/server';
+import type { ITelemetryEventsSender } from '../sender';
+import type { TelemetryConfiguration } from '../types';
+import type { ITelemetryReceiver } from '../receiver';
+import type { TaskExecutionPeriod } from '../task';
+import { artifactService } from '../artifact';
+import { telemetryConfiguration } from '../configuration';
+import { tlog } from '../helpers';
+
+export function createTelemetryConfigurationTaskConfig() {
+  return {
+    type: 'security:telemetry-configuration',
+    title: 'Security Solution Telemetry Configuration Task',
+    interval: '45m',
+    timeout: '1m',
+    version: '1.0.0',
+    runTask: async (
+      taskId: string,
+      logger: Logger,
+      receiver: ITelemetryReceiver,
+      sender: ITelemetryEventsSender,
+      taskExecutionPeriod: TaskExecutionPeriod
+    ) => {
+      try {
+        const artifactName = 'telemetry-configuration-v1';
+        const configArtifact = (await artifactService.getArtifact(
+          artifactName
+        )) as unknown as TelemetryConfiguration;
+        telemetryConfiguration.max_detection_alerts_batch =
+          configArtifact.max_detection_alerts_batch;
+        telemetryConfiguration.telemetry_max_buffer_size = configArtifact.telemetry_max_buffer_size;
+        telemetryConfiguration.max_detection_rule_telemetry_batch =
+          configArtifact.max_detection_rule_telemetry_batch;
+        telemetryConfiguration.max_endpoint_telemetry_batch =
+          configArtifact.max_endpoint_telemetry_batch;
+        telemetryConfiguration.max_security_list_telemetry_batch =
+          configArtifact.max_security_list_telemetry_batch;
+        return 0;
+      } catch (err) {
+        tlog(logger, `Failed to set telemetry configuration due to ${err.message}`);
+        telemetryConfiguration.resetAllToDefault();
+        return 0;
+      }
+    },
+  };
+}

x-pack/plugins/security_solution/server/lib/telemetry/tasks/index.ts

@@ -12,20 +12,19 @@ import { createTelemetrySecurityListTaskConfig } from './security_lists';
 import { createTelemetryDetectionRuleListsTaskConfig } from './detection_rule';
 import { createTelemetryPrebuiltRuleAlertsTaskConfig } from './prebuilt_rule_alerts';
 import { createTelemetryTimelineTaskConfig } from './timelines';
-import {
-  MAX_SECURITY_LIST_TELEMETRY_BATCH,
-  MAX_ENDPOINT_TELEMETRY_BATCH,
-  MAX_DETECTION_RULE_TELEMETRY_BATCH,
-  MAX_DETECTION_ALERTS_BATCH,
-} from '../constants';
+import { createTelemetryConfigurationTaskConfig } from './configuration';
+import { telemetryConfiguration } from '../configuration';
 
 export function createTelemetryTaskConfigs(): SecurityTelemetryTaskConfig[] {
   return [
     createTelemetryDiagnosticsTaskConfig(),
-    createTelemetryEndpointTaskConfig(MAX_SECURITY_LIST_TELEMETRY_BATCH),
-    createTelemetrySecurityListTaskConfig(MAX_ENDPOINT_TELEMETRY_BATCH),
-    createTelemetryDetectionRuleListsTaskConfig(MAX_DETECTION_RULE_TELEMETRY_BATCH),
-    createTelemetryPrebuiltRuleAlertsTaskConfig(MAX_DETECTION_ALERTS_BATCH),
+    createTelemetryEndpointTaskConfig(telemetryConfiguration.max_security_list_telemetry_batch),
+    createTelemetrySecurityListTaskConfig(telemetryConfiguration.max_endpoint_telemetry_batch),
+    createTelemetryDetectionRuleListsTaskConfig(
+      telemetryConfiguration.max_detection_rule_telemetry_batch
+    ),
+    createTelemetryPrebuiltRuleAlertsTaskConfig(telemetryConfiguration.max_detection_alerts_batch),
     createTelemetryTimelineTaskConfig(),
+    createTelemetryConfigurationTaskConfig(),
   ];
 }

x-pack/plugins/security_solution/server/lib/telemetry/types.ts

@@ -421,3 +421,11 @@ export interface TaskMetric {
   end_time: number;
   error_message?: string;
 }
+
+export interface TelemetryConfiguration {
+  telemetry_max_buffer_size: number;
+  max_security_list_telemetry_batch: number;
+  max_endpoint_telemetry_batch: number;
+  max_detection_rule_telemetry_batch: number;
+  max_detection_alerts_batch: number;
+}

x-pack/plugins/security_solution/server/plugin.ts

@@ -101,6 +101,7 @@ import { alertsFieldMap, rulesFieldMap } from '../common/field_maps';
 import { EndpointFleetServicesFactory } from './endpoint/services/fleet';
 import { featureUsageService } from './endpoint/services/feature_usage';
 import { setIsElasticCloudDeployment } from './lib/telemetry/helpers';
+import { artifactService } from './lib/telemetry/artifact';
 
 export type { SetupPlugins, StartPlugins, PluginSetup, PluginStart } from './plugin_contract';
 
@@ -481,6 +482,8 @@ export class Plugin implements ISecuritySolutionPlugin {
       exceptionListClient
     );
 
+    artifactService.start(this.telemetryReceiver);
+
     this.telemetryEventsSender.start(
       plugins.telemetry,
       plugins.taskManager,

yarn.lock

@@ -6002,6 +6002,13 @@
   dependencies:
     "@turf/helpers" "6.x"
 
+"@types/adm-zip@^0.5.0":
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/@types/adm-zip/-/adm-zip-0.5.0.tgz#94c90a837ce02e256c7c665a6a1eb295906333c1"
+  integrity sha512-FCJBJq9ODsQZUNURo5ILAQueuA8WJhRvuihS3ke2iI25mJlfV2LK8jG2Qj2z2AWg8U0FtWWqBHVRetceLskSaw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/apidoc@^0.22.3":
   version "0.22.3"
   resolved "https://registry.yarnpkg.com/@types/apidoc/-/apidoc-0.22.3.tgz#0227f4b8189b5cde42d23ed81a858526959fc2b7"
@@ -9321,7 +9328,7 @@ address@^1.0.1:
   resolved "https://registry.yarnpkg.com/address/-/address-1.1.2.tgz#bf1116c9c758c51b7a933d296b72c221ed9428b6"
   integrity sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA==
 
-adm-zip@0.5.9:
+adm-zip@0.5.9, adm-zip@^0.5.9:
   version "0.5.9"
   resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.9.tgz#b33691028333821c0cf95c31374c5462f2905a83"
   integrity sha512-s+3fXLkeeLjZ2kLjCBwQufpI5fuN+kIGBxu6530nVQZGVol0d7Y/M88/xw9HGGUcJjKf8LutN3VPRUBq6N7Ajg==