[Security Solution] update endpoint metering service (#162934)

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -45,7 +45,7 @@ export const policyIndexPattern = 'metrics-endpoint.policy-*';
 
 export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
 
-export const ENDPOINT_HEARTBEAT_INDEX = 'logs-endpoint.heartbeat-default';
+export const ENDPOINT_HEARTBEAT_INDEX = '.logs-endpoint.heartbeat-default';
 
 // File storage indexes supporting endpoint Upload/download
 export const FILE_STORAGE_METADATA_INDEX = getFileMetadataIndexName('endpoint');

x-pack/plugins/security_solution_serverless/server/cloud_security/types.ts

@@ -30,7 +30,7 @@ export type PostureType =
   | typeof CNVM_POLICY_TEMPLATE;
 
 export interface CloudSecurityMeteringCallbackInput
-  extends Omit<MeteringCallbackInput, 'cloudSetup' | 'abortController'> {
+  extends Omit<MeteringCallbackInput, 'cloudSetup' | 'abortController' | 'config'> {
   projectId: string;
   postureType: PostureType;
 }

x-pack/plugins/security_solution_serverless/server/constants.ts

@@ -5,7 +5,6 @@
  * 2.0.
  */
 
-// TODO: this probably shouldn't live in code
 const namespace = 'elastic-system';
 const USAGE_SERVICE_BASE_API_URL = `https://usage-api.${namespace}/api`;
 const USAGE_SERVICE_BASE_API_URL_V1 = `${USAGE_SERVICE_BASE_API_URL}/v1`;

x-pack/plugins/security_solution_serverless/server/endpoint/services/metering_service.ts

@@ -7,29 +7,49 @@
 
 import type { AggregationsAggregate, SearchResponse } from '@elastic/elasticsearch/lib/api/types';
 import type { ElasticsearchClient } from '@kbn/core/server';
-// import { ENDPOINT_HEARTBEAT_INDEX } from '@kbn/security-solution-plugin/common/endpoint/constants';
+import { ENDPOINT_HEARTBEAT_INDEX } from '@kbn/security-solution-plugin/common/endpoint/constants';
 import type { EndpointHeartbeat } from '@kbn/security-solution-plugin/common/endpoint/types';
 
+import { ProductLine, type ProductTier } from '../../../common/product';
+
 import type { UsageRecord, MeteringCallbackInput } from '../../types';
+import type { ServerlessSecurityConfig } from '../../config';
 
 // 1 hour
 const SAMPLE_PERIOD_SECONDS = 3600;
-// const THRESHOLD_MINUTES = 30;
+const THRESHOLD_MINUTES = 30;
 
 export class EndpointMeteringService {
-  public async getUsageRecords({
+  private tier: ProductTier | undefined;
+
+  public getUsageRecords = async ({
     taskId,
     cloudSetup,
     esClient,
     abortController,
     lastSuccessfulReport,
-  }: MeteringCallbackInput): Promise<UsageRecord[]> {
+    config,
+  }: MeteringCallbackInput): Promise<UsageRecord[]> => {
+    this.setTier(config);
+
     const heartbeatsResponse = await this.getHeartbeatsSince(
       esClient,
       abortController,
       lastSuccessfulReport
     );
 
+    if (!heartbeatsResponse?.hits?.hits) {
+      return [];
+    }
+
+    if (!this.tier) {
+      throw new Error(
+        `no product tier information found for heartbeats: ${JSON.stringify(
+          heartbeatsResponse.hits.hits
+        )}`
+      );
+    }
+
     return heartbeatsResponse.hits.hits.reduce((acc, { _source }) => {
       if (!_source) {
         return acc;
@@ -45,50 +65,30 @@ export class EndpointMeteringService {
 
       return [...acc, record];
     }, [] as UsageRecord[]);
-  }
+  };
 
   private async getHeartbeatsSince(
     esClient: ElasticsearchClient,
     abortController: AbortController,
     since?: Date
   ): Promise<SearchResponse<EndpointHeartbeat, Record<string, AggregationsAggregate>>> {
-    const timestamp = new Date().toISOString();
-    return {
-      hits: {
-        hits: [
-          {
-            _source: {
-              '@timestamp': timestamp,
-              agent: {
-                id: '123',
-              },
-              event: {
-                ingested: timestamp,
-              },
+    const thresholdDate = new Date(Date.now() - THRESHOLD_MINUTES * 60 * 1000);
+    const searchFrom = since && since > thresholdDate ? since : thresholdDate;
+
+    return esClient.search<EndpointHeartbeat>(
+      {
+        index: ENDPOINT_HEARTBEAT_INDEX,
+        sort: 'event.ingested',
+        query: {
+          range: {
+            'event.ingested': {
+              gt: searchFrom.toISOString(),
             },
           },
-        ],
+        },
       },
-    } as SearchResponse<EndpointHeartbeat, Record<string, AggregationsAggregate>>;
-
-    // TODO: enable when heartbeat index is ready
-    // const thresholdDate = new Date(Date.now() - THRESHOLD_MINUTES * 60 * 1000);
-    // const searchFrom = since && since > thresholdDate ? since : thresholdDate;
-
-    // return esClient.search<EndpointHeartbeat>(
-    //   {
-    //     index: ENDPOINT_HEARTBEAT_INDEX,
-    //     sort: 'event.ingested',
-    //     query: {
-    //       range: {
-    //         'event.ingested': {
-    //           gt: searchFrom.toISOString(),
-    //         },
-    //       },
-    //     },
-    //   },
-    //   { signal: abortController.signal }
-    // );
+      { signal: abortController.signal, ignore: [404] }
+    );
   }
 
   private buildMeteringRecord({
@@ -113,8 +113,7 @@ export class EndpointMeteringService {
       creation_timestamp: timestampStr,
       usage: {
         type: 'security_solution_endpoint',
-        // TODO: get actual sub_type
-        sub_type: 'essential',
+        sub_type: this.tier,
         period_seconds: SAMPLE_PERIOD_SECONDS,
         quantity: 1,
       },
@@ -124,6 +123,17 @@ export class EndpointMeteringService {
       },
     };
   }
+
+  private setTier(config: ServerlessSecurityConfig) {
+    if (this.tier) {
+      return;
+    }
+
+    const endpoint = config.productTypes.find(
+      (productType) => productType.product_line === ProductLine.endpoint
+    );
+    this.tier = endpoint?.product_tier;
+  }
 }
 
 export const endpointMeteringService = new EndpointMeteringService();

x-pack/plugins/security_solution_serverless/server/plugin.ts

@@ -68,6 +68,7 @@ export class SecuritySolutionServerlessPlugin
     this.cspmUsageReportingTask = new SecurityUsageReportingTask({
       core: _coreSetup,
       logFactory: this.initializerContext.logger,
+      config: this.config,
       taskManager: pluginsSetup.taskManager,
       cloudSetup: pluginsSetup.cloudSetup,
       taskType: cloudSecurityMetringTaskProperties.taskType,
@@ -79,6 +80,7 @@ export class SecuritySolutionServerlessPlugin
     this.endpointUsageReportingTask = new SecurityUsageReportingTask({
       core: _coreSetup,
       logFactory: this.initializerContext.logger,
+      config: this.config,
       taskType: ENDPOINT_METERING_TASK.TYPE,
       taskTitle: ENDPOINT_METERING_TASK.TITLE,
       version: ENDPOINT_METERING_TASK.VERSION,

x-pack/plugins/security_solution_serverless/server/task_manager/usage_reporting_task.ts

@@ -15,7 +15,9 @@ import type {
   MeteringCallback,
   SecurityUsageReportingTaskStartContract,
   SecurityUsageReportingTaskSetupContract,
+  UsageRecord,
 } from '../types';
+import type { ServerlessSecurityConfig } from '../config';
 
 const SCOPE = ['serverlessSecurity'];
 const TIMEOUT = '1m';
@@ -29,11 +31,13 @@ export class SecurityUsageReportingTask {
   private version: string;
   private logger: Logger;
   private abortController = new AbortController();
+  private config: ServerlessSecurityConfig;
 
   constructor(setupContract: SecurityUsageReportingTaskSetupContract) {
     const {
       core,
       logFactory,
+      config,
       taskManager,
       cloudSetup,
       taskType,
@@ -46,6 +50,7 @@ export class SecurityUsageReportingTask {
     this.taskType = taskType;
     this.version = version;
     this.logger = logFactory.get(this.taskId);
+    this.config = config;
 
     try {
       taskManager.registerTaskDefinitions({
@@ -114,14 +119,21 @@ export class SecurityUsageReportingTask {
 
     const lastSuccessfulReport = taskInstance.state.lastSuccessfulReport;
 
-    const usageRecords = await meteringCallback({
-      esClient,
-      cloudSetup: this.cloudSetup,
-      logger: this.logger,
-      taskId: this.taskId,
-      lastSuccessfulReport,
-      abortController: this.abortController,
-    });
+    let usageRecords: UsageRecord[] = [];
+    try {
+      usageRecords = await meteringCallback({
+        esClient,
+        cloudSetup: this.cloudSetup,
+        logger: this.logger,
+        taskId: this.taskId,
+        lastSuccessfulReport,
+        abortController: this.abortController,
+        config: this.config,
+      });
+    } catch (err) {
+      this.logger.error(`failed to retrieve usage records: ${JSON.stringify(err)}`);
+      return;
+    }
 
     this.logger.debug(`received usage records: ${JSON.stringify(usageRecords)}`);
 

x-pack/plugins/security_solution_serverless/server/types.ts

@@ -19,6 +19,8 @@ import type { CloudSetup } from '@kbn/cloud-plugin/server';
 import type { SecuritySolutionEssPluginSetup } from '@kbn/security-solution-ess/server';
 import type { MlPluginSetup } from '@kbn/ml-plugin/server';
 
+import type { ServerlessSecurityConfig } from './config';
+
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface SecuritySolutionServerlessPluginSetup {}
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
@@ -66,6 +68,7 @@ export interface UsageSource {
 export interface SecurityUsageReportingTaskSetupContract {
   core: CoreSetup;
   logFactory: LoggerFactory;
+  config: ServerlessSecurityConfig;
   taskManager: TaskManagerSetupContract;
   cloudSetup: CloudSetup;
   taskType: string;
@@ -90,6 +93,7 @@ export interface MeteringCallbackInput {
   taskId: string;
   lastSuccessfulReport: Date;
   abortController: AbortController;
+  config: ServerlessSecurityConfig;
 }
 
 export interface MetringTaskProperties {