mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
[Cloud Security][Billing] D4C metering
This commit is contained in:
parent
6ee512b86e
commit
888873e7fc
7 changed files with 413 additions and 178 deletions
|
@ -4,21 +4,13 @@
|
|||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
} from '@kbn/cloud-security-posture-plugin/common/constants';
|
||||
import { ProductLine } from '../../common/product';
|
||||
import { getCloudSecurityUsageRecord } from './cloud_security_metering_task';
|
||||
import type { PostureType } from './types';
|
||||
import { CLOUD_DEFEND, CNVM, CSPM, KSPM } from './constants';
|
||||
import type { CloudSecuritySolutions } from './types';
|
||||
import type { MeteringCallbackInput, Tier, UsageRecord } from '../types';
|
||||
import type { ServerlessSecurityConfig } from '../config';
|
||||
|
||||
export const CLOUD_SECURITY_TASK_TYPE = 'cloud_security';
|
||||
export const AGGREGATION_PRECISION_THRESHOLD = 40000;
|
||||
|
||||
export const cloudSecurityMetringCallback = async ({
|
||||
esClient,
|
||||
cloudSetup,
|
||||
|
@ -36,28 +28,26 @@ export const cloudSecurityMetringCallback = async ({
|
|||
const tier: Tier = getCloudProductTier(config);
|
||||
|
||||
try {
|
||||
const postureTypes: PostureType[] = [
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
];
|
||||
const cloudSecuritySolutions: CloudSecuritySolutions[] = [CSPM, KSPM, CNVM, CLOUD_DEFEND];
|
||||
|
||||
const cloudSecurityUsageRecords = await Promise.all(
|
||||
postureTypes.map((postureType) =>
|
||||
cloudSecuritySolutions.map((cloudSecuritySolution) =>
|
||||
getCloudSecurityUsageRecord({
|
||||
esClient,
|
||||
projectId,
|
||||
logger,
|
||||
taskId,
|
||||
lastSuccessfulReport,
|
||||
postureType,
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
// remove any potential undefined values from the array,
|
||||
return cloudSecurityUsageRecords.filter(Boolean) as UsageRecord[];
|
||||
return cloudSecurityUsageRecords
|
||||
.filter((record) => record !== undefined && record.length > 0)
|
||||
.flatMap((record) => record) as UsageRecord[];
|
||||
} catch (err) {
|
||||
logger.error(`Failed to fetch Cloud Security metering data ${err}`);
|
||||
return [];
|
||||
|
|
|
@ -6,40 +6,33 @@
|
|||
*/
|
||||
import Chance from 'chance';
|
||||
import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks';
|
||||
import {
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
} from '@kbn/cloud-security-posture-plugin/common/constants';
|
||||
import { CLOUD_SECURITY_TASK_TYPE, getCloudProductTier } from './cloud_security_metering';
|
||||
|
||||
import { getCloudProductTier } from './cloud_security_metering';
|
||||
import { getCloudSecurityUsageRecord } from './cloud_security_metering_task';
|
||||
|
||||
import type { ServerlessSecurityConfig } from '../config';
|
||||
import type { PostureType } from './types';
|
||||
import type { CloudSecuritySolutions } from './types';
|
||||
import type { ProductTier } from '../../common/product';
|
||||
import { CLOUD_SECURITY_TASK_TYPE, CSPM, KSPM, CNVM } from './constants';
|
||||
|
||||
const mockEsClient = elasticsearchServiceMock.createStart().client.asInternalUser;
|
||||
const logger: ReturnType<typeof loggingSystemMock.createLogger> = loggingSystemMock.createLogger();
|
||||
const chance = new Chance();
|
||||
|
||||
const postureTypes: PostureType[] = [
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
];
|
||||
const cloudSecuritySolutions: CloudSecuritySolutions[] = [CSPM, KSPM, CNVM];
|
||||
|
||||
describe('getCloudSecurityUsageRecord', () => {
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
it('should return undefined if postureType is missing', async () => {
|
||||
it('should return undefined if cloudSecuritySolution is missing', async () => {
|
||||
// Mock Elasticsearch search to throw an error
|
||||
mockEsClient.search.mockRejectedValue({});
|
||||
|
||||
const projectId = chance.guid();
|
||||
const taskId = chance.guid();
|
||||
const postureType = CSPM_POLICY_TEMPLATE;
|
||||
const cloudSecuritySolution = CSPM;
|
||||
|
||||
const tier = 'essentials' as ProductTier;
|
||||
|
||||
|
@ -49,16 +42,16 @@ describe('getCloudSecurityUsageRecord', () => {
|
|||
logger,
|
||||
taskId,
|
||||
lastSuccessfulReport: new Date(),
|
||||
postureType,
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
});
|
||||
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
test.each(postureTypes)(
|
||||
'should return usageRecords with correct values for cspm and kspm when Elasticsearch response has aggregations',
|
||||
async (postureType) => {
|
||||
test.each(cloudSecuritySolutions)(
|
||||
'should return usageRecords with correct values for cspm, kspm, and cnvm when Elasticsearch response has aggregations',
|
||||
async (cloudSecuritySolution) => {
|
||||
// @ts-ignore
|
||||
mockEsClient.search.mockResolvedValueOnce({
|
||||
hits: { hits: [{ _id: 'someRecord', _index: 'mockIndex' }] }, // mocking for indexHasDataInDateRange
|
||||
|
@ -87,28 +80,32 @@ describe('getCloudSecurityUsageRecord', () => {
|
|||
logger,
|
||||
taskId,
|
||||
lastSuccessfulReport: new Date(),
|
||||
postureType,
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
id: expect.stringContaining(`${CLOUD_SECURITY_TASK_TYPE}_${postureType}_${projectId}`),
|
||||
usage_timestamp: '2023-07-30T15:11:41.738Z',
|
||||
creation_timestamp: expect.any(String), // Expect a valid ISO string
|
||||
usage: {
|
||||
type: CLOUD_SECURITY_TASK_TYPE,
|
||||
sub_type: postureType,
|
||||
quantity: 10,
|
||||
period_seconds: expect.any(Number),
|
||||
},
|
||||
source: {
|
||||
id: taskId,
|
||||
instance_group_id: projectId,
|
||||
metadata: {
|
||||
tier: 'essentials',
|
||||
expect(result).toEqual([
|
||||
{
|
||||
id: expect.stringContaining(
|
||||
`${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}`
|
||||
),
|
||||
usage_timestamp: '2023-07-30T15:11:41.738Z',
|
||||
creation_timestamp: expect.any(String), // Expect a valid ISO string
|
||||
usage: {
|
||||
type: CLOUD_SECURITY_TASK_TYPE,
|
||||
sub_type: cloudSecuritySolution,
|
||||
quantity: 10,
|
||||
period_seconds: expect.any(Number),
|
||||
},
|
||||
source: {
|
||||
id: taskId,
|
||||
instance_group_id: projectId,
|
||||
metadata: {
|
||||
tier: 'essentials',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
]);
|
||||
}
|
||||
);
|
||||
|
||||
|
@ -118,7 +115,7 @@ describe('getCloudSecurityUsageRecord', () => {
|
|||
|
||||
const projectId = chance.guid();
|
||||
const taskId = chance.guid();
|
||||
const postureType = CSPM_POLICY_TEMPLATE;
|
||||
const cloudSecuritySolution = CSPM;
|
||||
|
||||
const tier = 'essentials' as ProductTier;
|
||||
|
||||
|
@ -128,7 +125,7 @@ describe('getCloudSecurityUsageRecord', () => {
|
|||
logger,
|
||||
taskId,
|
||||
lastSuccessfulReport: new Date(),
|
||||
postureType,
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
});
|
||||
|
||||
|
@ -141,7 +138,7 @@ describe('getCloudSecurityUsageRecord', () => {
|
|||
|
||||
const projectId = chance.guid();
|
||||
const taskId = chance.guid();
|
||||
const postureType = CSPM_POLICY_TEMPLATE;
|
||||
const cloudSecuritySolution = CSPM;
|
||||
|
||||
const tier = 'essentials' as ProductTier;
|
||||
|
||||
|
@ -151,7 +148,7 @@ describe('getCloudSecurityUsageRecord', () => {
|
|||
logger,
|
||||
taskId,
|
||||
lastSuccessfulReport: new Date(),
|
||||
postureType,
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
});
|
||||
|
||||
|
@ -175,6 +172,100 @@ describe('should return the relevant product tier', () => {
|
|||
expect(tier).toBe('complete');
|
||||
});
|
||||
|
||||
it('should return usageRecords with correct values for cloud defend', async () => {
|
||||
const cloudSecuritySolution = 'cloud_defend';
|
||||
// @ts-ignore
|
||||
mockEsClient.search.mockResolvedValueOnce({
|
||||
hits: { hits: [{ _id: 'someRecord', _index: 'mockIndex' }] }, // mocking for indexHasDataInDateRange
|
||||
});
|
||||
|
||||
// @ts-ignore
|
||||
mockEsClient.search.mockResolvedValueOnce({
|
||||
aggregations: {
|
||||
asset_count_groups: {
|
||||
buckets: [
|
||||
{
|
||||
key_as_string: 'true',
|
||||
unique_assets: {
|
||||
value: 10,
|
||||
},
|
||||
min_timestamp: {
|
||||
value_as_string: '2023-07-30T15:11:41.738Z',
|
||||
},
|
||||
},
|
||||
{
|
||||
key_as_string: 'false',
|
||||
unique_assets: {
|
||||
value: 5,
|
||||
},
|
||||
min_timestamp: {
|
||||
value_as_string: '2023-07-30T15:11:41.738Z',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const projectId = chance.guid();
|
||||
const taskId = chance.guid();
|
||||
|
||||
const tier = 'essentials' as ProductTier;
|
||||
|
||||
const result = await getCloudSecurityUsageRecord({
|
||||
esClient: mockEsClient,
|
||||
projectId,
|
||||
logger,
|
||||
taskId,
|
||||
lastSuccessfulReport: new Date(),
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
});
|
||||
|
||||
expect(result).toEqual([
|
||||
{
|
||||
id: expect.stringContaining(
|
||||
`${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}`
|
||||
),
|
||||
usage_timestamp: '2023-07-30T15:11:41.738Z',
|
||||
creation_timestamp: expect.any(String), // Expect a valid ISO string
|
||||
usage: {
|
||||
type: CLOUD_SECURITY_TASK_TYPE,
|
||||
sub_type: `${cloudSecuritySolution}_block_action_enabled_true`,
|
||||
quantity: 10,
|
||||
period_seconds: expect.any(Number),
|
||||
},
|
||||
source: {
|
||||
id: taskId,
|
||||
instance_group_id: projectId,
|
||||
metadata: {
|
||||
tier: 'essentials',
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: expect.stringContaining(
|
||||
`${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}`
|
||||
),
|
||||
usage_timestamp: '2023-07-30T15:11:41.738Z',
|
||||
creation_timestamp: expect.any(String), // Expect a valid ISO string
|
||||
usage: {
|
||||
type: CLOUD_SECURITY_TASK_TYPE,
|
||||
sub_type: `${cloudSecuritySolution}_block_action_enabled_false`,
|
||||
quantity: 5,
|
||||
period_seconds: expect.any(Number),
|
||||
},
|
||||
source: {
|
||||
id: taskId,
|
||||
instance_group_id: projectId,
|
||||
metadata: {
|
||||
tier: 'essentials',
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return none tier in case cloud product line is missing ', async () => {
|
||||
const serverlessSecurityConfig = {
|
||||
enabled: true,
|
||||
|
|
|
@ -5,88 +5,65 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
LATEST_FINDINGS_INDEX_PATTERN,
|
||||
LATEST_VULNERABILITIES_INDEX_PATTERN,
|
||||
} from '@kbn/cloud-security-posture-plugin/common/constants';
|
||||
import type { Logger } from '@kbn/core/server';
|
||||
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import type { UsageRecord } from '../types';
|
||||
|
||||
import {
|
||||
AGGREGATION_PRECISION_THRESHOLD,
|
||||
ASSETS_SAMPLE_GRANULARITY,
|
||||
CLOUD_DEFEND,
|
||||
CLOUD_SECURITY_TASK_TYPE,
|
||||
} from './cloud_security_metering';
|
||||
import { cloudSecurityMetringTaskProperties } from './cloud_security_metering_task_config';
|
||||
CNVM,
|
||||
CSPM,
|
||||
KSPM,
|
||||
METERING_CONFIGS,
|
||||
THRESHOLD_MINUTES,
|
||||
} from './constants';
|
||||
import type { Tier, UsageRecord } from '../types';
|
||||
import type {
|
||||
CloudSecurityMeteringCallbackInput,
|
||||
PostureType,
|
||||
ResourceCountAggregation,
|
||||
CloudSecuritySolutions,
|
||||
AssetCountAggregation,
|
||||
CloudDefendAssetCountAggregation,
|
||||
} from './types';
|
||||
|
||||
const ASSETS_SAMPLE_GRANULARITY = '24h';
|
||||
export const getUsageRecords = (
|
||||
assetCountAggregations: AssetCountAggregation[],
|
||||
cloudSecuritySolution: CloudSecuritySolutions,
|
||||
taskId: string,
|
||||
tier: Tier,
|
||||
projectId: string,
|
||||
periodSeconds: number,
|
||||
logger: Logger
|
||||
): UsageRecord[] => {
|
||||
const usageRecords = assetCountAggregations.map((assetCountAggregation) => {
|
||||
const assetCount = assetCountAggregation.unique_assets.value;
|
||||
|
||||
const queryParams = {
|
||||
[CSPM_POLICY_TEMPLATE]: {
|
||||
index: LATEST_FINDINGS_INDEX_PATTERN,
|
||||
assets_identifier: 'resource.id',
|
||||
},
|
||||
[KSPM_POLICY_TEMPLATE]: {
|
||||
index: LATEST_FINDINGS_INDEX_PATTERN,
|
||||
assets_identifier: 'agent.id',
|
||||
},
|
||||
[CNVM_POLICY_TEMPLATE]: {
|
||||
index: LATEST_VULNERABILITIES_INDEX_PATTERN,
|
||||
assets_identifier: 'cloud.instance.id',
|
||||
},
|
||||
};
|
||||
|
||||
export const getCloudSecurityUsageRecord = async ({
|
||||
esClient,
|
||||
projectId,
|
||||
logger,
|
||||
taskId,
|
||||
postureType,
|
||||
tier,
|
||||
}: CloudSecurityMeteringCallbackInput): Promise<UsageRecord | undefined> => {
|
||||
try {
|
||||
if (!postureType) {
|
||||
logger.error('posture type is missing');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!(await indexHasDataInDateRange(esClient, postureType))) return;
|
||||
|
||||
const response = await esClient.search<unknown, ResourceCountAggregation>(
|
||||
getAggQueryByPostureType(postureType)
|
||||
);
|
||||
|
||||
if (!response.aggregations) {
|
||||
return;
|
||||
}
|
||||
const resourceCount = response.aggregations.unique_assets.value;
|
||||
if (resourceCount > AGGREGATION_PRECISION_THRESHOLD) {
|
||||
if (assetCount > AGGREGATION_PRECISION_THRESHOLD) {
|
||||
logger.warn(
|
||||
`The number of unique resources for {${postureType}} is ${resourceCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.`
|
||||
`The number of unique resources for {${cloudSecuritySolution}} is ${assetCount}, which is higher than the AGGREGATION_PRECISION_THRESHOLD of ${AGGREGATION_PRECISION_THRESHOLD}.`
|
||||
);
|
||||
}
|
||||
const minTimestamp = response.aggregations
|
||||
? new Date(response.aggregations.min_timestamp.value_as_string).toISOString()
|
||||
: new Date().toISOString();
|
||||
|
||||
const minTimestamp = new Date(
|
||||
assetCountAggregation.min_timestamp.value_as_string
|
||||
).toISOString();
|
||||
|
||||
const creationTimestamp = new Date().toISOString();
|
||||
|
||||
const usageRecord = {
|
||||
id: `${CLOUD_SECURITY_TASK_TYPE}_${postureType}_${projectId}_${creationTimestamp}`,
|
||||
const subType =
|
||||
cloudSecuritySolution === CLOUD_DEFEND
|
||||
? `${CLOUD_DEFEND}_block_action_enabled_${assetCountAggregation.key_as_string}`
|
||||
: cloudSecuritySolution;
|
||||
|
||||
const usageRecord: UsageRecord = {
|
||||
id: `${CLOUD_SECURITY_TASK_TYPE}_${cloudSecuritySolution}_${projectId}_${creationTimestamp}`,
|
||||
usage_timestamp: minTimestamp,
|
||||
creation_timestamp: creationTimestamp,
|
||||
usage: {
|
||||
type: CLOUD_SECURITY_TASK_TYPE,
|
||||
sub_type: postureType,
|
||||
quantity: resourceCount,
|
||||
period_seconds: cloudSecurityMetringTaskProperties.periodSeconds,
|
||||
sub_type: subType,
|
||||
quantity: assetCount,
|
||||
period_seconds: periodSeconds,
|
||||
},
|
||||
source: {
|
||||
id: taskId,
|
||||
|
@ -95,40 +72,85 @@ export const getCloudSecurityUsageRecord = async ({
|
|||
},
|
||||
};
|
||||
|
||||
logger.debug(`Fetched ${postureType} metring data`);
|
||||
|
||||
return usageRecord;
|
||||
} catch (err) {
|
||||
logger.error(`Failed to fetch ${postureType} metering data ${err}`);
|
||||
}
|
||||
};
|
||||
|
||||
const indexHasDataInDateRange = async (esClient: ElasticsearchClient, postureType: PostureType) => {
|
||||
const response = await esClient.search({
|
||||
index: queryParams[postureType].index,
|
||||
size: 1,
|
||||
_source: false,
|
||||
query: getSearchQueryByPostureType(postureType),
|
||||
});
|
||||
|
||||
return response.hits.hits.length > 0;
|
||||
return usageRecords;
|
||||
};
|
||||
|
||||
export const getSearchQueryByPostureType = (postureType: PostureType) => {
|
||||
const mustFilters = [];
|
||||
export const getAggregationByCloudSecuritySolution = (
|
||||
cloudSecuritySolution: CloudSecuritySolutions
|
||||
) => {
|
||||
if (cloudSecuritySolution === CLOUD_DEFEND) {
|
||||
return {
|
||||
asset_count_groups: {
|
||||
terms: {
|
||||
field: 'cloud_defend.block_action_enabled',
|
||||
},
|
||||
aggs: {
|
||||
unique_assets: {
|
||||
cardinality: {
|
||||
field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier,
|
||||
},
|
||||
},
|
||||
min_timestamp: {
|
||||
min: {
|
||||
field: '@timestamp',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
mustFilters.push({
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: `now-${ASSETS_SAMPLE_GRANULARITY}`,
|
||||
return {
|
||||
unique_assets: {
|
||||
cardinality: {
|
||||
field: METERING_CONFIGS[cloudSecuritySolution].assets_identifier,
|
||||
precision_threshold: AGGREGATION_PRECISION_THRESHOLD,
|
||||
},
|
||||
},
|
||||
});
|
||||
min_timestamp: {
|
||||
min: {
|
||||
field: '@timestamp',
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
if (postureType === CSPM_POLICY_TEMPLATE || postureType === KSPM_POLICY_TEMPLATE) {
|
||||
export const getSearchQueryByCloudSecuritySolution = (
|
||||
cloudSecuritySolution: CloudSecuritySolutions,
|
||||
searchFrom: Date
|
||||
) => {
|
||||
const mustFilters = [];
|
||||
|
||||
if (cloudSecuritySolution === CLOUD_DEFEND) {
|
||||
mustFilters.push({
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gt: searchFrom.toISOString(),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
cloudSecuritySolution === CSPM ||
|
||||
cloudSecuritySolution === KSPM ||
|
||||
cloudSecuritySolution === CNVM
|
||||
) {
|
||||
mustFilters.push({
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: `now-${ASSETS_SAMPLE_GRANULARITY}`,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (cloudSecuritySolution === CSPM || cloudSecuritySolution === KSPM) {
|
||||
mustFilters.push({
|
||||
term: {
|
||||
'rule.benchmark.posture_type': postureType,
|
||||
'rule.benchmark.posture_type': cloudSecuritySolution,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
@ -140,25 +162,111 @@ export const getSearchQueryByPostureType = (postureType: PostureType) => {
|
|||
};
|
||||
};
|
||||
|
||||
export const getAggQueryByPostureType = (postureType: PostureType) => {
|
||||
const query = {
|
||||
index: queryParams[postureType].index,
|
||||
query: getSearchQueryByPostureType(postureType),
|
||||
size: 0,
|
||||
aggs: {
|
||||
unique_assets: {
|
||||
cardinality: {
|
||||
field: queryParams[postureType].assets_identifier,
|
||||
precision_threshold: AGGREGATION_PRECISION_THRESHOLD,
|
||||
},
|
||||
},
|
||||
min_timestamp: {
|
||||
min: {
|
||||
field: '@timestamp',
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
export const getAssetAggQueryByCloudSecuritySolution = (
|
||||
cloudSecuritySolution: CloudSecuritySolutions,
|
||||
searchFrom: Date
|
||||
) => {
|
||||
const query = getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom);
|
||||
const aggs = getAggregationByCloudSecuritySolution(cloudSecuritySolution);
|
||||
|
||||
return query;
|
||||
return {
|
||||
index: METERING_CONFIGS[cloudSecuritySolution].index,
|
||||
query,
|
||||
size: 0,
|
||||
aggs,
|
||||
};
|
||||
};
|
||||
|
||||
export const getAssetAggByCloudSecuritySolution = async (
|
||||
esClient: ElasticsearchClient,
|
||||
cloudSecuritySolution: CloudSecuritySolutions,
|
||||
searchFrom: Date
|
||||
): Promise<AssetCountAggregation[]> => {
|
||||
const assetsAggQuery = getAssetAggQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom);
|
||||
|
||||
if (cloudSecuritySolution === CLOUD_DEFEND) {
|
||||
const response = await esClient.search<unknown, CloudDefendAssetCountAggregation>(
|
||||
assetsAggQuery
|
||||
);
|
||||
|
||||
if (!response.aggregations || !response.aggregations.asset_count_groups.buckets.length)
|
||||
return [];
|
||||
return response.aggregations.asset_count_groups.buckets;
|
||||
}
|
||||
|
||||
const response = await esClient.search<unknown, AssetCountAggregation>(assetsAggQuery);
|
||||
if (!response.aggregations) return [];
|
||||
|
||||
return [response.aggregations];
|
||||
};
|
||||
|
||||
const indexHasDataInDateRange = async (
|
||||
esClient: ElasticsearchClient,
|
||||
cloudSecuritySolution: CloudSecuritySolutions,
|
||||
searchFrom: Date
|
||||
) => {
|
||||
const response = await esClient.search({
|
||||
index: METERING_CONFIGS[cloudSecuritySolution].index,
|
||||
size: 1,
|
||||
_source: false,
|
||||
query: getSearchQueryByCloudSecuritySolution(cloudSecuritySolution, searchFrom),
|
||||
});
|
||||
|
||||
return response.hits.hits.length > 0;
|
||||
};
|
||||
|
||||
const getSearchStartDate = (lastSuccessfulReport: Date): Date => {
|
||||
const initialDate = new Date();
|
||||
const thresholdDate = new Date(initialDate.getTime() - THRESHOLD_MINUTES * 60 * 1000);
|
||||
|
||||
let lastSuccessfulReport1;
|
||||
|
||||
if (lastSuccessfulReport) {
|
||||
lastSuccessfulReport1 = new Date(lastSuccessfulReport);
|
||||
|
||||
const searchFrom =
|
||||
lastSuccessfulReport && lastSuccessfulReport1 > thresholdDate
|
||||
? lastSuccessfulReport1
|
||||
: thresholdDate;
|
||||
return searchFrom;
|
||||
}
|
||||
return thresholdDate;
|
||||
};
|
||||
|
||||
export const getCloudSecurityUsageRecord = async ({
|
||||
esClient,
|
||||
projectId,
|
||||
taskId,
|
||||
lastSuccessfulReport,
|
||||
cloudSecuritySolution,
|
||||
tier,
|
||||
logger,
|
||||
}: CloudSecurityMeteringCallbackInput): Promise<UsageRecord[] | undefined> => {
|
||||
try {
|
||||
const searchFrom = getSearchStartDate(lastSuccessfulReport);
|
||||
|
||||
if (!(await indexHasDataInDateRange(esClient, cloudSecuritySolution, searchFrom))) return;
|
||||
|
||||
const periodSeconds = Math.floor((new Date().getTime() - searchFrom.getTime()) / 1000);
|
||||
|
||||
const assetCountAggregations = await getAssetAggByCloudSecuritySolution(
|
||||
esClient,
|
||||
cloudSecuritySolution,
|
||||
searchFrom
|
||||
);
|
||||
|
||||
const usageRecords = await getUsageRecords(
|
||||
assetCountAggregations,
|
||||
cloudSecuritySolution,
|
||||
taskId,
|
||||
tier,
|
||||
projectId,
|
||||
periodSeconds,
|
||||
logger
|
||||
);
|
||||
|
||||
return usageRecords;
|
||||
} catch (err) {
|
||||
logger.error(`Failed to fetch ${cloudSecuritySolution} metering data ${err}`);
|
||||
}
|
||||
};
|
||||
|
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
LATEST_FINDINGS_INDEX_PATTERN,
|
||||
LATEST_VULNERABILITIES_INDEX_PATTERN,
|
||||
} from '@kbn/cloud-security-posture-plugin/common/constants';
|
||||
import { INTEGRATION_PACKAGE_NAME } from '@kbn/cloud-defend-plugin/common/constants';
|
||||
|
||||
const CLOUD_DEFEND_HEARTBEAT_INDEX = 'metrics-cloud_defend.heartbeat';
|
||||
export const CLOUD_SECURITY_TASK_TYPE = 'cloud_security';
|
||||
export const AGGREGATION_PRECISION_THRESHOLD = 40000;
|
||||
export const ASSETS_SAMPLE_GRANULARITY = '124h';
|
||||
export const THRESHOLD_MINUTES = 30;
|
||||
|
||||
export const CSPM = CSPM_POLICY_TEMPLATE;
|
||||
export const KSPM = KSPM_POLICY_TEMPLATE;
|
||||
export const CNVM = CNVM_POLICY_TEMPLATE;
|
||||
export const CLOUD_DEFEND = INTEGRATION_PACKAGE_NAME;
|
||||
|
||||
export const METERING_CONFIGS = {
|
||||
[CSPM]: {
|
||||
index: LATEST_FINDINGS_INDEX_PATTERN,
|
||||
assets_identifier: 'resource.id',
|
||||
},
|
||||
[KSPM]: {
|
||||
index: LATEST_FINDINGS_INDEX_PATTERN,
|
||||
assets_identifier: 'agent.id',
|
||||
},
|
||||
[CNVM]: {
|
||||
index: LATEST_VULNERABILITIES_INDEX_PATTERN,
|
||||
assets_identifier: 'cloud.instance.id',
|
||||
},
|
||||
[CLOUD_DEFEND]: {
|
||||
index: CLOUD_DEFEND_HEARTBEAT_INDEX,
|
||||
assets_identifier: 'agent.id',
|
||||
},
|
||||
};
|
|
@ -5,14 +5,17 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import type {
|
||||
CSPM_POLICY_TEMPLATE,
|
||||
KSPM_POLICY_TEMPLATE,
|
||||
CNVM_POLICY_TEMPLATE,
|
||||
} from '@kbn/cloud-security-posture-plugin/common/constants';
|
||||
import type { CSPM, KSPM, CNVM, CLOUD_DEFEND } from './constants';
|
||||
import type { MeteringCallbackInput, Tier } from '../types';
|
||||
|
||||
export interface ResourceCountAggregation {
|
||||
export interface CloudDefendAssetCountAggregation {
|
||||
asset_count_groups: AssetCountAggregationBucket;
|
||||
}
|
||||
export interface AssetCountAggregationBucket {
|
||||
buckets: AssetCountAggregation[];
|
||||
}
|
||||
export interface AssetCountAggregation {
|
||||
key_as_string: string;
|
||||
min_timestamp: MinTimestamp;
|
||||
unique_assets: {
|
||||
value: number;
|
||||
|
@ -24,14 +27,11 @@ export interface MinTimestamp {
|
|||
value_as_string: string;
|
||||
}
|
||||
|
||||
export type PostureType =
|
||||
| typeof CSPM_POLICY_TEMPLATE
|
||||
| typeof KSPM_POLICY_TEMPLATE
|
||||
| typeof CNVM_POLICY_TEMPLATE;
|
||||
export type CloudSecuritySolutions = typeof CSPM | typeof KSPM | typeof CNVM | typeof CLOUD_DEFEND;
|
||||
|
||||
export interface CloudSecurityMeteringCallbackInput
|
||||
extends Omit<MeteringCallbackInput, 'cloudSetup' | 'abortController' | 'config'> {
|
||||
projectId: string;
|
||||
postureType: PostureType;
|
||||
cloudSecuritySolution: CloudSecuritySolutions;
|
||||
tier: Tier;
|
||||
}
|
||||
|
|
|
@ -131,7 +131,7 @@ export class SecurityUsageReportingTask {
|
|||
config: this.config,
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.error(`failed to retrieve usage records: ${JSON.stringify(err)}`);
|
||||
this.logger.error(`failed to retrieve usage records: ${err}`);
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -153,7 +153,7 @@ export class SecurityUsageReportingTask {
|
|||
`usage records report was sent successfully: ${usageReportResponse.status}, ${usageReportResponse.statusText}`
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to send usage records report ${JSON.stringify(err)} `);
|
||||
this.logger.error(`Failed to send usage records report ${err} `);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -41,6 +41,7 @@
|
|||
"@kbn/cases-plugin",
|
||||
"@kbn/fleet-plugin",
|
||||
"@kbn/core-elasticsearch-server",
|
||||
"@kbn/usage-collection-plugin"
|
||||
"@kbn/usage-collection-plugin",
|
||||
"@kbn/cloud-defend-plugin"
|
||||
]
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue