[Cloud Security] update score trendline to support muting rules

This commit is contained in:
Ido Cohen 2024-01-03 14:21:38 +02:00 committed by GitHub
parent eb83faccb3
commit c05b8935d9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 187 additions and 103 deletions

View file

@ -6,7 +6,6 @@
*/
import { schema, TypeOf } from '@kbn/config-schema';
import type { SavedObjectsUpdateResponse } from '@kbn/core-saved-objects-api-server';
import { CSPM_POLICY_TEMPLATE, KSPM_POLICY_TEMPLATE } from '../../constants';
const DEFAULT_BENCHMARK_RULES_PER_PAGE = 25;
@ -136,46 +135,3 @@ export interface FindCspBenchmarkRuleResponse {
}
export type PageUrlParams = Record<'policyId' | 'packagePolicyId', string>;
export const rulesToUpdate = schema.arrayOf(
schema.object({
rule_id: schema.string(),
benchmark_id: schema.string(),
benchmark_version: schema.string(),
rule_number: schema.string(),
})
);
export const cspBenchmarkRulesBulkActionRequestSchema = schema.object({
action: schema.oneOf([schema.literal('mute'), schema.literal('unmute')]),
rules: rulesToUpdate,
});
export type RulesToUpdate = TypeOf<typeof rulesToUpdate>;
export type CspBenchmarkRulesBulkActionRequestSchema = TypeOf<
typeof cspBenchmarkRulesBulkActionRequestSchema
>;
const rulesStates = schema.recordOf(
schema.string(),
schema.object({
muted: schema.boolean(),
benchmark_id: schema.string(),
benchmark_version: schema.string(),
rule_number: schema.string(),
rule_id: schema.string(),
})
);
export const cspSettingsSchema = schema.object({
rules: rulesStates,
});
export type CspBenchmarkRulesStates = TypeOf<typeof rulesStates>;
export type CspSettings = TypeOf<typeof cspSettingsSchema>;
export interface BulkActionBenchmarkRulesResponse {
newCspSettings: SavedObjectsUpdateResponse<CspSettings>;
disabledRulesCounter: number;
}

View file

@ -6,16 +6,14 @@
*/
import { schema, TypeOf } from '@kbn/config-schema';
import { SavedObjectsUpdateResponse } from '@kbn/core-saved-objects-api-server';
import { BenchmarksCisId } from '../latest';
export type {
cspBenchmarkRuleMetadataSchema,
CspBenchmarkRuleMetadata,
cspBenchmarkRuleSchema,
CspBenchmarkRule,
FindCspBenchmarkRuleResponse,
CspSettings,
CspBenchmarkRulesStates,
} from './v3';
const DEFAULT_BENCHMARK_RULES_PER_PAGE = 25;
@ -113,3 +111,54 @@ export interface PageUrlParams {
benchmarkId: BenchmarksCisId;
benchmarkVersion: string;
}
export const rulesToUpdate = schema.arrayOf(
schema.object({
rule_id: schema.string(),
benchmark_id: schema.string(),
benchmark_version: schema.string(),
rule_number: schema.string(),
})
);
export const cspBenchmarkRulesBulkActionRequestSchema = schema.object({
action: schema.oneOf([schema.literal('mute'), schema.literal('unmute')]),
rules: rulesToUpdate,
});
export type RulesToUpdate = TypeOf<typeof rulesToUpdate>;
export type CspBenchmarkRulesBulkActionRequestSchema = TypeOf<
typeof cspBenchmarkRulesBulkActionRequestSchema
>;
export interface CspBenchmarkRulesBulkActionResponse {
updated_benchmark_rules: CspBenchmarkRulesStates;
disabled_detection_rules?: string[];
message: string;
}
const ruleStateAttributes = schema.object({
muted: schema.boolean(),
benchmark_id: schema.string(),
benchmark_version: schema.string(),
rule_number: schema.string(),
rule_id: schema.string(),
});
export type RuleStateAttributes = TypeOf<typeof ruleStateAttributes>;
const rulesStates = schema.recordOf(schema.string(), ruleStateAttributes);
export type CspBenchmarkRulesStates = TypeOf<typeof rulesStates>;
export const cspSettingsSchema = schema.object({
rules: rulesStates,
});
export type CspSettings = TypeOf<typeof cspSettingsSchema>;
export interface BulkActionBenchmarkRulesResponse {
newCspSettings: SavedObjectsUpdateResponse<CspSettings>;
disabledRules: string[];
}

View file

@ -47,5 +47,8 @@ export const benchmarkScoreMapping: MappingTypeMapping = {
low: {
type: 'long',
},
is_enabled_rules_score: {
type: 'boolean',
},
},
};

View file

@ -10,7 +10,8 @@ import {
CspBenchmarkRulesBulkActionRequestSchema,
CspBenchmarkRulesStates,
cspBenchmarkRulesBulkActionRequestSchema,
} from '../../../../common/types/rules/v3';
CspBenchmarkRulesBulkActionResponse,
} from '../../../../common/types/rules/v4';
import { CspRouter } from '../../../types';
import { CSP_BENCHMARK_RULES_BULK_ACTION_ROUTE_PATH } from '../../../../common/constants';
@ -79,13 +80,16 @@ export const defineBulkActionCspBenchmarkRulesRoute = (router: CspRouter) =>
const updatedBenchmarkRules: CspBenchmarkRulesStates =
handlerResponse.newCspSettings.attributes.rules!;
return response.ok({
body: {
updated_benchmark_rules: updatedBenchmarkRules,
detection_rules: `disabled ${handlerResponse.disabledRulesCounter} detections rules.`,
message: 'The bulk operation has been executed successfully.',
},
});
const body: CspBenchmarkRulesBulkActionResponse = {
updated_benchmark_rules: updatedBenchmarkRules,
message: 'The bulk operation has been executed successfully.',
};
if (requestBody.action === 'mute' && handlerResponse.disabledRules) {
body.disabled_detection_rules = handlerResponse.disabledRules;
}
return response.ok({ body });
} catch (err) {
const error = transformError(err);

View file

@ -16,7 +16,7 @@ import type {
RulesToUpdate,
CspBenchmarkRulesStates,
CspSettings,
} from '../../../../common/types/rules/v3';
} from '../../../../common/types/rules/v4';
import {
convertRuleTagsToKQL,
generateBenchmarkRuleTags,
@ -28,7 +28,9 @@ import {
INTERNAL_CSP_SETTINGS_SAVED_OBJECT_TYPE,
} from '../../../../common/constants';
export const getRuleIdsToDisable = async (detectionRules: Array<FindResult<RuleParams>>) => {
export const getDetectionRuleIdsToDisable = async (
detectionRules: Array<FindResult<RuleParams>>
) => {
const idsToDisable = detectionRules
.map((detectionRule) => {
return detectionRule.data.map((data) => data.id);
@ -40,10 +42,11 @@ export const getRuleIdsToDisable = async (detectionRules: Array<FindResult<RuleP
const disableDetectionRules = async (
detectionRulesClient: RulesClient,
detectionRules: Array<FindResult<RuleParams>>
) => {
const idsToDisable = await getRuleIdsToDisable(detectionRules);
if (!idsToDisable.length) return;
return await detectionRulesClient.bulkDisableRules({ ids: idsToDisable });
): Promise<string[]> => {
const detectionRulesIdsToDisable = await getDetectionRuleIdsToDisable(detectionRules);
if (!detectionRulesIdsToDisable.length) return [];
await detectionRulesClient.bulkDisableRules({ ids: detectionRulesIdsToDisable });
return detectionRulesIdsToDisable;
};
export const getDetectionRules = async (
@ -87,7 +90,7 @@ export const muteDetectionRules = async (
soClient: SavedObjectsClientContract,
detectionRulesClient: RulesClient,
rulesIds: string[]
): Promise<number> => {
): Promise<string[]> => {
const benchmarkRules = await getBenchmarkRules(soClient, rulesIds);
if (benchmarkRules.includes(undefined)) {
throw new Error('At least one of the provided benchmark rule IDs does not exist');
@ -99,8 +102,7 @@ export const muteDetectionRules = async (
const detectionRules = await getDetectionRules(detectionRulesClient, benchmarkRulesTags);
const disabledDetectionRules = await disableDetectionRules(detectionRulesClient, detectionRules);
return disabledDetectionRules ? disabledDetectionRules.rules.length : 0;
return disabledDetectionRules;
};
export const updateRulesStates = async (

View file

@ -17,7 +17,7 @@ import {
import type {
BulkActionBenchmarkRulesResponse,
RulesToUpdate,
} from '../../../../common/types/rules/v3';
} from '../../../../common/types/rules/v4';
const muteStatesMap = {
mute: true,
@ -41,12 +41,11 @@ export const bulkActionBenchmarkRulesHandler = async (
const rulesKeys = rulesToUpdate.map((rule) =>
buildRuleKey(rule.benchmark_id, rule.benchmark_version, rule.rule_number)
);
const newRulesStates = setRulesStates(rulesKeys, muteStatesMap[action], rulesToUpdate);
const newCspSettings = await updateRulesStates(encryptedSoClient, newRulesStates);
const disabledRulesCounter =
action === 'mute' ? await muteDetectionRules(soClient, detectionRulesClient, rulesIds) : 0;
const disabledDetectionRules =
action === 'mute' ? await muteDetectionRules(soClient, detectionRulesClient, rulesIds) : [];
return { newCspSettings, disabledRulesCounter };
return { newCspSettings, disabledRules: disabledDetectionRules };
};

View file

@ -8,7 +8,7 @@
import { transformError } from '@kbn/securitysolution-es-utils';
import { CspRouter } from '../../../types';
import { CSP_GET_BENCHMARK_RULES_STATE_ROUTE_PATH } from '../../../../common/constants';
import { CspBenchmarkRulesStates } from '../../../../common/types/rules/v3';
import { CspBenchmarkRulesStates } from '../../../../common/types/rules/v4';
import { getCspBenchmarkRulesStatesHandler } from './v1';
export const defineGetCspBenchmarkRulesStatesRoute = (router: CspRouter) =>

View file

@ -4,9 +4,13 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
import {
ISavedObjectsRepository,
SavedObjectsClientContract,
} from '@kbn/core-saved-objects-api-server';
import { transformError } from '@kbn/securitysolution-es-utils';
import { CspBenchmarkRulesStates, CspSettings } from '../../../../common/types/rules/v3';
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { CspBenchmarkRulesStates, CspSettings } from '../../../../common/types/rules/v4';
import {
INTERNAL_CSP_SETTINGS_SAVED_OBJECT_ID,
INTERNAL_CSP_SETTINGS_SAVED_OBJECT_TYPE,
@ -23,7 +27,7 @@ export const createCspSettingObject = async (soClient: SavedObjectsClientContrac
};
export const getCspBenchmarkRulesStatesHandler = async (
encryptedSoClient: SavedObjectsClientContract
encryptedSoClient: SavedObjectsClientContract | ISavedObjectsRepository
): Promise<CspBenchmarkRulesStates> => {
try {
const getSoResponse = await encryptedSoClient.get<CspSettings>(
@ -43,3 +47,27 @@ export const getCspBenchmarkRulesStatesHandler = async (
);
}
};
export const buildMutedRulesFilter = async (
encryptedSoClient: ISavedObjectsRepository
): Promise<QueryDslQueryContainer[]> => {
const rulesStates = await getCspBenchmarkRulesStatesHandler(encryptedSoClient);
const mutedRules = Object.fromEntries(
Object.entries(rulesStates).filter(([key, value]) => value.muted === true)
);
const mutedRulesFilterQuery = Object.keys(mutedRules).map((key) => {
const rule = mutedRules[key];
return {
bool: {
must: [
{ term: { 'rule.benchmark.id': rule.benchmark_id } },
{ term: { 'rule.benchmark.version': rule.benchmark_version } },
{ term: { 'rule.benchmark.rule_number': rule.rule_number } },
],
},
};
});
return mutedRulesFilterQuery;
};

View file

@ -54,14 +54,19 @@ export const getTrendsQuery = (policyTemplate: PosturePolicyTemplate) => ({
query: {
bool: {
filter: [{ term: { policy_template: policyTemplate } }],
must: {
range: {
'@timestamp': {
gte: 'now-1d',
lte: 'now',
must: [
{
range: {
'@timestamp': {
gte: 'now-1d',
lte: 'now',
},
},
},
},
{
term: { is_enabled_rules_score: true },
},
],
},
},
});

View file

@ -7,7 +7,7 @@
import { SavedObjectsType } from '@kbn/core/server';
import { SECURITY_SOLUTION_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server';
import { cspSettingsSchema } from '../../common/types/rules/v3';
import { cspSettingsSchema } from '../../common/types/rules/v4';
import { cspSettingsSavedObjectMapping } from './mappings';
import { INTERNAL_CSP_SETTINGS_SAVED_OBJECT_TYPE } from '../../common/constants';

View file

@ -13,13 +13,16 @@ import {
} from '@kbn/task-manager-plugin/server';
import { SearchRequest } from '@kbn/data-plugin/common';
import { ElasticsearchClient } from '@kbn/core/server';
import type { Logger } from '@kbn/core/server';
import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import type { ISavedObjectsRepository, Logger } from '@kbn/core/server';
import { buildMutedRulesFilter } from '../routes/benchmark_rules/get_states/v1';
import { getSafePostureTypeRuntimeMapping } from '../../common/runtime_mappings/get_safe_posture_type_runtime_mapping';
import { getIdentifierRuntimeMapping } from '../../common/runtime_mappings/get_identifier_runtime_mapping';
import { FindingsStatsTaskResult, ScoreByPolicyTemplateBucket, VulnSeverityAggs } from './types';
import {
BENCHMARK_SCORE_INDEX_DEFAULT_NS,
CSPM_FINDINGS_STATS_INTERVAL,
INTERNAL_CSP_SETTINGS_SAVED_OBJECT_TYPE,
LATEST_FINDINGS_INDEX_DEFAULT_NS,
LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
VULNERABILITIES_SEVERITY,
@ -93,8 +96,13 @@ export function taskRunner(coreStartServices: CspServerPluginStartServices, logg
async run(): Promise<FindingsStatsTaskResult> {
try {
logger.info(`Runs task: ${CSPM_FINDINGS_STATS_TASK_TYPE}`);
const esClient = (await coreStartServices)[0].elasticsearch.client.asInternalUser;
const status = await aggregateLatestFindings(esClient, logger);
const startServices = await coreStartServices;
const esClient = startServices[0].elasticsearch.client.asInternalUser;
const encryptedSoClient = startServices[0].savedObjects.createInternalRepository([
INTERNAL_CSP_SETTINGS_SAVED_OBJECT_TYPE,
]);
const status = await aggregateLatestFindings(esClient, encryptedSoClient, logger);
const updatedState: LatestTaskStateSchema = {
runs: state.runs + 1,
@ -119,13 +127,15 @@ export function taskRunner(coreStartServices: CspServerPluginStartServices, logg
};
}
const getScoreQuery = (): SearchRequest => ({
const getScoreQuery = (filteredRules: QueryDslQueryContainer[]): SearchRequest => ({
index: LATEST_FINDINGS_INDEX_DEFAULT_NS,
size: 0,
// creates the safe_posture_type and asset_identifier runtime fields
runtime_mappings: { ...getIdentifierRuntimeMapping(), ...getSafePostureTypeRuntimeMapping() },
query: {
match_all: {},
bool: {
must_not: filteredRules,
},
},
aggs: {
score_by_policy_template: {
@ -271,7 +281,8 @@ const getVulnStatsTrendQuery = (): SearchRequest => ({
const getFindingsScoresDocIndexingPromises = (
esClient: ElasticsearchClient,
scoresByPolicyTemplatesBuckets: ScoreByPolicyTemplateBucket['score_by_policy_template']['buckets']
scoresByPolicyTemplatesBuckets: ScoreByPolicyTemplateBucket['score_by_policy_template']['buckets'],
isCustomScore: boolean
) =>
scoresByPolicyTemplatesBuckets.map((policyTemplateTrend) => {
// creating score per cluster id objects
@ -321,6 +332,7 @@ const getFindingsScoresDocIndexingPromises = (
total_findings: policyTemplateTrend.total_findings.value,
score_by_cluster_id: clustersStats,
score_by_benchmark_id: benchmarkStats,
is_enabled_rules_score: isCustomScore,
},
});
});
@ -364,18 +376,27 @@ const getVulnStatsTrendDocIndexingPromises = (
export const aggregateLatestFindings = async (
esClient: ElasticsearchClient,
encryptedSoClient: ISavedObjectsRepository,
logger: Logger
): Promise<TaskHealthStatus> => {
try {
const startAggTime = performance.now();
const scoreIndexQueryResult = await esClient.search<unknown, ScoreByPolicyTemplateBucket>(
getScoreQuery()
const rulesFilter = await buildMutedRulesFilter(encryptedSoClient);
const customScoreIndexQueryResult = await esClient.search<unknown, ScoreByPolicyTemplateBucket>(
getScoreQuery(rulesFilter)
);
const fullScoreIndexQueryResult = await esClient.search<unknown, ScoreByPolicyTemplateBucket>(
getScoreQuery([])
);
const vulnStatsTrendIndexQueryResult = await esClient.search<unknown, VulnSeverityAggs>(
getVulnStatsTrendQuery()
);
if (!scoreIndexQueryResult.aggregations && !vulnStatsTrendIndexQueryResult.aggregations) {
if (!customScoreIndexQueryResult.aggregations && !vulnStatsTrendIndexQueryResult.aggregations) {
logger.warn(`No data found in latest findings index`);
return 'warning';
}
@ -388,13 +409,23 @@ export const aggregateLatestFindings = async (
);
// getting score per policy template buckets
const scoresByPolicyTemplatesBuckets =
scoreIndexQueryResult.aggregations?.score_by_policy_template.buckets || [];
const customScoresByPolicyTemplatesBuckets =
customScoreIndexQueryResult.aggregations?.score_by_policy_template.buckets || [];
const fullScoresByPolicyTemplatesBuckets =
fullScoreIndexQueryResult.aggregations?.score_by_policy_template.buckets || [];
// iterating over the buckets and return promises which will index a modified document into the scores index
const findingsScoresDocIndexingPromises = getFindingsScoresDocIndexingPromises(
const findingsCustomScoresDocIndexingPromises = getFindingsScoresDocIndexingPromises(
esClient,
scoresByPolicyTemplatesBuckets
customScoresByPolicyTemplatesBuckets,
true
);
const findingsFullScoresDocIndexingPromises = getFindingsScoresDocIndexingPromises(
esClient,
fullScoresByPolicyTemplatesBuckets,
false
);
const vulnStatsTrendDocIndexingPromises = getVulnStatsTrendDocIndexingPromises(
@ -406,7 +437,11 @@ export const aggregateLatestFindings = async (
// executing indexing commands
await Promise.all(
[...findingsScoresDocIndexingPromises, vulnStatsTrendDocIndexingPromises].filter(Boolean)
[
...findingsCustomScoresDocIndexingPromises,
findingsFullScoresDocIndexingPromises,
vulnStatsTrendDocIndexingPromises,
].filter(Boolean)
);
const totalIndexTime = Number(performance.now() - startIndexTime).toFixed(2);

View file

@ -45,7 +45,7 @@ export default function ({ getService }: FtrProviderContext) {
};
const createDetectionRule = async (rule: CspBenchmarkRule) => {
await supertest
const detectionRule = await supertest
.post(DETECTION_ENGINE_RULES_URL)
.set('version', DETECTION_RULE_RULES_API_CURRENT_VERSION)
.set('kbn-xsrf', 'xxxx')
@ -74,7 +74,9 @@ export default function ({ getService }: FtrProviderContext) {
name: rule.metadata.name,
description: rule.metadata.rationale,
tags: generateBenchmarkRuleTags(rule.metadata),
});
})
.expect(200);
return detectionRule;
};
/**
@ -98,7 +100,7 @@ export default function ({ getService }: FtrProviderContext) {
beforeEach(async () => {
await kibanaServer.savedObjects.clean({
types: ['cloud-security-posture-settings'],
types: ['cloud-security-posture-settings', 'alert'],
});
});
@ -152,7 +154,7 @@ export default function ({ getService }: FtrProviderContext) {
},
})
);
expectExpect(body.detection_rules).toEqual('disabled 0 detections rules.');
expectExpect(body.disabled_detection_rules).toEqual([]);
});
it('unmute rules successfully', async () => {
@ -312,7 +314,7 @@ export default function ({ getService }: FtrProviderContext) {
it('mute detection rule successfully', async () => {
const rule1 = await getRandomCspBenchmarkRule();
await createDetectionRule(rule1);
const detectionRule = await createDetectionRule(rule1);
const { body } = await supertest
.post(`/internal/cloud_security_posture/rules/_bulk_action`)
@ -332,14 +334,14 @@ export default function ({ getService }: FtrProviderContext) {
})
.expect(200);
expectExpect(body.detection_rules).toEqual('disabled 1 detections rules.');
expectExpect(body.disabled_detection_rules).toEqual([detectionRule.body.id]);
});
it('Expect to two benchmark rules and one detection rule', async () => {
it('Expect to mute two benchmark rules and one detection rule', async () => {
const rule1 = await getRandomCspBenchmarkRule();
const rule2 = await getRandomCspBenchmarkRule();
await createDetectionRule(rule1);
const detectionRule = await createDetectionRule(rule1);
const { body } = await supertest
.post(`/internal/cloud_security_posture/rules/_bulk_action`)
@ -365,7 +367,7 @@ export default function ({ getService }: FtrProviderContext) {
})
.expect(200);
expectExpect(body.detection_rules).toEqual('disabled 1 detections rules.');
expectExpect(body.disabled_detection_rules).toEqual([detectionRule.body.id]);
});
it('set wrong action input', async () => {

View file

@ -9,6 +9,7 @@ export const getBenchmarkScoreMockData = (postureType: string) => [
{
total_findings: 1,
policy_template: postureType,
is_enabled_rules_score: true,
'@timestamp': '2023-11-22T16:10:55.229268215Z',
score_by_cluster_id: {
'Another Upper case account id': {