github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Cloud Security] fetch k8s version and account id for telemetry

Browse source
This commit is contained in:
Ido Cohen 2023-03-28 11:27:08 +03:00 • committed by GitHub
parent 8ce205999a
commit 8eb127da9a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 210 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

17
x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/accounts_stats_collector.ts
View file

@ -30,6 +30,10 @@ interface BenchmarkVersion {
metrics: { 'rule.benchmark.version': string };
}
interface KubernetesVersion {
metrics: { 'cloudbeat.kubernetes.version': string };
}
interface AccountsStats {
accounts: {
buckets: AccountEntity[];
@ -43,6 +47,7 @@ interface AccountEntity {
benchmark_name: { top: BenchmarkName[] };
benchmark_id: { top: BenchmarkId[] };
benchmark_version: { top: BenchmarkVersion[] };
kubernetes_version: { top: KubernetesVersion[] };
agents_count: Value;
nodes_count: Value;
pods_count: Value;
@ -110,6 +115,17 @@ const getAccountsStatsQuery = (): SearchRequest => ({
},
},
},
kubernetes_version: {
top_metrics: {
metrics: {
field: 'cloudbeat.kubernetes.version',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
passed_findings_count: {
filter: {
bool: {
@ -203,6 +219,7 @@ const getCspmAccountsStats = (
benchmark_name: account.benchmark_name.top[0].metrics['rule.benchmark.name'],
benchmark_id: account.benchmark_id.top[0].metrics['rule.benchmark.id'],
benchmark_version: account.benchmark_version.top[0].metrics['rule.benchmark.version'],
kubernetes_version: account.kubernetes_version.top[0].metrics['cloudbeat.kubernetes.version'],
agents_count: account.agents_count.value,
nodes_count: account.nodes_count.value,
pods_count: account.resources.pods_count.value,

309
x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/rules_stats_collector.ts
View file

@ -10,6 +10,7 @@ import type {
AggregationsMultiBucketBase,
SearchRequest,
} from '@elastic/elasticsearch/lib/api/types';
import { getIdentifierRuntimeMapping } from '../../../../common/runtime_mappings/get_identifier_runtime_mapping';
import type { CspmRulesStats } from './types';
import { LATEST_FINDINGS_INDEX_DEFAULT_NS } from '../../../../common/constants';
@ -40,7 +41,14 @@ interface PostureType {
metrics: { 'rule.benchmark.posture_type': string };
}
interface Accounts {
accounts: {
buckets: RulesStats[];
};
}
interface RulesStats {
key: string; // account_id
rules: {
buckets: RuleEntity[];
};
@ -65,141 +73,154 @@ const getRulesStatsQuery = (): SearchRequest => ({
query: {
match_all: {},
},
// generates the 'asset_identifier' field
runtime_mappings: getIdentifierRuntimeMapping(),
aggs: {
rules: {
accounts: {
terms: {
field: 'rule.id',
field: 'asset_identifier',
order: {
_count: 'desc',
},
size: 100,
size: 1000,
},
aggs: {
rule_name: {
top_metrics: {
metrics: {
field: 'rule.name',
},
size: 1,
sort: {
'@timestamp': 'desc',
rules: {
terms: {
field: 'rule.id',
order: {
_count: 'desc',
},
size: 1000,
},
},
rule_section: {
top_metrics: {
metrics: {
field: 'rule.section',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
rule_version: {
top_metrics: {
metrics: {
field: 'rule.version',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
posture_type: {
top_metrics: {
metrics: {
field: 'rule.benchmark.posture_type',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
rule_number: {
top_metrics: {
metrics: {
field: 'rule.benchmark.rule_number',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
benchmark_id: {
top_metrics: {
metrics: {
field: 'rule.benchmark.id',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
benchmark_version: {
top_metrics: {
metrics: {
field: 'rule.benchmark.version',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
benchmark_name: {
top_metrics: {
metrics: {
field: 'rule.benchmark.name',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
passed_findings_count: {
filter: {
bool: {
filter: [
{
bool: {
should: [
{
term: {
'result.evaluation': 'passed',
},
},
],
minimum_should_match: 1,
},
aggs: {
rule_name: {
top_metrics: {
metrics: {
field: 'rule.name',
},
],
},
},
},
failed_findings_count: {
filter: {
bool: {
filter: [
{
bool: {
should: [
{
term: {
'result.evaluation': 'failed',
},
},
],
minimum_should_match: 1,
},
size: 1,
sort: {
'@timestamp': 'desc',
},
],
},
},
rule_section: {
top_metrics: {
metrics: {
field: 'rule.section',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
rule_version: {
top_metrics: {
metrics: {
field: 'rule.version',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
posture_type: {
top_metrics: {
metrics: {
field: 'rule.benchmark.posture_type',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
rule_number: {
top_metrics: {
metrics: {
field: 'rule.benchmark.rule_number',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
benchmark_id: {
top_metrics: {
metrics: {
field: 'rule.benchmark.id',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
benchmark_version: {
top_metrics: {
metrics: {
field: 'rule.benchmark.version',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
benchmark_name: {
top_metrics: {
metrics: {
field: 'rule.benchmark.name',
},
size: 1,
sort: {
'@timestamp': 'desc',
},
},
},
passed_findings_count: {
filter: {
bool: {
filter: [
{
bool: {
should: [
{
term: {
'result.evaluation': 'passed',
},
},
],
minimum_should_match: 1,
},
},
],
},
},
},
failed_findings_count: {
filter: {
bool: {
filter: [
{
bool: {
should: [
{
term: {
'result.evaluation': 'failed',
},
},
],
minimum_should_match: 1,
},
},
],
},
},
},
},
},
@ -211,24 +232,30 @@ const getRulesStatsQuery = (): SearchRequest => ({
_source: false,
});
const getCspmRulesStats = (aggregatedRulesStats: RulesStats, logger: Logger): CspmRulesStats[] => {
const rules = aggregatedRulesStats.rules.buckets;
const getCspmRulesStats = (aggregatedRulesStats: Accounts, logger: Logger): CspmRulesStats[] => {
const accounts = aggregatedRulesStats.accounts.buckets;
const cspmRulesStats = rules.map((rule) => ({
rule_id: rule.key,
rule_name: rule.rule_name.top[0].metrics['rule.name'],
rule_section: rule.rule_section.top[0].metrics['rule.section'],
rule_version: rule.rule_version.top[0].metrics['rule.version'],
rule_number: rule.rule_number.top[0].metrics['rule.benchmark.rule_number'],
posture_type: rule.posture_type.top[0].metrics['rule.benchmark.posture_type'],
benchmark_name: rule.benchmark_name.top[0].metrics['rule.benchmark.name'],
benchmark_id: rule.benchmark_id.top[0].metrics['rule.benchmark.id'],
passed_findings_count: rule.passed_findings_count.doc_count,
benchmark_version: rule.benchmark_version.top[0].metrics['rule.benchmark.version'],
failed_findings_count: rule.failed_findings_count.doc_count,
}));
const cspmRulesStats = accounts.map((account) => {
const accountId = account.key;
return account.rules.buckets.map((rule) => {
return {
account_id: accountId,
rule_id: rule.key,
rule_name: rule.rule_name.top[0].metrics['rule.name'],
rule_section: rule.rule_section.top[0].metrics['rule.section'],
rule_version: rule.rule_version.top[0].metrics['rule.version'],
rule_number: rule.rule_number.top[0].metrics['rule.benchmark.rule_number'],
posture_type: rule.posture_type.top[0].metrics['rule.benchmark.posture_type'],
benchmark_name: rule.benchmark_name.top[0].metrics['rule.benchmark.name'],
benchmark_id: rule.benchmark_id.top[0].metrics['rule.benchmark.id'],
benchmark_version: rule.benchmark_version.top[0].metrics['rule.benchmark.version'],
passed_findings_count: rule.passed_findings_count.doc_count,
failed_findings_count: rule.failed_findings_count.doc_count,
};
});
});
return cspmRulesStats;
return cspmRulesStats.flat(2);
};
export const getRulesStats = async (
@ -241,7 +268,7 @@ export const getRulesStats = async (
});
if (isIndexExists) {
const rulesStatsResponse = await esClient.search<unknown, RulesStats>(getRulesStatsQuery());
const rulesStatsResponse = await esClient.search<unknown, Accounts>(getRulesStatsQuery());
const cspmRulesStats = rulesStatsResponse.aggregations
? getCspmRulesStats(rulesStatsResponse.aggregations, logger)

2
x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/schema.ts
View file

@ -74,6 +74,7 @@ export const cspmUsageSchema: MakeSchemaFrom<CspmUsage> = {
benchmark_id: { type: 'keyword' },
benchmark_name: { type: 'keyword' },
benchmark_version: { type: 'keyword' },
kubernetes_version: { type: 'keyword' },
passed_findings_count: { type: 'long' },
failed_findings_count: { type: 'long' },
agents_count: { type: 'short' },
@ -84,6 +85,7 @@ export const cspmUsageSchema: MakeSchemaFrom<CspmUsage> = {
rules_stats: {
type: 'array',
items: {
account_id: { type: 'keyword' },
rule_id: { type: 'keyword' },
rule_name: { type: 'keyword' },
rule_section: { type: 'keyword' },

2
x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/types.ts
View file

@ -41,6 +41,7 @@ export interface CspmAccountsStats {
benchmark_id: string;
benchmark_name: string;
benchmark_version: string;
kubernetes_version: string | null;
passed_findings_count: number;
failed_findings_count: number;
agents_count: number;
@ -48,6 +49,7 @@ export interface CspmAccountsStats {
pods_count: number;
}
export interface CspmRulesStats {
account_id: string;
rule_id: string;
rule_name: string;
rule_section: string;

6
x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
View file

@ -5467,6 +5467,9 @@
"benchmark_version": {
"type": "keyword"
},
"kubernetes_version": {
"type": "keyword"
},
"passed_findings_count": {
"type": "long"
},
@ -5489,6 +5492,9 @@
"type": "array",
"items": {
"properties": {
"account_id": {
"type": "keyword"
},
"rule_id": {
"type": "keyword"
},

7
x-pack/test/cloud_security_posture_api/telemetry/data.ts
View file

@ -14,7 +14,8 @@ export interface MockTelemetryFindings {
result: { evaluation: string };
host: { name: string };
cluster_id?: string;
cloud?: { account: { id: string } };
cloud?: { account?: { id: string } };
cloudbeat?: { kubernetes: { version: string } };
}
export interface MockTelemetryData {
@ -81,6 +82,7 @@ export const data: MockTelemetryData = {
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' },
result: { evaluation: 'passed' },
host: { name: 'docker-fleet-agent' },
cloudbeat: { kubernetes: { version: 'v1.23.0' } },
},
{
cluster_id: 'my-k8s-cluster-5555',
@ -100,6 +102,7 @@ export const data: MockTelemetryData = {
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae3' },
result: { evaluation: 'passed' },
host: { name: 'control-plane' },
cloudbeat: { kubernetes: { version: 'v1.23.0' } },
},
],
kspmFindingsNoPostureType: [
@ -120,6 +123,7 @@ export const data: MockTelemetryData = {
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' },
result: { evaluation: 'passed' },
host: { name: 'docker-fleet-agent' },
cloudbeat: { kubernetes: { version: 'v1.23.0' } },
},
{
cluster_id: 'my-k8s-cluster-5555',
@ -138,6 +142,7 @@ export const data: MockTelemetryData = {
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae3' },
result: { evaluation: 'passed' },
host: { name: 'control-plane' },
cloudbeat: { kubernetes: { version: 'v1.23.0' } },
},
],
};

9
x-pack/test/cloud_security_posture_api/telemetry/telemetry.ts
View file

@ -82,6 +82,7 @@ export default function ({ getService }: FtrProviderContext) {
failed_findings_count: 0,
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
kubernetes_version: 'v1.23.0',
benchmark_version: 'v1.0.0',
agents_count: 2,
nodes_count: 2,
@ -134,6 +135,7 @@ export default function ({ getService }: FtrProviderContext) {
benchmark_name: 'CIS Amazon Web Services Foundations',
benchmark_id: 'cis_aws',
benchmark_version: 'v1.5.0',
kubernetes_version: null,
agents_count: 1,
nodes_count: 1,
pods_count: 0,
@ -178,6 +180,7 @@ export default function ({ getService }: FtrProviderContext) {
benchmark_name: 'CIS Amazon Web Services Foundations',
benchmark_id: 'cis_aws',
benchmark_version: 'v1.5.0',
kubernetes_version: null,
agents_count: 1,
nodes_count: 1,
pods_count: 0,
@ -191,6 +194,7 @@ export default function ({ getService }: FtrProviderContext) {
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
kubernetes_version: 'v1.23.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,
@ -228,7 +232,7 @@ export default function ({ getService }: FtrProviderContext) {
]);
});
it('includes only KSPM findings without posture_type', async () => {
it(`'includes only KSPM findings without posture_type'`, async () => {
await index.add(data.kspmFindingsNoPostureType);
const {
@ -252,6 +256,7 @@ export default function ({ getService }: FtrProviderContext) {
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
kubernetes_version: 'v1.23.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,
@ -305,6 +310,7 @@ export default function ({ getService }: FtrProviderContext) {
benchmark_name: 'CIS Amazon Web Services Foundations',
benchmark_id: 'cis_aws',
benchmark_version: 'v1.5.0',
kubernetes_version: null,
agents_count: 1,
nodes_count: 1,
pods_count: 0,
@ -318,6 +324,7 @@ export default function ({ getService }: FtrProviderContext) {
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
kubernetes_version: 'v1.23.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,

2
x-pack/test/cloud_security_posture_functional/config.ts
View file

@ -38,7 +38,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
* 2. merge the updated version number change to kibana
*/
`--xpack.fleet.packages.0.name=cloud_security_posture`,
`--xpack.fleet.packages.0.version=1.0.8`,
`--xpack.fleet.packages.0.version=1.2.10`,
// `--xpack.fleet.registryUrl=https://localhost:8080`,
],
},