github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 10:40:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[cloud security] support telemetry for cis_aws (#148964)

Browse source
This commit is contained in:
Ido Cohen 2023-01-29 13:13:29 +02:00 • committed by GitHub
parent c21df7a1cb
commit d9f4039ed6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 605 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.buildkite/ftr_configs.yml
View file

@ -142,6 +142,7 @@ enabled:
- x-pack/test/cases_api_integration/security_and_spaces/config_no_public_base_url.ts
- x-pack/test/cases_api_integration/spaces_only/config.ts
- x-pack/test/cloud_security_posture_functional/config.ts
- x-pack/test/cloud_security_posture_api/config.ts
- x-pack/test/detection_engine_api_integration/basic/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
- x-pack/test/detection_engine_api_integration/security_and_spaces/group2/config.ts

46
x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/accounts_stats_collector.ts
View file

@ -6,7 +6,7 @@
*/
import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import type { Logger } from '@kbn/core/server';
import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
import type { MappingRuntimeFields, SearchRequest } from '@elastic/elasticsearch/lib/api/types';
import { calculatePostureScore } from '../../../../common/utils/helpers';
import type { CspmAccountsStats } from './types';
import { LATEST_FINDINGS_INDEX_DEFAULT_NS } from '../../../../common/constants';
@ -52,15 +52,57 @@ interface AccountEntity {
};
}
// The runtime field help to have unique identifier for CSPM and KSPM
export const getIdentifierRuntimeMapping = (): MappingRuntimeFields => ({
asset_identifier: {
type: 'keyword',
script: {
source: `
if (!doc.containsKey('rule.benchmark.posture_type'))
{
def identifier = doc["cluster_id"].value;
emit(identifier);
return
}
else
{
if(doc["rule.benchmark.posture_type"].size() > 0)
{
def policy_template_type = doc["rule.benchmark.posture_type"].value;
if (policy_template_type == "cspm")
{
def identifier = doc["cloud.account.id"].value;
emit(identifier);
return
}
if (policy_template_type == "kspm")
{
def identifier = doc["cluster_id"].value;
emit(identifier);
return
}
}
def identifier = doc["cluster_id"].value;
emit(identifier);
return
}
`,
},
},
});
const getAccountsStatsQuery = (index: string): SearchRequest => ({
index,
runtime_mappings: getIdentifierRuntimeMapping(),
query: {
match_all: {},
},
aggs: {
accounts: {
terms: {
field: 'cluster_id',
field: 'asset_identifier',
order: {
_count: 'desc',
},

4
x-pack/plugins/cloud_security_posture/server/lib/telemetry/collectors/resources_stats_collector.ts
View file

@ -9,6 +9,7 @@ import type { Logger } from '@kbn/core/server';
import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types';
import type { CspmResourcesStats } from './types';
import { LATEST_FINDINGS_INDEX_DEFAULT_NS } from '../../../../common/constants';
import { getIdentifierRuntimeMapping } from './accounts_stats_collector';
interface ResourcesStats {
accounts: {
@ -50,10 +51,11 @@ const getResourcesStatsQuery = (index: string): SearchRequest => ({
query: {
match_all: {},
},
runtime_mappings: getIdentifierRuntimeMapping(),
aggs: {
accounts: {
terms: {
field: 'cluster_id',
field: 'asset_identifier',
order: {
_count: 'desc',
},

44
x-pack/test/cloud_security_posture_api/config.ts Normal file
View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { FtrConfigProviderContext } from '@kbn/test';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const xpackFunctionalConfig = await readConfigFile(
require.resolve('../functional/config.base.js')
);
return {
...xpackFunctionalConfig.getAll(),
testFiles: [require.resolve('./telemetry/telemetry.ts')],
junit: {
reportName: 'X-Pack Cloud Security Posture API Tests',
},
kbnTestServer: {
...xpackFunctionalConfig.get('kbnTestServer'),
serverArgs: [
...xpackFunctionalConfig.get('kbnTestServer.serverArgs'),
/**
* Package version is fixed (not latest) so FTR won't suddenly break when package is changed.
*
* test a new package:
* 1. build the package and start the registry with elastic-package and uncomment the 'registryUrl' flag below
* 2. locally checkout the kibana version that matches the new package
* 3. update the package version below to use the new package version
* 4. run tests with NODE_EXTRA_CA_CERTS pointing to the elastic-package certificate. example:
* NODE_EXTRA_CA_CERTS=HOME/.elastic-package/profiles/default/certs/kibana/ca-cert.pem yarn start
* 5. when test pass:
* 1. release a new package to EPR
* 2. merge the updated version number change to kibana
*/
`--xpack.fleet.packages.0.name=cloud_security_posture`,
`--xpack.fleet.packages.0.version=1.2.8`,
// `--xpack.fleet.registryUrl=https://localhost:8080`,
],
},
};
}

12
x-pack/test/cloud_security_posture_api/ftr_provider_context.d.ts vendored Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { GenericFtrProviderContext } from '@kbn/test';
import { services } from '../api_integration/services';
export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;

143
x-pack/test/cloud_security_posture_api/telemetry/data.ts Normal file
View file

@ -0,0 +1,143 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface MockTelemetryFindings {
rule: {
benchmark: { id: string; posture_type?: string | undefined; version: string; name: string };
};
resource: { type: string; sub_type: string; id: string };
agent: { id: string };
result: { evaluation: string };
host: { name: string };
cluster_id?: string;
cloud?: { account: { id: string } };
}
export interface MockTelemetryData {
[key: string]: MockTelemetryFindings[];
}
export const data: MockTelemetryData = {
cspmFindings: [
{
rule: {
benchmark: {
id: 'cis_aws',
posture_type: 'cspm',
version: 'v1.5.0',
name: 'CIS Amazon Web Services Foundations',
},
},
resource: {
type: 'identifyingType',
sub_type: 'aws-password-policy',
id: '15e450b7-8980-5bca-ade2-a0c795f9ea9d',
},
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' },
result: { evaluation: 'failed' },
cloud: { account: { id: 'my-aws-12345' } },
host: { name: 'docker-fleet-agent' },
},
{
rule: {
benchmark: {
id: 'cis_aws',
posture_type: 'cspm',
version: 'v1.5.0',
name: 'CIS Amazon Web Services Foundations',
},
},
resource: {
type: 'identifyingType',
sub_type: 'aws-password-policy',
id: '15e450b7-8980-5bca-ade2-a0c795f9ea99',
},
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' },
result: { evaluation: 'passed' },
cloud: { account: { id: 'my-aws-12345' } },
host: { name: 'docker-fleet-agent' },
},
],
kspmFindings: [
{
cluster_id: 'my-k8s-cluster-5555',
rule: {
benchmark: {
id: 'cis_k8s',
version: 'v1.0.0',
name: 'CIS Kubernetes V1.23',
posture_type: 'kspm',
},
},
resource: {
type: 'k8s_object',
sub_type: 'ServiceAccount',
id: '1111',
},
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' },
result: { evaluation: 'passed' },
host: { name: 'docker-fleet-agent' },
},
{
cluster_id: 'my-k8s-cluster-5555',
rule: {
benchmark: {
id: 'cis_k8s',
version: 'v1.0.0',
name: 'CIS Kubernetes V1.23',
posture_type: 'kspm',
},
},
resource: {
type: 'process',
sub_type: 'process',
id: '1111',
},
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae3' },
result: { evaluation: 'passed' },
host: { name: 'control-plane' },
},
],
kspmFindingsNoPostureType: [
{
cluster_id: 'my-k8s-cluster-5555',
rule: {
benchmark: {
id: 'cis_k8s',
version: 'v1.0.0',
name: 'CIS Kubernetes V1.23',
},
},
resource: {
type: 'k8s_object',
sub_type: 'ServiceAccount',
id: '1111',
},
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae2' },
result: { evaluation: 'passed' },
host: { name: 'docker-fleet-agent' },
},
{
cluster_id: 'my-k8s-cluster-5555',
rule: {
benchmark: {
id: 'cis_k8s',
version: 'v1.0.0',
name: 'CIS Kubernetes V1.23',
},
},
resource: {
type: 'process',
sub_type: 'process',
id: '1111',
},
agent: { id: '07bd3686-98ef-4b23-99cb-9ff544b25ae3' },
result: { evaluation: 'passed' },
host: { name: 'control-plane' },
},
],
};

358
x-pack/test/cloud_security_posture_api/telemetry/telemetry.ts Normal file
View file

@ -0,0 +1,358 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { data, MockTelemetryFindings } from './data';
import type { FtrProviderContext } from '../ftr_provider_context';
const FINDINGS_INDEX = 'logs-cloud_security_posture.findings_latest-default';
// eslint-disable-next-line import/no-default-export
export default function ({ getService }: FtrProviderContext) {
const retry = getService('retry');
const es = getService('es');
const supertest = getService('supertest');
const log = getService('log');
/**
* required before indexing findings
*/
const waitForPluginInitialized = (): Promise<void> =>
retry.try(async () => {
log.debug('Check CSP plugin is initialized');
const response = await supertest
.get('/internal/cloud_security_posture/status?check=init')
.expect(200);
expect(response.body).to.eql({ isPluginInitialized: true });
log.debug('CSP plugin is initialized');
});
const index = {
remove: () =>
es.deleteByQuery({
index: FINDINGS_INDEX,
query: { match_all: {} },
refresh: true,
}),
add: async (mockTelemetryFindings: MockTelemetryFindings[]) => {
const operations = mockTelemetryFindings.flatMap((doc) => [
{ index: { _index: FINDINGS_INDEX } },
doc,
]);
const response = await es.bulk({ refresh: 'wait_for', index: FINDINGS_INDEX, operations });
expect(response.errors).to.eql(false);
},
};
describe('Verify cloud_security_posture telemetry payloads', async () => {
before(async () => {
await waitForPluginInitialized();
});
afterEach(async () => {
await index.remove();
});
it('includes only KSPM findings', async () => {
await index.add(data.kspmFindings);
const {
body: [{ stats: apiResponse }],
} = await supertest
.post(`/api/telemetry/v2/clusters/_stats`)
.set('kbn-xsrf', 'xxxx')
.send({
unencrypted: true,
refreshCache: true,
})
.expect(200);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.accounts_stats).to.eql([
{
account_id: 'my-k8s-cluster-5555',
latest_findings_doc_count: 2,
posture_score: 100,
passed_findings_count: 2,
failed_findings_count: 0,
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,
},
]);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.resources_stats).to.eql([
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'k8s_object',
resource_type_doc_count: 1,
resource_sub_type: 'ServiceAccount',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'process',
resource_type_doc_count: 1,
resource_sub_type: 'process',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
]);
});
it('includes only CSPM findings', async () => {
await index.add(data.cspmFindings);
const {
body: [{ stats: apiResponse }],
} = await supertest
.post(`/api/telemetry/v2/clusters/_stats`)
.set('kbn-xsrf', 'xxxx')
.send({
unencrypted: true,
refreshCache: true,
})
.expect(200);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.accounts_stats).to.eql([
{
account_id: 'my-aws-12345',
latest_findings_doc_count: 2,
posture_score: 50,
passed_findings_count: 1,
failed_findings_count: 1,
benchmark_name: 'CIS Amazon Web Services Foundations',
benchmark_id: 'cis_aws',
benchmark_version: 'v1.5.0',
agents_count: 1,
nodes_count: 1,
pods_count: 0,
},
]);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.resources_stats).to.eql([
{
account_id: 'my-aws-12345',
resource_type: 'identifyingType',
resource_type_doc_count: 2,
resource_sub_type: 'aws-password-policy',
resource_sub_type_doc_count: 2,
passed_findings_count: 1,
failed_findings_count: 1,
},
]);
});
it('includes CSPM and KSPM findings', async () => {
await index.add(data.kspmFindings);
await index.add(data.cspmFindings);
const {
body: [{ stats: apiResponse }],
} = await supertest
.post(`/api/telemetry/v2/clusters/_stats`)
.set('kbn-xsrf', 'xxxx')
.send({
unencrypted: true,
refreshCache: true,
})
.expect(200);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.accounts_stats).to.eql([
{
account_id: 'my-aws-12345',
latest_findings_doc_count: 2,
posture_score: 50,
passed_findings_count: 1,
failed_findings_count: 1,
benchmark_name: 'CIS Amazon Web Services Foundations',
benchmark_id: 'cis_aws',
benchmark_version: 'v1.5.0',
agents_count: 1,
nodes_count: 1,
pods_count: 0,
},
{
account_id: 'my-k8s-cluster-5555',
latest_findings_doc_count: 2,
posture_score: 100,
passed_findings_count: 2,
failed_findings_count: 0,
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,
},
]);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.resources_stats).to.eql([
{
account_id: 'my-aws-12345',
resource_type: 'identifyingType',
resource_type_doc_count: 2,
resource_sub_type: 'aws-password-policy',
resource_sub_type_doc_count: 2,
passed_findings_count: 1,
failed_findings_count: 1,
},
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'k8s_object',
resource_type_doc_count: 1,
resource_sub_type: 'ServiceAccount',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'process',
resource_type_doc_count: 1,
resource_sub_type: 'process',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
]);
});
it('includes only KSPM findings without posture_type', async () => {
await index.add(data.kspmFindingsNoPostureType);
const {
body: [{ stats: apiResponse }],
} = await supertest
.post(`/api/telemetry/v2/clusters/_stats`)
.set('kbn-xsrf', 'xxxx')
.send({
unencrypted: true,
refreshCache: true,
})
.expect(200);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.accounts_stats).to.eql([
{
account_id: 'my-k8s-cluster-5555',
latest_findings_doc_count: 2,
posture_score: 100,
passed_findings_count: 2,
failed_findings_count: 0,
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,
},
]);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.resources_stats).to.eql([
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'k8s_object',
resource_type_doc_count: 1,
resource_sub_type: 'ServiceAccount',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'process',
resource_type_doc_count: 1,
resource_sub_type: 'process',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
]);
});
it('includes KSPM findings without posture_type and CSPM findings as well', async () => {
await index.add(data.kspmFindingsNoPostureType);
await index.add(data.cspmFindings);
const {
body: [{ stats: apiResponse }],
} = await supertest
.post(`/api/telemetry/v2/clusters/_stats`)
.set('kbn-xsrf', 'xxxx')
.send({
unencrypted: true,
refreshCache: true,
})
.expect(200);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.accounts_stats).to.eql([
{
account_id: 'my-aws-12345',
latest_findings_doc_count: 2,
posture_score: 50,
passed_findings_count: 1,
failed_findings_count: 1,
benchmark_name: 'CIS Amazon Web Services Foundations',
benchmark_id: 'cis_aws',
benchmark_version: 'v1.5.0',
agents_count: 1,
nodes_count: 1,
pods_count: 0,
},
{
account_id: 'my-k8s-cluster-5555',
latest_findings_doc_count: 2,
posture_score: 100,
passed_findings_count: 2,
failed_findings_count: 0,
benchmark_name: 'CIS Kubernetes V1.23',
benchmark_id: 'cis_k8s',
benchmark_version: 'v1.0.0',
agents_count: 2,
nodes_count: 2,
pods_count: 0,
},
]);
expect(apiResponse.stack_stats.kibana.plugins.cloud_security_posture.resources_stats).to.eql([
{
account_id: 'my-aws-12345',
resource_type: 'identifyingType',
resource_type_doc_count: 2,
resource_sub_type: 'aws-password-policy',
resource_sub_type_doc_count: 2,
passed_findings_count: 1,
failed_findings_count: 1,
},
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'k8s_object',
resource_type_doc_count: 1,
resource_sub_type: 'ServiceAccount',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
{
account_id: 'my-k8s-cluster-5555',
resource_type: 'process',
resource_type_doc_count: 1,
resource_sub_type: 'process',
resource_sub_type_doc_count: 1,
passed_findings_count: 1,
failed_findings_count: 0,
},
]);
});
});
}