github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Cloud Security][Bug Fix] Dashboard and Findings page now will only show result within retention period (#160313)

Browse source 
## Summary

Currently Findings and Dashboard page kept showing data even if the data
past its retention period. This PR will make it so that, it will only
show data within that retention period, If theres no more data past that
retention period it will show No Agent deployed prompt instead

Our solution/workaround for this will be adding a retention periods for
each kind of posture type:
**KSPM** and **CSPM** : 26 hours
**CNVM** and **all** : 3 days or 72 hours
if posture type doesnt exist : 72 hours (this is to cover the situation
where findings came before CSPM gets implemented back then)

Before when having Findings outside retention period:
<img width="1391" alt="Screenshot 2023-11-07 at 5 03 15 PM"
src="26f74ebf-8002-4da9-b4f1-8238e81cea5e">

After: (27 hours later)
<img width="1452" alt="Screenshot 2023-11-07 at 4 52 09 PM"
src="98a33379-dc21-4edf-97b6-32de584598e9">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Rickyanto Ang 2023-11-20 11:22:21 -08:00 • committed by GitHub
parent 8fd18fbb05
commit 5d8c050826
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 198 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

43
x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts
View file

@ -9,25 +9,45 @@ import { ElasticsearchClient, type Logger } from '@kbn/core/server';
import { getSafePostureTypeRuntimeMapping } from '../../common/runtime_mappings/get_safe_posture_type_runtime_mapping';
import { IndexStatus, PostureTypes } from '../../common/types';
export interface PostureTypeAndRetention {
postureType?: PostureTypes;
retentionTime?: string;
}
export const checkIndexStatus = async (
esClient: ElasticsearchClient,
index: string,
logger: Logger,
postureType?: PostureTypes
PostureTypeAndRetention?: PostureTypeAndRetention
): Promise<IndexStatus> => {
const query =
!postureType || postureType === 'all' || postureType === 'vuln_mgmt'
? undefined
: {
bool: {
filter: {
term: {
safe_posture_type: postureType,
const isNotKspmOrCspm =
!PostureTypeAndRetention?.postureType ||
PostureTypeAndRetention?.postureType === 'all' ||
PostureTypeAndRetention?.postureType === 'vuln_mgmt';
const query = {
bool: {
filter: [
...(isNotKspmOrCspm
? []
: [
{
term: {
safe_posture_type: PostureTypeAndRetention?.postureType,
},
},
]),
{
range: {
'@timestamp': {
gte: `now-${PostureTypeAndRetention?.retentionTime}`,
lte: 'now',
},
},
};
},
],
},
};
try {
const queryResult = await esClient.search({
index,
@ -37,7 +57,6 @@ export const checkIndexStatus = async (
query,
size: 1,
});
return queryResult.hits.hits.length ? 'not-empty' : 'empty';
} catch (e) {
logger.debug(e);

59
x-pack/plugins/cloud_security_posture/server/routes/status/status.ts
View file

@ -29,6 +29,9 @@ import {
POSTURE_TYPES,
LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
VULN_MGMT_POLICY_TEMPLATE,
POSTURE_TYPE_ALL,
LATEST_VULNERABILITIES_RETENTION_POLICY,
LATEST_FINDINGS_RETENTION_POLICY,
} from '../../../common/constants';
import type {
CspApiRequestHandlerContext,
@ -168,20 +171,53 @@ export const getCspStatus = async ({
installedPackagePoliciesVulnMgmt,
installedPolicyTemplates,
] = await Promise.all([
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger),
checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger),
checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
postureType: POSTURE_TYPE_ALL,
retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
}),
checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, {
postureType: POSTURE_TYPE_ALL,
retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
}),
checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, {
postureType: POSTURE_TYPE_ALL,
retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
}),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, 'cspm'),
checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, 'cspm'),
checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'cspm'),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
postureType: CSPM_POLICY_TEMPLATE,
retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
}),
checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, {
postureType: CSPM_POLICY_TEMPLATE,
retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
}),
checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, {
postureType: CSPM_POLICY_TEMPLATE,
retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
}),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, 'kspm'),
checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, 'kspm'),
checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'kspm'),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {
postureType: KSPM_POLICY_TEMPLATE,
retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
}),
checkIndexStatus(esClient, FINDINGS_INDEX_PATTERN, logger, {
postureType: KSPM_POLICY_TEMPLATE,
retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
}),
checkIndexStatus(esClient, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, {
postureType: KSPM_POLICY_TEMPLATE,
retentionTime: LATEST_FINDINGS_RETENTION_POLICY,
}),
checkIndexStatus(esClient, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, logger),
checkIndexStatus(esClient, VULNERABILITIES_INDEX_PATTERN, logger, VULN_MGMT_POLICY_TEMPLATE),
checkIndexStatus(esClient, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, logger, {
postureType: VULN_MGMT_POLICY_TEMPLATE,
retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
}),
checkIndexStatus(esClient, VULNERABILITIES_INDEX_PATTERN, logger, {
postureType: VULN_MGMT_POLICY_TEMPLATE,
retentionTime: LATEST_VULNERABILITIES_RETENTION_POLICY,
}),
packageService.asInternalUser.getInstallation(CLOUD_SECURITY_POSTURE_PACKAGE_NAME),
packageService.asInternalUser.fetchFindLatestPackage(CLOUD_SECURITY_POSTURE_PACKAGE_NAME),
@ -295,6 +331,7 @@ export const getCspStatus = async ({
{
latest: vulnerabilitiesLatestIndexStatus,
stream: vulnerabilitiesIndexStatus,
score: scoreIndexStatus,
},
installation,
healthyAgentsVulMgmt,

8
x-pack/test/api_integration/apis/cloud_security_posture/mock_data.ts
View file

@ -24,6 +24,10 @@ export const findingsMockData = [
type: 'process',
},
cluster_id: 'Upper case cluster id',
event: {
ingested: '2023-08-19T18:20:41Z',
created: '2023-08-19T18:17:15.609124281Z',
},
},
{
resource: { id: chance.guid(), name: `Pod`, sub_type: 'Upper case sub type' },
@ -40,6 +44,10 @@ export const findingsMockData = [
type: 'process',
},
cluster_id: 'Another Upper case cluster id',
event: {
ingested: '2023-08-19T18:20:41Z',
created: '2023-08-19T18:17:15.609124281Z',
},
},
];

4
x-pack/test/api_integration/apis/cloud_security_posture/status/status_unprivileged.ts
View file

@ -247,8 +247,8 @@ export default function (providerContext: FtrProviderContext) {
`expected unprivileged but got ${res.cspm.status} instead`
);
expect(res.vuln_mgmt.status).to.eql(
'not-installed',
`expected not-installed but got ${res.vuln_mgmt.status} instead`
'unprivileged',
`expected unprivileged but got ${res.vuln_mgmt.status} instead`
);
assertIndexStatus(res.indicesDetails, LATEST_FINDINGS_INDEX_DEFAULT_NS, 'unprivileged');

4
x-pack/test/cloud_security_posture_functional/mocks/vulnerabilities_latest_mock.ts
View file

@ -83,7 +83,7 @@ export const vulnerabilitiesLatestMock = [
id: '704479110758',
},
},
'@timestamp': '2023-06-29T02:08:44.993Z',
'@timestamp': (Date.now() - 249200000).toString(),
cloudbeat: {
commit_sha: '4d990caa0c9c1594441da6bf24a685599aeb2bd5',
commit_time: '2023-05-15T14:48:10Z',
@ -189,7 +189,7 @@ export const vulnerabilitiesLatestMock = [
id: '704479110758',
},
},
'@timestamp': '2023-06-29T02:08:16.535Z',
'@timestamp': (Date.now() - 249200000).toString(),
ecs: {
version: '8.6.0',
},

7
x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
View file

@ -410,6 +410,12 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
},
});
const isLatestFindingsTableThere = async () => {
const table = await testSubjects.findAll('docTable');
const trueOrFalse = table.length > 0 ? true : false;
return trueOrFalse;
};
return {
navigateToLatestFindingsPage,
navigateToVulnerabilities,
@ -426,5 +432,6 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
misconfigurationsFlyout,
toastMessage,
detectionRuleApi,
isLatestFindingsTableThere,
};
}

10
x-pack/test/cloud_security_posture_functional/pages/findings.ts
View file

@ -17,12 +17,13 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
const retry = getService('retry');
const pageObjects = getPageObjects(['common', 'findings', 'header']);
const chance = new Chance();
const timeFiveHoursAgo = (Date.now() - 18000000).toString();
// We need to use a dataset for the tests to run
// We intentionally make some fields start with a capital letter to test that the query bar is case-insensitive/case-sensitive
const data = [
{
'@timestamp': '1695819664234',
'@timestamp': timeFiveHoursAgo,
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
@ -39,7 +40,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
cluster_id: 'Upper case cluster id',
},
{
'@timestamp': '1695819673242',
'@timestamp': timeFiveHoursAgo,
resource: { id: chance.guid(), name: `Pod`, sub_type: 'Upper case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
@ -56,7 +57,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
cluster_id: 'Another Upper case cluster id',
},
{
'@timestamp': '1695819676242',
'@timestamp': timeFiveHoursAgo,
resource: { id: chance.guid(), name: `process`, sub_type: 'another lower case type' },
result: { evaluation: 'passed' },
rule: {
@ -73,7 +74,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
cluster_id: 'lower case cluster id',
},
{
'@timestamp': '1695819680202',
'@timestamp': timeFiveHoursAgo,
resource: { id: chance.guid(), name: `process`, sub_type: 'Upper case type again' },
result: { evaluation: 'failed' },
rule: {
@ -122,7 +123,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
await findings.index.add(data);
await findings.navigateToLatestFindingsPage();
await retry.waitFor(
'Findings table to be loaded',
async () => (await latestFindingsTable.getRowsCount()) === data.length

94
x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts Normal file
View file

@ -0,0 +1,94 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import Chance from 'chance';
import type { FtrProviderContext } from '../ftr_provider_context';
// eslint-disable-next-line import/no-default-export
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const pageObjects = getPageObjects(['common', 'findings', 'header']);
const chance = new Chance();
const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000;
const dataOldKspm = [
{
'@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
name: 'Upper case rule name',
section: 'Upper case section',
benchmark: {
id: 'cis_k8s',
posture_type: 'kspm',
name: 'CIS Kubernetes V1.23',
version: 'v1.0.0',
},
type: 'process',
},
cluster_id: 'Upper case cluster id',
},
];
const dataOldCspm = [
{
'@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
name: 'Upper case rule name',
section: 'Upper case section',
benchmark: {
id: 'cis_aws',
posture_type: 'cspm',
name: 'CIS AWS V1.23',
version: 'v1.0.0',
},
type: 'process',
},
cluster_id: 'Upper case cluster id',
},
];
describe('Old Data', function () {
this.tags(['cloud_security_posture_findings']);
let findings: typeof pageObjects.findings;
before(async () => {
findings = pageObjects.findings;
// Before we start any test we must wait for cloud_security_posture plugin to complete its initialization
await findings.waitForPluginInitialized();
});
after(async () => {
await findings.index.remove();
});
describe('Findings page with old data', () => {
it('returns no Findings KSPM', async () => {
// Prepare mocked findings
await findings.index.remove();
await findings.index.add(dataOldKspm);
await findings.navigateToLatestFindingsPage();
pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns no Findings CSPM', async () => {
// Prepare mocked findings
await findings.index.remove();
await findings.index.add(dataOldCspm);
await findings.navigateToLatestFindingsPage();
pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
});
});
}

1
x-pack/test/cloud_security_posture_functional/pages/index.ts
View file

@ -16,5 +16,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./compliance_dashboard'));
loadTestFile(require.resolve('./vulnerability_dashboard'));
loadTestFile(require.resolve('./cis_integration'));
loadTestFile(require.resolve('./findings_old_data'));
});
}