github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.x] [Cloud Security] Increase retention period on queries related to 3rd party data loading (#195636) (#195871)

Browse source 
# Backport

This will backport the following commits from `main` to `8.x`:
- [[Cloud Security] Increase retention period on queries related to 3rd
party data loading
(#195636)](https://github.com/elastic/kibana/pull/195636)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Maxim
Kholod","email":"maxim.kholod@elastic.co"},"sourceCommit":{"committedDate":"2024-10-11T09:34:12Z","message":"[Cloud
Security] Increase retention period on queries related to 3rd party data
loading (#195636)\n\n## Summary\r\n\r\nIncrease retention on Cloud
Security queries to accommodate a longer\r\nretention period on
third-party CDR integrations, such as Wiz and AWS\r\nSecurityHub. This
introduces regression
for\r\nhttps://github.com/elastic/kibana/issues/142198\r\nThis is meant
is a temporary workaround until we find a robust way to\r\nget full
posture for third-party CDR integrations\r\nThis change goes together
with increasing retention period on Wiz: \r\n-
https://github.com/elastic/integrations/pull/11393\r\n\r\nfixes:\r\n-
https://github.com/elastic/security-team/issues/10683\r\n\r\n## How to
test\r\nThe CI deployed a serverless project where I installed Wiz and
CSP\r\nintegrations and ingested some
data.","sha":"e18c52eec2cb18dc2590b61d7649de4507f060a7","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Cloud
Security","backport:prev-major","ci:cloud-deploy","ci:project-deploy-security","v8.16.0"],"title":"[Cloud
Security] Increase retention period on queries related to 3rd party data
loading","number":195636,"url":"https://github.com/elastic/kibana/pull/195636","mergeCommit":{"message":"[Cloud
Security] Increase retention period on queries related to 3rd party data
loading (#195636)\n\n## Summary\r\n\r\nIncrease retention on Cloud
Security queries to accommodate a longer\r\nretention period on
third-party CDR integrations, such as Wiz and AWS\r\nSecurityHub. This
introduces regression
for\r\nhttps://github.com/elastic/kibana/issues/142198\r\nThis is meant
is a temporary workaround until we find a robust way to\r\nget full
posture for third-party CDR integrations\r\nThis change goes together
with increasing retention period on Wiz: \r\n-
https://github.com/elastic/integrations/pull/11393\r\n\r\nfixes:\r\n-
https://github.com/elastic/security-team/issues/10683\r\n\r\n## How to
test\r\nThe CI deployed a serverless project where I installed Wiz and
CSP\r\nintegrations and ingested some
data.","sha":"e18c52eec2cb18dc2590b61d7649de4507f060a7"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195636","number":195636,"mergeCommit":{"message":"[Cloud
Security] Increase retention period on queries related to 3rd party data
loading (#195636)\n\n## Summary\r\n\r\nIncrease retention on Cloud
Security queries to accommodate a longer\r\nretention period on
third-party CDR integrations, such as Wiz and AWS\r\nSecurityHub. This
introduces regression
for\r\nhttps://github.com/elastic/kibana/issues/142198\r\nThis is meant
is a temporary workaround until we find a robust way to\r\nget full
posture for third-party CDR integrations\r\nThis change goes together
with increasing retention period on Wiz: \r\n-
https://github.com/elastic/integrations/pull/11393\r\n\r\nfixes:\r\n-
https://github.com/elastic/security-team/issues/10683\r\n\r\n## How to
test\r\nThe CI deployed a serverless project where I installed Wiz and
CSP\r\nintegrations and ingested some
data.","sha":"e18c52eec2cb18dc2590b61d7649de4507f060a7"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Maxim Kholod <maxim.kholod@elastic.co>
This commit is contained in:
Kibana Machine 2024-10-11 22:43:30 +11:00 • committed by GitHub
parent 994d97ffd4
commit 8f86639706
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 121 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/packages/kbn-cloud-security-posture-common/constants.ts
View file

@ -36,6 +36,10 @@ export const CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN =
export const CDR_VULNERABILITIES_INDEX_PATTERN = `${CDR_LATEST_THIRD_PARTY_VULNERABILITIES_INDEX_PATTERN},${CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN}`;
export const LATEST_VULNERABILITIES_RETENTION_POLICY = '3d';
// TODO: remove once https://github.com/elastic/security-team/issues/10801 is done
// meant as a temp workaround to get good enough posture view for 3rd party integrations, see https://github.com/elastic/security-team/issues/10683
export const CDR_3RD_PARTY_RETENTION_POLICY = '90d';
export const VULNERABILITIES_SEVERITY: Record<VulnSeverity, VulnSeverity> = {
LOW: 'LOW',
MEDIUM: 'MEDIUM',

7
x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts
View file

@ -9,8 +9,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import {
CDR_MISCONFIGURATIONS_INDEX_PATTERN,
CDR_VULNERABILITIES_INDEX_PATTERN,
LATEST_FINDINGS_RETENTION_POLICY,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
import { buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
@ -103,7 +102,7 @@ const buildMisconfigurationsFindingsQueryWithFilters = (
{
range: {
'@timestamp': {
gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},
@ -182,7 +181,7 @@ const buildVulnerabilityFindingsQueryWithFilters = (query: UseCspOptions['query'
{
range: {
'@timestamp': {
gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},

4
x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts
View file

@ -15,7 +15,7 @@ import { showErrorToast } from '@kbn/cloud-security-posture';
import { MAX_FINDINGS_TO_LOAD, buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
import {
CDR_MISCONFIGURATIONS_INDEX_PATTERN,
LATEST_FINDINGS_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import type { CspFinding } from '@kbn/cloud-security-posture-common';
import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
@ -77,7 +77,7 @@ export const getFindingsQuery = (
{
range: {
'@timestamp': {
gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},

4
x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx
View file

@ -16,7 +16,7 @@ import {
import { useMemo } from 'react';
import { buildEsQuery, Filter } from '@kbn/es-query';
import {
LATEST_FINDINGS_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
buildMutedRulesFilter,
} from '@kbn/cloud-security-posture-common';
import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
@ -249,7 +249,7 @@ export const useLatestFindingsGrouping = ({
additionalFilters: query ? [query, additionalFilters] : [additionalFilters],
groupByField: currentSelectedGroup,
uniqueValue,
from: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
to: 'now',
pageNumber: activePageIndex * pageSize,
size: pageSize,

4
x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx
View file

@ -19,7 +19,7 @@ import { EsHitRecord } from '@kbn/discover-utils/types';
import {
MAX_FINDINGS_TO_LOAD,
CDR_VULNERABILITIES_INDEX_PATTERN,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
@ -90,7 +90,7 @@ export const getVulnerabilitiesQuery = (
{
range: {
'@timestamp': {
gte: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
gte: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
lte: 'now',
},
},

4
x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx
View file

@ -15,7 +15,7 @@ import {
} from '@kbn/grouping/src';
import { useMemo } from 'react';
import {
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
VULNERABILITIES_SEVERITY,
} from '@kbn/cloud-security-posture-common';
import { buildEsQuery, Filter } from '@kbn/es-query';
@ -202,7 +202,7 @@ export const useLatestVulnerabilitiesGrouping = ({
additionalFilters: query ? [query, additionalFilters] : [additionalFilters],
groupByField: currentSelectedGroup,
uniqueValue,
from: `now-${LATEST_VULNERABILITIES_RETENTION_POLICY}`,
from: `now-${CDR_3RD_PARTY_RETENTION_POLICY}`,
to: 'now',
pageNumber: activePageIndex * pageSize,
size: pageSize,

5
x-pack/plugins/cloud_security_posture/server/routes/status/status.ts
View file

@ -15,6 +15,7 @@ import {
CDR_LATEST_NATIVE_VULNERABILITIES_INDEX_PATTERN,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_VULNERABILITIES_INDEX_PATTERN,
CDR_3RD_PARTY_RETENTION_POLICY,
} from '@kbn/cloud-security-posture-common';
import type {
CspSetupStatus,
@ -218,13 +219,13 @@ export const getCspStatus = async ({
checkIndexHasFindings(
esClient,
CDR_MISCONFIGURATIONS_INDEX_PATTERN,
LATEST_FINDINGS_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
logger
),
checkIndexHasFindings(
esClient,
CDR_VULNERABILITIES_INDEX_PATTERN,
LATEST_VULNERABILITIES_RETENTION_POLICY,
CDR_3RD_PARTY_RETENTION_POLICY,
logger
),
checkIndexStatus(esClient, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger, {

3
x-pack/test/cloud_security_posture_functional/page_objects/findings_page.ts
View file

@ -366,8 +366,7 @@ export function FindingsPageProvider({ getService, getPageObjects }: FtrProvider
});
const isLatestFindingsTableThere = async () => {
const table = await testSubjects.findAll('docTable');
const trueOrFalse = table.length > 0 ? true : false;
return trueOrFalse;
return table.length > 0;
};
const getUnprivilegedPrompt = async () => {

111
x-pack/test/cloud_security_posture_functional/pages/findings_old_data.ts
View file

@ -8,16 +8,38 @@
import expect from '@kbn/expect';
import Chance from 'chance';
import type { FtrProviderContext } from '../ftr_provider_context';
import { vulnerabilitiesLatestMock } from '../mocks/vulnerabilities_latest_mock';
// eslint-disable-next-line import/no-default-export
export default function ({ getPageObjects }: FtrProviderContext) {
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const retry = getService('retry');
const pageObjects = getPageObjects(['common', 'findings', 'header']);
const chance = new Chance();
const hoursToMillisecond = (hours: number) => hours * 60 * 60 * 1000;
const daysToMillisecond = (days: number) => days * 24 * 60 * 60 * 1000;
const RETENTION = 90;
const dataOldKspm = [
{
'@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
'@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
name: 'Upper case rule name',
section: 'Upper case section',
benchmark: {
id: 'cis_k8s',
posture_type: 'kspm',
name: 'CIS Kubernetes V1.23',
version: 'v1.0.0',
},
type: 'process',
},
cluster_id: 'Upper case cluster id',
},
];
const dataWithinRetentionKspm = [
{
'@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
@ -37,7 +59,26 @@ export default function ({ getPageObjects }: FtrProviderContext) {
const dataOldCspm = [
{
'@timestamp': (Date.now() - hoursToMillisecond(27)).toString(),
'@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
name: 'Upper case rule name',
section: 'Upper case section',
benchmark: {
id: 'cis_aws',
posture_type: 'cspm',
name: 'CIS AWS V1.23',
version: 'v1.0.0',
},
type: 'process',
},
cluster_id: 'Upper case cluster id',
},
];
const dataWithinRetentionCspm = [
{
'@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },
result: { evaluation: chance.integer() % 2 === 0 ? 'passed' : 'failed' },
rule: {
@ -55,25 +96,41 @@ export default function ({ getPageObjects }: FtrProviderContext) {
},
];
const dataOldCnvm = [
{
...vulnerabilitiesLatestMock[0],
'@timestamp': (Date.now() - daysToMillisecond(RETENTION + 1)).toString(),
},
];
const dataWithinRetentionCnvm = [
{
...vulnerabilitiesLatestMock[0],
'@timestamp': (Date.now() - daysToMillisecond(RETENTION - 1)).toString(),
},
];
describe('Old Data', function () {
this.tags(['cloud_security_posture_findings']);
let findings: typeof pageObjects.findings;
let latestFindingsTable: typeof findings.latestFindingsTable;
let latestVulnerabilitiesTable: typeof findings.latestVulnerabilitiesTable;
before(async () => {
findings = pageObjects.findings;
latestFindingsTable = findings.latestFindingsTable;
latestVulnerabilitiesTable = findings.latestVulnerabilitiesTable;
// Before we start any test we must wait for cloud_security_posture plugin to complete its initialization
await findings.waitForPluginInitialized();
});
after(async () => {
afterEach(async () => {
await findings.index.remove();
await findings.vulnerabilitiesIndex.remove();
});
describe('Findings page with old data', () => {
it('returns no Findings KSPM', async () => {
// Prepare mocked findings
await findings.index.remove();
await findings.index.add(dataOldKspm);
await findings.navigateToLatestFindingsPage();
@ -81,14 +138,50 @@ export default function ({ getPageObjects }: FtrProviderContext) {
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns no Findings CSPM', async () => {
// Prepare mocked findings
await findings.index.remove();
await findings.index.add(dataOldCspm);
await findings.navigateToLatestFindingsPage();
await pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns no Findings CNVM', async () => {
await findings.vulnerabilitiesIndex.add(dataOldCnvm);
await findings.navigateToLatestVulnerabilitiesPage();
await pageObjects.header.waitUntilLoadingHasFinished();
expect(await findings.isLatestFindingsTableThere()).to.be(false);
});
it('returns data grid with only data within retention KSPM', async () => {
await findings.index.add([...dataOldKspm, ...dataWithinRetentionKspm]);
await findings.navigateToLatestFindingsPage();
await retry.waitFor(
'Findings table to be loaded',
async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionKspm.length
);
await pageObjects.header.waitUntilLoadingHasFinished();
});
it('returns data grid with only data within retention CSPM', async () => {
await findings.index.add([...dataOldCspm, ...dataWithinRetentionCspm]);
await findings.navigateToLatestFindingsPage();
await retry.waitFor(
'Findings table to be loaded',
async () => (await latestFindingsTable.getRowsCount()) === dataWithinRetentionCspm.length
);
await pageObjects.header.waitUntilLoadingHasFinished();
});
it('returns data grid with only data within retention CSPM', async () => {
await findings.vulnerabilitiesIndex.add([...dataOldCnvm, ...dataWithinRetentionCnvm]);
await findings.navigateToLatestVulnerabilitiesPage();
await retry.waitFor(
'Findings table to be loaded',
async () =>
(await latestVulnerabilitiesTable.getRowsCount()) === dataWithinRetentionCnvm.length
);
await pageObjects.header.waitUntilLoadingHasFinished();
});
});
});
}