github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.x] [Cloud Security] [CDR] Handle grouping fields with missing mapping (#195702) (#195792)

Browse source 
# Backport

This will backport the following commits from `main` to `8.x`:
- [[Cloud Security] [CDR] Handle grouping fields with missing mapping
(#195702)](https://github.com/elastic/kibana/pull/195702)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Paulo
Silva","email":"paulo.henrique@elastic.co"},"sourceCommit":{"committedDate":"2024-10-10T14:52:49Z","message":"[Cloud
Security] [CDR] Handle grouping fields with missing mapping
(#195702)\n\n## Summary\r\n\r\nThis PR fixes
https://github.com/elastic/security-team/issues/10632 by\r\nadding
runtime mapping support for fields that are missing in mapping,\r\nthis
is useful when querying a DataView that points to multiple
indices\r\nwhere the mapping is not guaranteed to exist as it's the case
with CDR\r\nthat adds supports to Third Party data.\r\n\r\nAlso added
runtime mapping to sorted fields, as it's not guaranteed that\r\nall
fields shown on the table have mapped
fields.","sha":"e53e54550f9ab9ce2db83ec56a5c704a96f37355","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Cloud
Security","v8.16.0","backport:version"],"title":"[Cloud Security] [CDR]
Handle grouping fields with missing
mapping","number":195702,"url":"https://github.com/elastic/kibana/pull/195702","mergeCommit":{"message":"[Cloud
Security] [CDR] Handle grouping fields with missing mapping
(#195702)\n\n## Summary\r\n\r\nThis PR fixes
https://github.com/elastic/security-team/issues/10632 by\r\nadding
runtime mapping support for fields that are missing in mapping,\r\nthis
is useful when querying a DataView that points to multiple
indices\r\nwhere the mapping is not guaranteed to exist as it's the case
with CDR\r\nthat adds supports to Third Party data.\r\n\r\nAlso added
runtime mapping to sorted fields, as it's not guaranteed that\r\nall
fields shown on the table have mapped
fields.","sha":"e53e54550f9ab9ce2db83ec56a5c704a96f37355"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/195702","number":195702,"mergeCommit":{"message":"[Cloud
Security] [CDR] Handle grouping fields with missing mapping
(#195702)\n\n## Summary\r\n\r\nThis PR fixes
https://github.com/elastic/security-team/issues/10632 by\r\nadding
runtime mapping support for fields that are missing in mapping,\r\nthis
is useful when querying a DataView that points to multiple
indices\r\nwhere the mapping is not guaranteed to exist as it's the case
with CDR\r\nthat adds supports to Third Party data.\r\n\r\nAlso added
runtime mapping to sorted fields, as it's not guaranteed that\r\nall
fields shown on the table have mapped
fields.","sha":"e53e54550f9ab9ce2db83ec56a5c704a96f37355"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Paulo Silva <paulo.henrique@elastic.co>
This commit is contained in:
Kibana Machine 2024-10-11 05:21:12 +11:00 • committed by GitHub
parent c7fa004d30
commit e2bed0f502
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 162 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

16
x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings.ts
View file

@ -21,6 +21,7 @@ import type { CspFinding } from '@kbn/cloud-security-posture-common';
import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
import type { FindingsBaseEsQuery } from '@kbn/cloud-security-posture';
import { useGetCspBenchmarkRulesStatesApi } from '@kbn/cloud-security-posture/src/hooks/use_get_benchmark_rules_state_api';
import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
import { useKibana } from '../../../common/hooks/use_kibana';
import { getAggregationCount, getFindingsCountAggQuery } from '../utils/utils';
@ -39,6 +40,20 @@ interface FindingsAggs {
count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
}
const getRuntimeMappingsFromSort = (sort: string[][]) => {
return sort.reduce((acc, [field]) => {
// TODO: Add proper type for all fields available in the field selector
const type: RuntimePrimitiveTypes = field === '@timestamp' ? 'date' : 'keyword';
return {
...acc,
[field]: {
type,
},
};
}, {});
};
export const getFindingsQuery = (
{ query, sort }: UseFindingsOptions,
rulesStates: CspBenchmarkRulesStates,
@ -49,6 +64,7 @@ export const getFindingsQuery = (
return {
index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
sort: getMultiFieldsSort(sort),
runtime_mappings: getRuntimeMappingsFromSort(sort),
size: MAX_FINDINGS_TO_LOAD,
aggs: getFindingsCountAggQuery(),
ignore_unavailable: true,

72
x-pack/plugins/cloud_security_posture/public/pages/configurations/latest_findings/use_latest_findings_grouping.tsx
View file

@ -114,6 +114,72 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
return aggMetrics;
};
/**
* Get runtime mappings for the given group field
* Some fields require additional runtime mappings to aggregate additional information
* Fallback to keyword type to support custom fields grouping
*/
const getRuntimeMappingsByGroupField = (
field: string
): Record<string, { type: 'keyword' }> | undefined => {
switch (field) {
case FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME:
return {
[FINDINGS_GROUPING_OPTIONS.RESOURCE_NAME]: {
type: 'keyword',
},
'resource.id': {
type: 'keyword',
},
'resource.sub_type': {
type: 'keyword',
},
'resource.type': {
type: 'keyword',
},
};
case FINDINGS_GROUPING_OPTIONS.RULE_NAME:
return {
[FINDINGS_GROUPING_OPTIONS.RULE_NAME]: {
type: 'keyword',
},
'rule.benchmark.version': {
type: 'keyword',
},
};
case FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
return {
[FINDINGS_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
type: 'keyword',
},
'rule.benchmark.name': {
type: 'keyword',
},
'rule.benchmark.id': {
type: 'keyword',
},
};
case FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME:
return {
[FINDINGS_GROUPING_OPTIONS.ORCHESTRATOR_CLUSTER_NAME]: {
type: 'keyword',
},
'rule.benchmark.name': {
type: 'keyword',
},
'rule.benchmark.id': {
type: 'keyword',
},
};
default:
return {
[field]: {
type: 'keyword',
},
};
}
};
/**
* Type Guard for checking if the given source is a FindingsRootGroupingAggregation
*/
@ -189,6 +255,12 @@ export const useLatestFindingsGrouping = ({
size: pageSize,
sort: [{ groupByField: { order: 'desc' } }, { complianceScore: { order: 'asc' } }],
statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
runtimeMappings: {
...getRuntimeMappingsByGroupField(currentSelectedGroup),
'result.evaluation': {
type: 'keyword',
},
},
rootAggregations: [
{
failedFindings: {

21
x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities.tsx
View file

@ -23,6 +23,7 @@ import {
} from '@kbn/cloud-security-posture-common';
import { FindingsBaseEsQuery, showErrorToast } from '@kbn/cloud-security-posture';
import type { CspVulnerabilityFinding } from '@kbn/cloud-security-posture-common/schema/vulnerabilities/latest';
import type { RuntimePrimitiveTypes } from '@kbn/data-views-plugin/common';
import { VULNERABILITY_FIELDS } from '../../../common/constants';
import { useKibana } from '../../../common/hooks/use_kibana';
import { getCaseInsensitiveSortScript } from '../utils/custom_sort_script';
@ -52,6 +53,25 @@ const getMultiFieldsSort = (sort: string[][]) => {
});
};
const getRuntimeMappingsFromSort = (sort: string[][]) => {
return sort.reduce((acc, [field]) => {
// TODO: Add proper type for all fields available in the field selector
const type: RuntimePrimitiveTypes =
field === VULNERABILITY_FIELDS.SCORE_BASE
? 'double'
: field === '@timestamp'
? 'date'
: 'keyword';
return {
...acc,
[field]: {
type,
},
};
}, {});
};
export const getVulnerabilitiesQuery = (
{ query, sort }: VulnerabilitiesQuery,
pageParam: number
@ -59,6 +79,7 @@ export const getVulnerabilitiesQuery = (
index: CDR_VULNERABILITIES_INDEX_PATTERN,
ignore_unavailable: true,
sort: getMultiFieldsSort(sort),
runtime_mappings: getRuntimeMappingsFromSort(sort),
size: MAX_FINDINGS_TO_LOAD,
query: {
...query,

46
x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/hooks/use_latest_vulnerabilities_grouping.tsx
View file

@ -94,6 +94,51 @@ const getAggregationsByGroupField = (field: string): NamedAggregation[] => {
return aggMetrics;
};
/**
* Get runtime mappings for the given group field
* Some fields require additional runtime mappings to aggregate additional information
* Fallback to keyword type to support custom fields grouping
*/
const getRuntimeMappingsByGroupField = (
field: string
): Record<string, { type: 'keyword' }> | undefined => {
switch (field) {
case VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME:
return {
[VULNERABILITY_GROUPING_OPTIONS.CLOUD_ACCOUNT_NAME]: {
type: 'keyword',
},
[VULNERABILITY_FIELDS.CLOUD_PROVIDER]: {
type: 'keyword',
},
};
case VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME:
return {
[VULNERABILITY_GROUPING_OPTIONS.RESOURCE_NAME]: {
type: 'keyword',
},
[VULNERABILITY_FIELDS.RESOURCE_ID]: {
type: 'keyword',
},
};
case VULNERABILITY_GROUPING_OPTIONS.CVE:
return {
[VULNERABILITY_GROUPING_OPTIONS.CVE]: {
type: 'keyword',
},
[VULNERABILITY_FIELDS.DESCRIPTION]: {
type: 'keyword',
},
};
default:
return {
[field]: {
type: 'keyword',
},
};
}
};
/**
* Type Guard for checking if the given source is a VulnerabilitiesRootGroupingAggregation
*/
@ -163,6 +208,7 @@ export const useLatestVulnerabilitiesGrouping = ({
size: pageSize,
sort: [{ groupByField: { order: 'desc' } }],
statsAggregations: getAggregationsByGroupField(currentSelectedGroup),
runtimeMappings: getRuntimeMappingsByGroupField(currentSelectedGroup),
});
const { data, isFetching } = useGroupedVulnerabilities({

8
x-pack/plugins/cloud_security_posture/public/pages/vulnerabilities/utils/custom_sort_script.ts
View file

@ -14,7 +14,13 @@ export const getCaseInsensitiveSortScript = (field: string, direction: string) =
type: 'string',
order: direction,
script: {
source: `doc["${field}"].value.toLowerCase()`,
source: `
if (doc.containsKey('${field}') && !doc['${field}'].empty) {
return doc['${field}'].value.toLowerCase();
} else {
return "";
}
`,
lang: 'painless',
},
},