Merge branch 'main' into reportings/soft-disable

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/compliance_dashboard.ts

@@ -5,16 +5,10 @@
  * 2.0.
  */
 
-import type { ElasticsearchClient } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';
-import type {
-  AggregationsMultiBucketAggregateBase as Aggregation,
-  AggregationsTopHitsAggregate,
-  QueryDslQueryContainer,
-  SearchRequest,
-} from '@elastic/elasticsearch/lib/api/types';
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import type { ComplianceDashboardData } from '../../../common/types';
-import { CSP_KUBEBEAT_INDEX_PATTERN, STATS_ROUTE_PATH } from '../../../common/constants';
+import { LATEST_FINDINGS_INDEX_PATTERN, STATS_ROUTE_PATH } from '../../../common/constants';
 import { CspAppContext } from '../../plugin';
 import { getResourcesTypes } from './get_resources_types';
 import { ClusterWithoutTrend, getClusters } from './get_clusters';
@@ -22,59 +16,11 @@ import { getStats } from './get_stats';
 import { CspRouter } from '../../types';
 import { getTrends, Trends } from './get_trends';
 
-export interface ClusterBucket {
-  ordered_top_hits: AggregationsTopHitsAggregate;
-}
-
-interface ClustersQueryResult {
-  aggs_by_cluster_id: Aggregation<ClusterBucket>;
-}
-
 export interface KeyDocCount<TKey = string> {
   key: TKey;
   doc_count: number;
 }
 
-export const getLatestFindingQuery = (): SearchRequest => ({
-  index: CSP_KUBEBEAT_INDEX_PATTERN,
-  size: 0,
-  query: {
-    match_all: {},
-  },
-  aggs: {
-    aggs_by_cluster_id: {
-      terms: { field: 'cluster_id.keyword' },
-      aggs: {
-        ordered_top_hits: {
-          top_hits: {
-            size: 1,
-            sort: {
-              '@timestamp': {
-                order: 'desc',
-              },
-            },
-          },
-        },
-      },
-    },
-  },
-});
-
-const getLatestCyclesIds = async (esClient: ElasticsearchClient): Promise<string[]> => {
-  const queryResult = await esClient.search<unknown, ClustersQueryResult>(getLatestFindingQuery(), {
-    meta: true,
-  });
-
-  const clusters = queryResult.body.aggregations?.aggs_by_cluster_id.buckets;
-  if (!Array.isArray(clusters)) throw new Error('missing aggs by cluster id');
-
-  return clusters.map((c) => {
-    const topHit = c.ordered_top_hits.hits.hits[0];
-    if (!topHit) throw new Error('missing cluster latest hit');
-    return topHit._source.cycle_id;
-  });
-};
-
 const getClustersTrends = (clustersWithoutTrends: ClusterWithoutTrend[], trends: Trends) =>
   clustersWithoutTrends.map((cluster) => ({
     ...cluster,
@@ -87,7 +33,6 @@ const getClustersTrends = (clustersWithoutTrends: ClusterWithoutTrend[], trends:
 const getSummaryTrend = (trends: Trends) =>
   trends.map(({ timestamp, summary }) => ({ timestamp, ...summary }));
 
-// TODO: Utilize ES "Point in Time" feature https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html
 export const defineGetComplianceDashboardRoute = (
   router: CspRouter,
   cspContext: CspAppContext
@@ -100,22 +45,29 @@ export const defineGetComplianceDashboardRoute = (
     async (context, _, response) => {
       try {
         const esClient = context.core.elasticsearch.client.asCurrentUser;
-        const latestCyclesIds = await getLatestCyclesIds(esClient);
+
+        const { id: pitId } = await esClient.openPointInTime({
+          index: LATEST_FINDINGS_INDEX_PATTERN,
+          keep_alive: '30s',
+        });
+
         const query: QueryDslQueryContainer = {
-          bool: {
-            should: latestCyclesIds.map((id) => ({
-              match: { 'cycle_id.keyword': { query: id } },
-            })),
-          },
+          match_all: {},
         };
 
         const [stats, resourcesTypes, clustersWithoutTrends, trends] = await Promise.all([
-          getStats(esClient, query),
-          getResourcesTypes(esClient, query),
-          getClusters(esClient, query),
+          getStats(esClient, query, pitId),
+          getResourcesTypes(esClient, query, pitId),
+          getClusters(esClient, query, pitId),
           getTrends(esClient),
         ]);
 
+        // Try closing the PIT, if it fails we can safely ignore the error since it closes itself after the keep alive
+        //   ends. Not waiting on the promise returned from the `closePointInTime` call to avoid delaying the request
+        esClient.closePointInTime({ id: pitId }).catch((err) => {
+          cspContext.logger.warn(`Could not close PIT for stats endpoint: ${err}`);
+        });
+
         const clusters = getClustersTrends(clustersWithoutTrends, trends);
         const trend = getSummaryTrend(trends);
 
@@ -131,6 +83,7 @@ export const defineGetComplianceDashboardRoute = (
         });
       } catch (err) {
         const error = transformError(err);
+        cspContext.logger.error(`Error while fetching CSP stats: ${err}`);
 
         return response.customError({
           body: { message: error.message },

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_clusters.ts

@@ -14,7 +14,6 @@ import type {
 import { Cluster } from '../../../common/types';
 import { getResourceTypeFromAggs, resourceTypeAggQuery } from './get_resources_types';
 import type { ResourceTypeQueryResult } from './get_resources_types';
-import { CSP_KUBEBEAT_INDEX_PATTERN } from '../../../common/constants';
 import { findingsEvaluationAggsQuery, getStatsFromFindingsEvaluationsAggs } from './get_stats';
 import { KeyDocCount } from './compliance_dashboard';
 
@@ -37,8 +36,7 @@ interface ClustersQueryResult {
 
 export type ClusterWithoutTrend = Omit<Cluster, 'trend'>;
 
-export const getClustersQuery = (query: QueryDslQueryContainer): SearchRequest => ({
-  index: CSP_KUBEBEAT_INDEX_PATTERN,
+export const getClustersQuery = (query: QueryDslQueryContainer, pitId: string): SearchRequest => ({
   size: 0,
   query,
   aggs: {
@@ -66,6 +64,9 @@ export const getClustersQuery = (query: QueryDslQueryContainer): SearchRequest =
       },
     },
   },
+  pit: {
+    id: pitId,
+  },
 });
 
 export const getClustersFromAggs = (clusters: ClusterBucket[]): ClusterWithoutTrend[] =>
@@ -102,13 +103,14 @@ export const getClustersFromAggs = (clusters: ClusterBucket[]): ClusterWithoutTr
 
 export const getClusters = async (
   esClient: ElasticsearchClient,
-  query: QueryDslQueryContainer
+  query: QueryDslQueryContainer,
+  pitId: string
 ): Promise<ClusterWithoutTrend[]> => {
-  const queryResult = await esClient.search<unknown, ClustersQueryResult>(getClustersQuery(query), {
-    meta: true,
-  });
+  const queryResult = await esClient.search<unknown, ClustersQueryResult>(
+    getClustersQuery(query, pitId)
+  );
 
-  const clusters = queryResult.body.aggregations?.aggs_by_cluster_id.buckets;
+  const clusters = queryResult.aggregations?.aggs_by_cluster_id.buckets;
   if (!Array.isArray(clusters)) throw new Error('missing aggs by cluster id');
 
   return getClustersFromAggs(clusters);

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_resources_types.ts

@@ -13,7 +13,6 @@ import type {
 } from '@elastic/elasticsearch/lib/api/types';
 import type { ComplianceDashboardData } from '../../../common/types';
 import { KeyDocCount } from './compliance_dashboard';
-import { CSP_KUBEBEAT_INDEX_PATTERN } from '../../../common/constants';
 
 export interface ResourceTypeQueryResult {
   aggs_by_resource_type: Aggregation<ResourceTypeBucket>;
@@ -44,11 +43,13 @@ export const resourceTypeAggQuery = {
   },
 };
 
-export const getRisksEsQuery = (query: QueryDslQueryContainer): SearchRequest => ({
-  index: CSP_KUBEBEAT_INDEX_PATTERN,
+export const getRisksEsQuery = (query: QueryDslQueryContainer, pitId: string): SearchRequest => ({
   size: 0,
   query,
   aggs: resourceTypeAggQuery,
+  pit: {
+    id: pitId,
+  },
 });
 
 export const getResourceTypeFromAggs = (
@@ -63,14 +64,14 @@ export const getResourceTypeFromAggs = (
 
 export const getResourcesTypes = async (
   esClient: ElasticsearchClient,
-  query: QueryDslQueryContainer
+  query: QueryDslQueryContainer,
+  pitId: string
 ): Promise<ComplianceDashboardData['resourcesTypes']> => {
   const resourceTypesQueryResult = await esClient.search<unknown, ResourceTypeQueryResult>(
-    getRisksEsQuery(query),
-    { meta: true }
+    getRisksEsQuery(query, pitId)
   );
 
-  const resourceTypes = resourceTypesQueryResult.body.aggregations?.aggs_by_resource_type.buckets;
+  const resourceTypes = resourceTypesQueryResult.aggregations?.aggs_by_resource_type.buckets;
   if (!Array.isArray(resourceTypes)) throw new Error('missing resources types buckets');
 
   return getResourceTypeFromAggs(resourceTypes);

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_stats.ts

@@ -7,7 +7,6 @@
 
 import { ElasticsearchClient } from '@kbn/core/server';
 import type { QueryDslQueryContainer, SearchRequest } from '@elastic/elasticsearch/lib/api/types';
-import { CSP_KUBEBEAT_INDEX_PATTERN } from '../../../common/constants';
 import type { ComplianceDashboardData, Score } from '../../../common/types';
 
 /**
@@ -36,10 +35,16 @@ export const findingsEvaluationAggsQuery = {
   },
 };
 
-export const getEvaluationsQuery = (query: QueryDslQueryContainer): SearchRequest => ({
-  index: CSP_KUBEBEAT_INDEX_PATTERN,
+export const getEvaluationsQuery = (
+  query: QueryDslQueryContainer,
+  pitId: string
+): SearchRequest => ({
   query,
+  size: 0,
   aggs: findingsEvaluationAggsQuery,
+  pit: {
+    id: pitId,
+  },
 });
 
 export const getStatsFromFindingsEvaluationsAggs = (
@@ -61,13 +66,14 @@ export const getStatsFromFindingsEvaluationsAggs = (
 
 export const getStats = async (
   esClient: ElasticsearchClient,
-  query: QueryDslQueryContainer
+  query: QueryDslQueryContainer,
+  pitId: string
 ): Promise<ComplianceDashboardData['stats']> => {
   const evaluationsQueryResult = await esClient.search<unknown, FindingsEvaluationsQueryResult>(
-    getEvaluationsQuery(query),
-    { meta: true }
+    getEvaluationsQuery(query, pitId)
   );
-  const findingsEvaluations = evaluationsQueryResult.body.aggregations;
+
+  const findingsEvaluations = evaluationsQueryResult.aggregations;
   if (!findingsEvaluations) throw new Error('missing findings evaluations');
 
   return getStatsFromFindingsEvaluationsAggs(findingsEvaluations);

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_trends.ts

@@ -25,7 +25,7 @@ export interface ScoreTrendDoc {
   >;
 }
 
-export const getTrendsAggsQuery = () => ({
+export const getTrendsQuery = () => ({
   index: BENCHMARK_SCORE_INDEX_PATTERN,
   size: 5,
   sort: '@timestamp:desc',
@@ -60,7 +60,7 @@ export const getTrendsFromQueryResult = (scoreTrendDocs: ScoreTrendDoc[]): Trend
   }));
 
 export const getTrends = async (esClient: ElasticsearchClient): Promise<Trends> => {
-  const trendsQueryResult = await esClient.search<ScoreTrendDoc>(getTrendsAggsQuery());
+  const trendsQueryResult = await esClient.search<ScoreTrendDoc>(getTrendsQuery());
 
   if (!trendsQueryResult.hits.hits) throw new Error('missing trend results from score index');