[Cloud Posture] Csp dashboard trendline (#129481)

x-pack/plugins/cloud_security_posture/common/constants.ts

@@ -27,7 +27,6 @@ export const RULE_FAILED = `failed`;
 // activated via a simple code change in a single location.
 export const INTERNAL_FEATURE_FLAGS = {
   showBenchmarks: false,
-  showTrendLineMock: false,
   showManageRulesMock: false,
   showRisksMock: false,
 } as const;

x-pack/plugins/cloud_security_posture/common/types.ts

@@ -25,6 +25,10 @@ export interface ResourceType extends FindingsEvaluation {
   name: string;
 }
 
+export interface PostureTrend extends Stats {
+  timestamp: string;
+}
+
 export interface Cluster {
   meta: {
     clusterId: string;
@@ -33,12 +37,14 @@ export interface Cluster {
   };
   stats: Stats;
   resourcesTypes: ResourceType[];
+  trend: PostureTrend[];
 }
 
 export interface ComplianceDashboardData {
   stats: Stats;
   resourcesTypes: ResourceType[];
   clusters: Cluster[];
+  trend: PostureTrend[];
 }
 
 export interface Benchmark {

x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/compliance_charts/cloud_posture_score_chart.tsx

@@ -19,23 +19,27 @@ import {
   timeFormatter,
 } from '@elastic/charts';
 import { EuiFlexGroup, EuiText, EuiHorizontalRule, EuiFlexItem } from '@elastic/eui';
+import { FormattedDate, FormattedTime } from '@kbn/i18n-react';
+import moment from 'moment';
 import { statusColors } from '../../../common/constants';
-import type { Stats } from '../../../../common/types';
+import type { PostureTrend, Stats } from '../../../../common/types';
 import * as TEXT from '../translations';
 import { CompactFormattedNumber } from '../../../components/compact_formatted_number';
-import { INTERNAL_FEATURE_FLAGS } from '../../../../common/constants';
 
 interface CloudPostureScoreChartProps {
+  trend: PostureTrend[];
   data: Stats;
   id: string;
   partitionOnElementClick: (elements: PartitionElementEvent[]) => void;
 }
 
+const getPostureScorePercentage = (postureScore: number): string => `${Math.round(postureScore)}%`;
+
 const ScoreChart = ({
   data: { totalPassed, totalFailed },
   id,
   partitionOnElementClick,
-}: CloudPostureScoreChartProps) => {
+}: Omit<CloudPostureScoreChartProps, 'trend'>) => {
   const data = [
     { label: TEXT.PASSED, value: totalPassed },
     { label: TEXT.FAILED, value: totalFailed },
@@ -79,7 +83,7 @@ const PercentageInfo = ({
   totalPassed,
   totalFindings,
 }: CloudPostureScoreChartProps['data']) => {
-  const percentage = `${Math.round(postureScore)}%`;
+  const percentage = getPostureScorePercentage(postureScore);
 
   return (
     <EuiFlexGroup direction="column" justifyContent="center">
@@ -94,32 +98,53 @@ const PercentageInfo = ({
   );
 };
 
-const mockData = [
-  [0, 9],
-  [1000, 70],
-  [2000, 40],
-  [4000, 90],
-  [5000, 53],
-];
+const convertTrendToEpochTime = (trend: PostureTrend) => ({
+  ...trend,
+  timestamp: moment(trend.timestamp).valueOf(),
+});
 
-const ComplianceTrendChart = () => (
-  <Chart>
-    <Settings showLegend={false} legendPosition="right" />
-    <AreaSeries
-      id="compliance_score"
-      // TODO: no api yet
-      data={INTERNAL_FEATURE_FLAGS.showTrendLineMock ? mockData : []}
-      xScaleType="time"
-      xAccessor={0}
-      yAccessors={[1]}
-    />
-    <Axis id="bottom-axis" position="bottom" tickFormat={timeFormatter(niceTimeFormatByDay(1))} />
-    <Axis ticks={3} id="left-axis" position="left" showGridLines domain={{ min: 0, max: 100 }} />
-  </Chart>
-);
+const ComplianceTrendChart = ({ trend }: { trend: PostureTrend[] }) => {
+  const epochTimeTrend = trend.map(convertTrendToEpochTime);
+
+  return (
+    <Chart>
+      <Settings
+        showLegend={false}
+        legendPosition="right"
+        tooltip={{
+          headerFormatter: ({ value }) => (
+            <>
+              <FormattedDate value={value} month="short" day="numeric" />
+              {', '}
+              <FormattedTime value={value} />
+            </>
+          ),
+        }}
+      />
+      <AreaSeries
+        // EuiChart is using this id in the tooltip label
+        id="Posture Score"
+        data={epochTimeTrend}
+        xScaleType="time"
+        xAccessor={'timestamp'}
+        yAccessors={['postureScore']}
+      />
+      <Axis id="bottom-axis" position="bottom" tickFormat={timeFormatter(niceTimeFormatByDay(2))} />
+      <Axis
+        ticks={3}
+        id="left-axis"
+        position="left"
+        showGridLines
+        domain={{ min: 0, max: 100 }}
+        tickFormat={(rawScore) => getPostureScorePercentage(rawScore)}
+      />
+    </Chart>
+  );
+};
 
 export const CloudPostureScoreChart = ({
   data,
+  trend,
   id,
   partitionOnElementClick,
 }: CloudPostureScoreChartProps) => (
@@ -136,7 +161,7 @@ export const CloudPostureScoreChart = ({
     </EuiFlexItem>
     <EuiHorizontalRule margin="xs" />
     <EuiFlexItem grow={6}>
-      <ComplianceTrendChart />
+      <ComplianceTrendChart trend={trend} />
     </EuiFlexItem>
   </EuiFlexGroup>
 );

x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/benchmarks_section.tsx

@@ -100,6 +100,7 @@ export const BenchmarksSection = ({
                     <CloudPostureScoreChart
                       id={`${cluster.meta.clusterId}_score_chart`}
                       data={cluster.stats}
+                      trend={cluster.trend}
                       partitionOnElementClick={(elements) =>
                         handleElementClick(cluster.meta.clusterId, elements)
                       }

x-pack/plugins/cloud_security_posture/public/pages/compliance_dashboard/dashboard_sections/summary_section.tsx

@@ -50,6 +50,7 @@ export const SummarySection = ({ complianceData }: { complianceData: ComplianceD
           <CloudPostureScoreChart
             id="cloud_posture_score_chart"
             data={complianceData.stats}
+            trend={complianceData.trend}
             partitionOnElementClick={handleElementClick}
           />
         </ChartPanel>

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/compliance_dashboard.ts

@@ -17,9 +17,10 @@ import type { ComplianceDashboardData } from '../../../common/types';
 import { CSP_KUBEBEAT_INDEX_PATTERN, STATS_ROUTE_PATH } from '../../../common/constants';
 import { CspAppContext } from '../../plugin';
 import { getResourcesTypes } from './get_resources_types';
-import { getClusters } from './get_clusters';
+import { ClusterWithoutTrend, getClusters } from './get_clusters';
 import { getStats } from './get_stats';
 import { CspRouter } from '../../types';
+import { getTrends, Trends } from './get_trends';
 
 export interface ClusterBucket {
   ordered_top_hits: AggregationsTopHitsAggregate;
@@ -74,6 +75,18 @@ const getLatestCyclesIds = async (esClient: ElasticsearchClient): Promise<string
   });
 };
 
+const getClustersTrends = (clustersWithoutTrends: ClusterWithoutTrend[], trends: Trends) =>
+  clustersWithoutTrends.map((cluster) => ({
+    ...cluster,
+    trend: trends.map(({ timestamp, clusters: clustersTrendData }) => ({
+      timestamp,
+      ...clustersTrendData[cluster.meta.clusterId],
+    })),
+  }));
+
+const getSummaryTrend = (trends: Trends) =>
+  trends.map(({ timestamp, summary }) => ({ timestamp, ...summary }));
+
 // TODO: Utilize ES "Point in Time" feature https://www.elastic.co/guide/en/elasticsearch/reference/current/point-in-time-api.html
 export const defineGetComplianceDashboardRoute = (
   router: CspRouter,
@@ -96,16 +109,21 @@ export const defineGetComplianceDashboardRoute = (
           },
         };
 
-        const [stats, resourcesTypes, clusters] = await Promise.all([
+        const [stats, resourcesTypes, clustersWithoutTrends, trends] = await Promise.all([
           getStats(esClient, query),
           getResourcesTypes(esClient, query),
           getClusters(esClient, query),
+          getTrends(esClient),
         ]);
 
+        const clusters = getClustersTrends(clustersWithoutTrends, trends);
+        const trend = getSummaryTrend(trends);
+
         const body: ComplianceDashboardData = {
           stats,
           resourcesTypes,
           clusters,
+          trend,
         };
 
         return response.ok({

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_clusters.ts

@@ -11,7 +11,7 @@ import type {
   QueryDslQueryContainer,
   SearchRequest,
 } from '@elastic/elasticsearch/lib/api/types';
-import { ComplianceDashboardData } from '../../../common/types';
+import { Cluster } from '../../../common/types';
 import { getResourceTypeFromAggs, resourceTypeAggQuery } from './get_resources_types';
 import type { ResourceTypeQueryResult } from './get_resources_types';
 import { CSP_KUBEBEAT_INDEX_PATTERN } from '../../../common/constants';
@@ -35,6 +35,8 @@ interface ClustersQueryResult {
   aggs_by_cluster_id: Aggregation<ClusterBucket>;
 }
 
+export type ClusterWithoutTrend = Omit<Cluster, 'trend'>;
+
 export const getClustersQuery = (query: QueryDslQueryContainer): SearchRequest => ({
   index: CSP_KUBEBEAT_INDEX_PATTERN,
   size: 0,
@@ -66,9 +68,7 @@ export const getClustersQuery = (query: QueryDslQueryContainer): SearchRequest =
   },
 });
 
-export const getClustersFromAggs = (
-  clusters: ClusterBucket[]
-): ComplianceDashboardData['clusters'] =>
+export const getClustersFromAggs = (clusters: ClusterBucket[]): ClusterWithoutTrend[] =>
   clusters.map((cluster) => {
     // get cluster's meta data
     const benchmarks = cluster.benchmarks.buckets;
@@ -103,7 +103,7 @@ export const getClustersFromAggs = (
 export const getClusters = async (
   esClient: ElasticsearchClient,
   query: QueryDslQueryContainer
-): Promise<ComplianceDashboardData['clusters']> => {
+): Promise<ClusterWithoutTrend[]> => {
   const queryResult = await esClient.search<unknown, ClustersQueryResult>(getClustersQuery(query), {
     meta: true,
   });

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_trends.test.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getTrendsFromQueryResult, ScoreTrendDoc } from './get_trends';
+
+const trendDocs: ScoreTrendDoc[] = [
+  {
+    '@timestamp': '2022-04-06T15:30:00Z',
+    total_findings: 20,
+    passed_findings: 5,
+    failed_findings: 15,
+    score_by_cluster_id: {
+      first_cluster_id: {
+        total_findings: 20,
+        passed_findings: 5,
+        failed_findings: 15,
+      },
+    },
+  },
+  {
+    '@timestamp': '2022-04-06T15:00:00Z',
+    total_findings: 40,
+    passed_findings: 25,
+    failed_findings: 15,
+    score_by_cluster_id: {
+      second_cluster_id: {
+        total_findings: 20,
+        passed_findings: 10,
+        failed_findings: 10,
+      },
+      third_cluster_id: {
+        total_findings: 20,
+        passed_findings: 15,
+        failed_findings: 5,
+      },
+    },
+  },
+  {
+    '@timestamp': '2022-04-05T15:30:00Z',
+    total_findings: 30,
+    passed_findings: 25,
+    failed_findings: 5,
+    score_by_cluster_id: {
+      forth_cluster_id: {
+        total_findings: 25,
+        passed_findings: 25,
+        failed_findings: 0,
+      },
+      fifth_cluster_id: {
+        total_findings: 5,
+        passed_findings: 0,
+        failed_findings: 5,
+      },
+    },
+  },
+];
+
+describe('getTrendsFromQueryResult', () => {
+  it('should return value matching Trends type definition, in descending order, and with postureScore', async () => {
+    const trends = getTrendsFromQueryResult(trendDocs);
+    expect(trends).toEqual([
+      {
+        timestamp: '2022-04-06T15:30:00Z',
+        summary: {
+          totalFindings: 20,
+          totalPassed: 5,
+          totalFailed: 15,
+          postureScore: 25.0,
+        },
+        clusters: {
+          first_cluster_id: {
+            totalFindings: 20,
+            totalPassed: 5,
+            totalFailed: 15,
+            postureScore: 25.0,
+          },
+        },
+      },
+      {
+        timestamp: '2022-04-06T15:00:00Z',
+        summary: {
+          totalFindings: 40,
+          totalPassed: 25,
+          totalFailed: 15,
+          postureScore: 62.5,
+        },
+        clusters: {
+          second_cluster_id: {
+            totalFindings: 20,
+            totalPassed: 10,
+            totalFailed: 10,
+            postureScore: 50.0,
+          },
+          third_cluster_id: {
+            totalFindings: 20,
+            totalPassed: 15,
+            totalFailed: 5,
+            postureScore: 75.0,
+          },
+        },
+      },
+      {
+        timestamp: '2022-04-05T15:30:00Z',
+        summary: {
+          totalFindings: 30,
+          totalPassed: 25,
+          totalFailed: 5,
+          postureScore: 83.3,
+        },
+        clusters: {
+          forth_cluster_id: {
+            totalFindings: 25,
+            totalPassed: 25,
+            totalFailed: 0,
+            postureScore: 100.0,
+          },
+          fifth_cluster_id: {
+            totalFindings: 5,
+            totalPassed: 0,
+            totalFailed: 5,
+            postureScore: 0,
+          },
+        },
+      },
+    ]);
+  });
+});

x-pack/plugins/cloud_security_posture/server/routes/compliance_dashboard/get_trends.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ElasticsearchClient } from 'kibana/server';
+import { BENCHMARK_SCORE_INDEX_PATTERN } from '../../../common/constants';
+import { Stats } from '../../../common/types';
+import { calculatePostureScore } from './get_stats';
+
+export interface ScoreTrendDoc {
+  '@timestamp': string;
+  total_findings: number;
+  passed_findings: number;
+  failed_findings: number;
+  score_by_cluster_id: Record<
+    string,
+    {
+      total_findings: number;
+      passed_findings: number;
+      failed_findings: number;
+    }
+  >;
+}
+
+export const getTrendsAggsQuery = () => ({
+  index: BENCHMARK_SCORE_INDEX_PATTERN,
+  size: 5,
+  sort: '@timestamp:desc',
+});
+
+export type Trends = Array<{
+  timestamp: string;
+  summary: Stats;
+  clusters: Record<string, Stats>;
+}>;
+
+export const getTrendsFromQueryResult = (scoreTrendDocs: ScoreTrendDoc[]): Trends =>
+  scoreTrendDocs.map((data) => ({
+    timestamp: data['@timestamp'],
+    summary: {
+      totalFindings: data.total_findings,
+      totalFailed: data.failed_findings,
+      totalPassed: data.passed_findings,
+      postureScore: calculatePostureScore(data.passed_findings, data.failed_findings),
+    },
+    clusters: Object.fromEntries(
+      Object.entries(data.score_by_cluster_id).map(([clusterId, cluster]) => [
+        clusterId,
+        {
+          totalFindings: cluster.total_findings,
+          totalFailed: cluster.failed_findings,
+          totalPassed: cluster.passed_findings,
+          postureScore: calculatePostureScore(cluster.passed_findings, cluster.failed_findings),
+        },
+      ])
+    ),
+  }));
+
+export const getTrends = async (esClient: ElasticsearchClient): Promise<Trends> => {
+  const trendsQueryResult = await esClient.search<ScoreTrendDoc>(getTrendsAggsQuery());
+
+  if (!trendsQueryResult.hits.hits) throw new Error('missing trend results from score index');
+
+  const scoreTrendDocs = trendsQueryResult.hits.hits.map((hit) => {
+    if (!hit._source) throw new Error('missing _source data for one or more of trend results');
+    return hit._source;
+  });
+
+  return getTrendsFromQueryResult(scoreTrendDocs);
+};