[Dataset quality] Implement Summary Panel (#175994)

closes https://github.com/elastic/kibana/issues/170247

## 📝  Summary

This PR introduces a new state machine for controlling the new Dataset
Quality Summary Panel.
As part of this work, we had to introduce a new endpoint to fetch and
calculate the Estimated Data in last 24h.

## 💡For Reviewers

### State Machine

The new state machine introduces 3 parallel states to fetch the values
displayed in the summary panel.
In case of failures in any of them, a retry mechanism is introduced to
try the fetch 1 more time after 5 seconds interval.
If the fetch fails again, we display an error toast notification.

![Screenshot 2024-02-08 at 09 01
34](f1db05f7-fd68-41f5-a950-533fd73aec27)

### New Endpoint

A new endpoint `GET
/internal/dataset_quality/data_streams/estimated_data` has been
introduced to calculate the Estimated Data in last 24h.
The endpoint first retrieves the doc count and total size in bytes for
`logs-*` and uses them to calculate the average size per doc which is
then multiplied by the number of total doc in the last 24h to get an
estimate of data in last 24h.


## ✅  Testing

1) Navigate to /app/observability-logs-explorer/dataset-quality
2) The summary panel is displayed at the top of the table
3) Filterations shouldn't affect the data displayed as the panel is
completely isolated

## 🎥 Demos

- Normal Scenario


c88c3e73-973e-4dd2-babe-63e2c6ae2dda


- Retry On Failures


b952963a-5d67-472a-bd69-9cd9e49b0ed1


- Failing Again After Max Retries


31cb2e4c-cb90-4490-8bcc-ccb11994f9fa

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -427,6 +427,7 @@ packages/kbn-find-used-node-modules @elastic/kibana-operations
 x-pack/plugins/fleet @elastic/fleet
 packages/kbn-flot-charts @elastic/kibana-operations
 x-pack/test/ui_capabilities/common/plugins/foo_plugin @elastic/kibana-security
+packages/kbn-formatters @elastic/obs-ux-logs-team
 src/plugins/ftr_apis @elastic/kibana-core
 packages/kbn-ftr-common-functional-services @elastic/kibana-operations @elastic/appex-qa
 packages/kbn-ftr-common-functional-ui-services @elastic/appex-qa

package.json

@@ -463,6 +463,7 @@
     "@kbn/fleet-plugin": "link:x-pack/plugins/fleet",
     "@kbn/flot-charts": "link:packages/kbn-flot-charts",
     "@kbn/foo-plugin": "link:x-pack/test/ui_capabilities/common/plugins/foo_plugin",
+    "@kbn/formatters": "link:packages/kbn-formatters",
     "@kbn/ftr-apis-plugin": "link:src/plugins/ftr_apis",
     "@kbn/functional-with-es-ssl-cases-test-plugin": "link:x-pack/test/functional_with_es_ssl/plugins/cases",
     "@kbn/gen-ai-streaming-response-example-plugin": "link:x-pack/examples/gen_ai_streaming_response_example",

packages/kbn-formatters/REASDME.md

@@ -0,0 +1,3 @@
+# @kbn/formatters
+
+Utilities for formatting common fields and values.

packages/kbn-formatters/index.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export { formatBytes } from './src/bytes_formatter';

packages/kbn-formatters/jest.config.js

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../..',
+  roots: ['<rootDir>/packages/kbn-formatters'],
+};

packages/kbn-formatters/kibana.jsonc

@@ -0,0 +1,5 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/formatters",
+  "owner": "@elastic/obs-ux-logs-team"
+}

packages/kbn-formatters/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/formatters",
+  "private": true,
+  "version": "1.0.0",
+  "license": "SSPL-1.0 OR Elastic License 2.0"
+}

packages/kbn-formatters/src/bytes_formatter/index.test.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { formatBytes } from '.';
+
+describe('BytesFormatter', () => {
+  it('should format bytes correctly', () => {
+    const result = formatBytes(1000);
+    expect(result).toBe('1000 Bytes');
+  });
+
+  it('should format bytes correctly if 0 is sent', () => {
+    const result = formatBytes(0);
+    expect(result).toBe('0 Bytes');
+  });
+
+  it('should format bytes correctly into KB', () => {
+    const result = formatBytes(10000);
+    expect(result).toBe('10 KB');
+  });
+
+  it('should format bytes correctly into MB', () => {
+    const result = formatBytes(1048576);
+    expect(result).toBe('1 MB');
+  });
+
+  it('should format bytes correctly with decimals', () => {
+    const result = formatBytes(10000, 3);
+    expect(result).toBe('9.766 KB');
+  });
+});

packages/kbn-formatters/src/bytes_formatter/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export const formatBytes = (bytes: number, decimals = 0) => {
+  if (bytes === 0) return '0 Bytes';
+
+  const k = 1024;
+  const dm = decimals < 0 ? 0 : decimals;
+  const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+};

packages/kbn-formatters/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+    ]
+  },
+  "include": [
+    "**/*.ts",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": []
+}

tsconfig.base.json

@@ -848,6 +848,8 @@
       "@kbn/flot-charts/*": ["packages/kbn-flot-charts/*"],
       "@kbn/foo-plugin": ["x-pack/test/ui_capabilities/common/plugins/foo_plugin"],
       "@kbn/foo-plugin/*": ["x-pack/test/ui_capabilities/common/plugins/foo_plugin/*"],
+      "@kbn/formatters": ["packages/kbn-formatters"],
+      "@kbn/formatters/*": ["packages/kbn-formatters/*"],
       "@kbn/ftr-apis-plugin": ["src/plugins/ftr_apis"],
       "@kbn/ftr-apis-plugin/*": ["src/plugins/ftr_apis/*"],
       "@kbn/ftr-common-functional-services": ["packages/kbn-ftr-common-functional-services"],

x-pack/plugins/dataset_quality/common/api_types.ts

@@ -79,3 +79,13 @@ export const getDataStreamsDegradedDocsStatsResponseRt = rt.exact(
 );
 
 export const getDataStreamsDetailsResponseRt = rt.exact(dataStreamDetailsRt);
+
+export const dataStreamsEstimatedDataInBytesRT = rt.type({
+  estimatedDataInBytes: rt.number,
+});
+
+export type DataStreamsEstimatedDataInBytes = rt.TypeOf<typeof dataStreamsEstimatedDataInBytesRT>;
+
+export const getDataStreamsEstimatedDataInBytesResponseRt = rt.exact(
+  dataStreamsEstimatedDataInBytesRT
+);

x-pack/plugins/dataset_quality/common/data_streams_stats/types.ts

@@ -32,5 +32,10 @@ export type GetDataStreamDetailsParams =
 export type GetDataStreamDetailsResponse =
   APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>;
 
+export type GetDataStreamsEstimatedDataInBytesParams =
+  APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/estimated_data`>['params'];
+export type GetDataStreamsEstimatedDataInBytesResponse =
+  APIReturnType<`GET /internal/dataset_quality/data_streams/estimated_data`>;
+
 export type { DataStreamStat } from './data_stream_stat';
 export type { DataStreamDetails } from '../api_types';

x-pack/plugins/dataset_quality/common/translations.ts

@@ -84,6 +84,80 @@ export const flyoutIntegrationNameText = i18n.translate(
   }
 );
 
+/*
+Summary Panel
+*/
+
+export const summaryPanelLast24hText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelLast24hText',
+  {
+    defaultMessage: 'Last 24h',
+  }
+);
+
+export const summaryPanelQualityText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelQualityText',
+  {
+    defaultMessage: 'Datasets Quality',
+  }
+);
+
+export const summaryPanelQualityTooltipText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelQualityTooltipText',
+  {
+    defaultMessage: 'Quality is based on the percentage of degraded docs in a dataset.',
+  }
+);
+
+export const summaryPanelQualityPoorText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelQualityPoorText',
+  {
+    defaultMessage: 'Poor',
+  }
+);
+
+export const summaryPanelQualityDegradedText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelQualityDegradedText',
+  {
+    defaultMessage: 'Degraded',
+  }
+);
+
+export const summaryPanelQualityGoodText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelQualityGoodText',
+  {
+    defaultMessage: 'Good',
+  }
+);
+
+export const summaryPanelDatasetsActivityText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelDatasetsActivityText',
+  {
+    defaultMessage: 'Active Datasets',
+  }
+);
+
+export const summaryPanelDatasetsActivityTooltipText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelDatasetsActivityTooltipText',
+  {
+    defaultMessage: 'The number of datasets with activity in the last 24 hours.',
+  }
+);
+
+export const summaryPanelEstimatedDataText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelEstimatedDataText',
+  {
+    defaultMessage: 'Estimated Data',
+  }
+);
+
+export const summaryPanelEstimatedDataTooltipText = i18n.translate(
+  'xpack.datasetQuality.summaryPanelEstimatedDataTooltipText',
+  {
+    defaultMessage: 'The approximate amount of data stored in the last 24 hours.',
+  }
+);
+
 export const inactiveDatasetsLabel = i18n.translate('xpack.datasetQuality.inactiveDatasetsLabel', {
   defaultMessage: 'Show inactive datasets',
 });

x-pack/plugins/dataset_quality/public/components/common/index.ts

@@ -6,3 +6,4 @@
  */
 
 export * from './integration_icon';
+export * from './types';

x-pack/plugins/dataset_quality/public/components/common/types.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export type QualityIndicators = 'good' | 'poor' | 'degraded';
+export type InfoIndicators = 'success' | 'danger' | 'warning';

x-pack/plugins/dataset_quality/public/components/dataset_quality/dataset_quality.tsx

@@ -12,6 +12,7 @@ import { DatasetQualityContext, DatasetQualityContextValue } from './context';
 import { useKibanaContextForPluginProvider } from '../../utils';
 import { DatasetQualityStartDeps } from '../../types';
 import { DatasetQualityController } from '../../controller';
+import { IDataStreamsStatsClient } from '../../services/data_streams_stats';
 
 export interface DatasetQualityProps {
   controller: DatasetQualityController;
@@ -20,10 +21,16 @@ export interface DatasetQualityProps {
 export interface CreateDatasetQualityArgs {
   core: CoreStart;
   plugins: DatasetQualityStartDeps;
+  dataStreamStatsClient: IDataStreamsStatsClient;
 }
 
-export const createDatasetQuality = ({ core, plugins }: CreateDatasetQualityArgs) => {
+export const createDatasetQuality = ({
+  core,
+  plugins,
+  dataStreamStatsClient,
+}: CreateDatasetQualityArgs) => {
   return ({ controller }: DatasetQualityProps) => {
+    const SummaryPanelProvider = dynamic(() => import('../../hooks/use_summary_panel'));
     const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider(core, plugins);
 
     const datasetQualityProviderValue: DatasetQualityContextValue = useMemo(
@@ -34,17 +41,23 @@ export const createDatasetQuality = ({ core, plugins }: CreateDatasetQualityArgs
     );
 
     return (
-      <DatasetQualityContext.Provider value={datasetQualityProviderValue}>
-        <KibanaContextProviderForPlugin>
-          <DatasetQuality />
-        </KibanaContextProviderForPlugin>
-      </DatasetQualityContext.Provider>
+      <SummaryPanelProvider
+        dataStreamStatsClient={dataStreamStatsClient}
+        toasts={core.notifications.toasts}
+      >
+        <DatasetQualityContext.Provider value={datasetQualityProviderValue}>
+          <KibanaContextProviderForPlugin>
+            <DatasetQuality />
+          </KibanaContextProviderForPlugin>
+        </DatasetQualityContext.Provider>
+      </SummaryPanelProvider>
     );
   };
 };
 
 const Header = dynamic(() => import('./header'));
 const Table = dynamic(() => import('./table'));
+const SummaryPanel = dynamic(() => import('./summary_panel/summary_panel'));
 
 function DatasetQuality() {
   return (
@@ -52,6 +65,9 @@ function DatasetQuality() {
       <EuiFlexItem grow={false}>
         <Header />
       </EuiFlexItem>
+      <EuiFlexItem grow={false}>
+        <SummaryPanel />
+      </EuiFlexItem>
       <EuiFlexItem grow={false}>
         <Table />
       </EuiFlexItem>

x-pack/plugins/dataset_quality/public/components/dataset_quality/summary_panel/datasets_activity.tsx

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { useSummaryPanelContext } from '../../../hooks';
+
+import {
+  summaryPanelDatasetsActivityText,
+  summaryPanelDatasetsActivityTooltipText,
+  tableSummaryOfText,
+} from '../../../../common/translations';
+import { LastDayDataPlaceholder } from './last_day_data_placeholder';
+
+export function DatasetsActivity() {
+  const { datasetsActivity, isDatasetsActivityLoading } = useSummaryPanelContext();
+  const text = `${datasetsActivity.active} ${tableSummaryOfText} ${datasetsActivity.total}`;
+
+  return (
+    <LastDayDataPlaceholder
+      title={summaryPanelDatasetsActivityText}
+      tooltip={summaryPanelDatasetsActivityTooltipText}
+      value={text}
+      isLoading={isDatasetsActivityLoading}
+    />
+  );
+}

x-pack/plugins/dataset_quality/public/components/dataset_quality/summary_panel/datasets_quality_indicators.tsx

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+import { euiThemeVars } from '@kbn/ui-theme';
+import { css } from '@emotion/react';
+
+import {
+  EuiFlexGroup,
+  EuiPanel,
+  EuiFlexItem,
+  EuiTitle,
+  EuiText,
+  EuiHealth,
+  EuiIconTip,
+  EuiSkeletonTitle,
+} from '@elastic/eui';
+import { useSummaryPanelContext } from '../../../hooks';
+import {
+  summaryPanelQualityDegradedText,
+  summaryPanelQualityGoodText,
+  summaryPanelQualityPoorText,
+  summaryPanelQualityText,
+  summaryPanelQualityTooltipText,
+} from '../../../../common/translations';
+import { mapPercentagesToQualityCounts } from '../../quality_indicator';
+import { InfoIndicators } from '../../common';
+
+export function DatasetsQualityIndicators() {
+  const { datasetsQuality, isDatasetsQualityLoading } = useSummaryPanelContext();
+  const qualityCounts = mapPercentagesToQualityCounts(datasetsQuality.percentages);
+
+  return (
+    <EuiPanel hasBorder>
+      <EuiFlexGroup direction="column" gutterSize="s">
+        <EuiFlexGroup alignItems="center" gutterSize="xs" responsive={false}>
+          <EuiFlexItem grow={false}>
+            <EuiText size="s">{summaryPanelQualityText}</EuiText>
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <EuiIconTip content={summaryPanelQualityTooltipText} />
+          </EuiFlexItem>
+        </EuiFlexGroup>
+        <EuiFlexGroup gutterSize="m" alignItems="flexEnd">
+          <QualityIndicator
+            value={qualityCounts.poor}
+            quality="danger"
+            description={summaryPanelQualityPoorText}
+            isLoading={isDatasetsQualityLoading}
+          />
+          <span css={verticalRule} />
+          <QualityIndicator
+            value={qualityCounts.degraded}
+            quality="warning"
+            description={summaryPanelQualityDegradedText}
+            isLoading={isDatasetsQualityLoading}
+          />
+          <span css={verticalRule} />
+          <QualityIndicator
+            value={qualityCounts.good}
+            quality="success"
+            description={summaryPanelQualityGoodText}
+            isLoading={isDatasetsQualityLoading}
+          />
+        </EuiFlexGroup>
+      </EuiFlexGroup>
+    </EuiPanel>
+  );
+}
+
+const QualityIndicator = ({
+  value,
+  quality,
+  description,
+  isLoading,
+}: {
+  value: number;
+  quality: InfoIndicators;
+  description: string;
+  isLoading: boolean;
+}) => {
+  return (
+    <EuiFlexGroup direction="column" gutterSize="xs">
+      {isLoading ? (
+        <EuiSkeletonTitle size="m" />
+      ) : (
+        <EuiTitle size="m">
+          <h3>
+            <EuiHealth textSize="inherit" color={quality}>
+              {value || 0}
+            </EuiHealth>
+          </h3>
+        </EuiTitle>
+      )}
+      <EuiText color={quality}>
+        <h5>{description}</h5>
+      </EuiText>
+    </EuiFlexGroup>
+  );
+};
+
+const verticalRule = css`
+  width: 1px;
+  height: 63px;
+  background-color: ${euiThemeVars.euiColorLightShade};
+`;

x-pack/plugins/dataset_quality/public/components/dataset_quality/summary_panel/estimated_data.tsx

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import { formatBytes } from '@kbn/formatters';
+import { useSummaryPanelContext } from '../../../hooks';
+import {
+  summaryPanelEstimatedDataText,
+  summaryPanelEstimatedDataTooltipText,
+} from '../../../../common/translations';
+import { LastDayDataPlaceholder } from './last_day_data_placeholder';
+
+export function EstimatedData() {
+  const { estimatedData, isEstimatedDataLoading } = useSummaryPanelContext();
+
+  return (
+    <LastDayDataPlaceholder
+      title={summaryPanelEstimatedDataText}
+      tooltip={summaryPanelEstimatedDataTooltipText}
+      value={formatBytes(estimatedData.estimatedDataInBytes)}
+      isLoading={isEstimatedDataLoading}
+    />
+  );
+}

x-pack/plugins/dataset_quality/public/components/dataset_quality/summary_panel/last_day_data_placeholder.tsx

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import {
+  EuiFlexGroup,
+  EuiPanel,
+  EuiFlexItem,
+  EuiTitle,
+  EuiText,
+  EuiIconTip,
+  EuiSkeletonTitle,
+} from '@elastic/eui';
+import { summaryPanelLast24hText } from '../../../../common/translations';
+
+interface LastDayDataPlaceholderParams {
+  title: string;
+  tooltip: string;
+  value: string | number;
+  isLoading: boolean;
+}
+
+export function LastDayDataPlaceholder({
+  title,
+  tooltip,
+  value,
+  isLoading,
+}: LastDayDataPlaceholderParams) {
+  return (
+    <EuiPanel hasBorder>
+      <EuiFlexGroup gutterSize="m" direction="column">
+        <EuiFlexGroup direction="column" gutterSize="xs">
+          <EuiFlexGroup alignItems="center" gutterSize="xs" responsive={false}>
+            <EuiText size="s">{title}</EuiText>
+            <EuiFlexItem grow={false}>
+              <EuiIconTip content={tooltip} />
+            </EuiFlexItem>
+          </EuiFlexGroup>
+          <EuiText color="subdued" size="xs">
+            {summaryPanelLast24hText}
+          </EuiText>
+        </EuiFlexGroup>
+        {isLoading ? (
+          <EuiSkeletonTitle size="m" />
+        ) : (
+          <EuiTitle size="m">
+            <h3>{value}</h3>
+          </EuiTitle>
+        )}
+      </EuiFlexGroup>
+    </EuiPanel>
+  );
+}

x-pack/plugins/dataset_quality/public/components/dataset_quality/summary_panel/summary_panel.tsx

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React from 'react';
+
+import { EuiFlexGroup } from '@elastic/eui';
+import { DatasetsQualityIndicators } from './datasets_quality_indicators';
+import { DatasetsActivity } from './datasets_activity';
+import { EstimatedData } from './estimated_data';
+
+// Allow for lazy loading
+// eslint-disable-next-line import/no-default-export
+export default function SummaryPanel() {
+  return (
+    <EuiFlexGroup gutterSize="m">
+      <DatasetsQualityIndicators />
+      <EuiFlexGroup gutterSize="m">
+        <DatasetsActivity />
+        <EstimatedData />
+      </EuiFlexGroup>
+    </EuiFlexGroup>
+  );
+}

x-pack/plugins/dataset_quality/public/components/quality_indicator/helpers.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { countBy } from 'lodash';
+import {
+  POOR_QUALITY_MINIMUM_PERCENTAGE,
+  DEGRADED_QUALITY_MINIMUM_PERCENTAGE,
+} from '../../../common/constants';
+import { QualityIndicators } from '../common';
+
+export const mapPercentageToQuality = (percentage: number): QualityIndicators => {
+  return percentage > POOR_QUALITY_MINIMUM_PERCENTAGE
+    ? 'poor'
+    : percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE
+    ? 'degraded'
+    : 'good';
+};
+
+export const mapPercentagesToQualityCounts = (
+  percentages: number[]
+): Record<QualityIndicators, number> =>
+  countBy(percentages.map(mapPercentageToQuality)) as Record<QualityIndicators, number>;

x-pack/plugins/dataset_quality/public/components/quality_indicator/index.ts

@@ -7,3 +7,4 @@
 
 export * from './indicator';
 export * from './percentage_indicator';
+export * from './helpers';

x-pack/plugins/dataset_quality/public/components/quality_indicator/indicator.tsx

@@ -7,15 +7,16 @@
 
 import { EuiHealth } from '@elastic/eui';
 import React, { ReactNode } from 'react';
+import { InfoIndicators, QualityIndicators } from '../common';
 
 export function QualityIndicator({
   quality,
   description,
 }: {
-  quality: 'good' | 'degraded' | 'poor';
+  quality: QualityIndicators;
   description: string | ReactNode;
 }) {
-  const qualityColors = {
+  const qualityColors: Record<QualityIndicators, InfoIndicators> = {
     poor: 'danger',
     degraded: 'warning',
     good: 'success',

x-pack/plugins/dataset_quality/public/components/quality_indicator/percentage_indicator.tsx

@@ -8,19 +8,11 @@
 import { EuiText } from '@elastic/eui';
 import { FormattedNumber } from '@kbn/i18n-react';
 import React from 'react';
-import {
-  DEGRADED_QUALITY_MINIMUM_PERCENTAGE,
-  POOR_QUALITY_MINIMUM_PERCENTAGE,
-} from '../../../common/constants';
+import { mapPercentageToQuality } from './helpers';
 import { QualityIndicator } from './indicator';
 
 export function QualityPercentageIndicator({ percentage = 0 }: { percentage?: number }) {
-  const quality =
-    percentage > POOR_QUALITY_MINIMUM_PERCENTAGE
-      ? 'poor'
-      : percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE
-      ? 'degraded'
-      : 'good';
+  const quality = mapPercentageToQuality(percentage);
 
   const description = (
     <EuiText size="s">

x-pack/plugins/dataset_quality/public/controller/create_controller.ts

@@ -10,12 +10,11 @@ import { getDevToolsOptions } from '@kbn/xstate-utils';
 import equal from 'fast-deep-equal';
 import { distinctUntilChanged, from, map } from 'rxjs';
 import { interpret } from 'xstate';
-import { DataStreamsStatsService } from '../services/data_streams_stats';
+import { IDataStreamsStatsClient } from '../services/data_streams_stats';
 import {
   createDatasetQualityControllerStateMachine,
   DEFAULT_CONTEXT,
 } from '../state_machines/dataset_quality_controller';
-import { DatasetQualityStartDeps } from '../types';
 import { getContextFromPublicState, getPublicStateFromContext } from './public_state';
 import { DatasetQualityController, DatasetQualityPublicStateUpdate } from './types';
 
@@ -23,11 +22,11 @@ type InitialState = DatasetQualityPublicStateUpdate;
 
 interface Dependencies {
   core: CoreStart;
-  plugins: DatasetQualityStartDeps;
+  dataStreamStatsClient: IDataStreamsStatsClient;
 }
 
 export const createDatasetQualityControllerFactory =
-  ({ core }: Dependencies) =>
+  ({ core, dataStreamStatsClient }: Dependencies) =>
   async ({
     initialState = DEFAULT_CONTEXT,
   }: {
@@ -35,10 +34,6 @@ export const createDatasetQualityControllerFactory =
   }): Promise<DatasetQualityController> => {
     const initialContext = getContextFromPublicState(initialState ?? {});
 
-    const dataStreamStatsClient = new DataStreamsStatsService().start({
-      http: core.http,
-    }).client;
-
     const machine = createDatasetQualityControllerStateMachine({
       initialContext,
       toasts: core.notifications.toasts,

x-pack/plugins/dataset_quality/public/hooks/index.ts

@@ -8,3 +8,4 @@
 export * from './use_dataset_quality_table';
 export * from './use_dataset_quality_flyout';
 export * from './use_link_to_logs_explorer';
+export * from './use_summary_panel';

x-pack/plugins/dataset_quality/public/hooks/use_summary_panel.tsx

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import createContainer from 'constate';
+import { useInterpret, useSelector } from '@xstate/react';
+import { IToasts } from '@kbn/core-notifications-browser';
+import { IDataStreamsStatsClient } from '../services/data_streams_stats';
+import { createDatasetsSummaryPanelStateMachine } from '../state_machines/summary_panel';
+
+interface SummaryPanelContextDeps {
+  dataStreamStatsClient: IDataStreamsStatsClient;
+  toasts: IToasts;
+}
+
+const useSummaryPanel = ({ dataStreamStatsClient, toasts }: SummaryPanelContextDeps) => {
+  const summaryPanelStateService = useInterpret(() =>
+    createDatasetsSummaryPanelStateMachine({
+      dataStreamStatsClient,
+      toasts,
+    })
+  );
+
+  /*
+    Datasets Quality
+  */
+  const datasetsQuality = useSelector(
+    summaryPanelStateService,
+    (state) => state.context.datasetsQuality
+  );
+  const isDatasetsQualityLoading = useSelector(
+    summaryPanelStateService,
+    (state) =>
+      state.matches('datasetsQuality.fetching') || state.matches('datasetsQuality.retrying')
+  );
+
+  /*
+    Datasets Activity
+  */
+  const datasetsActivity = useSelector(
+    summaryPanelStateService,
+    (state) => state.context.datasetsActivity
+  );
+  const isDatasetsActivityLoading = useSelector(
+    summaryPanelStateService,
+    (state) =>
+      state.matches('datasetsActivity.fetching') || state.matches('datasetsActivity.retrying')
+  );
+
+  /*
+    Estimated Data
+  */
+  const estimatedData = useSelector(
+    summaryPanelStateService,
+    (state) => state.context.estimatedData
+  );
+  const isEstimatedDataLoading = useSelector(
+    summaryPanelStateService,
+    (state) => state.matches('estimatedData.fetching') || state.matches('estimatedData.retrying')
+  );
+
+  return {
+    datasetsQuality,
+    isDatasetsQualityLoading,
+
+    isEstimatedDataLoading,
+    estimatedData,
+
+    isDatasetsActivityLoading,
+    datasetsActivity,
+  };
+};
+
+const [SummaryPanelProvider, useSummaryPanelContext] = createContainer(useSummaryPanel);
+
+export { useSummaryPanelContext };
+
+// eslint-disable-next-line import/no-default-export
+export default SummaryPanelProvider;

x-pack/plugins/dataset_quality/public/plugin.tsx

@@ -8,6 +8,7 @@
 import { CoreSetup, CoreStart, Plugin, PluginInitializerContext } from '@kbn/core/public';
 import { createDatasetQuality } from './components/dataset_quality';
 import { createDatasetQualityControllerLazyFactory } from './controller/lazy_create_controller';
+import { DataStreamsStatsService } from './services/data_streams_stats';
 import {
   DatasetQualityPluginSetup,
   DatasetQualityPluginStart,
@@ -25,14 +26,19 @@ export class DatasetQualityPlugin
   }
 
   public start(core: CoreStart, plugins: DatasetQualityStartDeps): DatasetQualityPluginStart {
+    const dataStreamStatsClient = new DataStreamsStatsService().start({
+      http: core.http,
+    }).client;
+
     const DatasetQuality = createDatasetQuality({
       core,
       plugins,
+      dataStreamStatsClient,
     });
 
     const createDatasetQualityController = createDatasetQualityControllerLazyFactory({
       core,
-      plugins,
+      dataStreamStatsClient,
     });
 
     return { DatasetQuality, createDatasetQualityController };

x-pack/plugins/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts

@@ -12,6 +12,7 @@ import {
   getDataStreamsDegradedDocsStatsResponseRt,
   getDataStreamsStatsResponseRt,
   getDataStreamsDetailsResponseRt,
+  getDataStreamsEstimatedDataInBytesResponseRt,
 } from '../../../common/api_types';
 import { DEFAULT_DATASET_TYPE } from '../../../common/constants';
 import {
@@ -23,6 +24,8 @@ import {
   GetDataStreamsStatsResponse,
   GetDataStreamDetailsParams,
   GetDataStreamDetailsResponse,
+  GetDataStreamsEstimatedDataInBytesParams,
+  GetDataStreamsEstimatedDataInBytesResponse,
 } from '../../../common/data_streams_stats';
 import { DataStreamDetails } from '../../../common/data_streams_stats';
 import { DataStreamStat } from '../../../common/data_streams_stats/data_stream_stat';
@@ -100,4 +103,31 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
 
     return dataStreamDetails as DataStreamDetails;
   }
+
+  public async getDataStreamsEstimatedDataInBytes(
+    params: GetDataStreamsEstimatedDataInBytesParams
+  ) {
+    const response = await this.http
+      .get<GetDataStreamsEstimatedDataInBytesResponse>(
+        `/internal/dataset_quality/data_streams/estimated_data`,
+        {
+          ...params,
+        }
+      )
+      .catch((error) => {
+        throw new GetDataStreamsStatsError(
+          `Failed to fetch data streams estimated data in bytes": ${error}`
+        );
+      });
+
+    const dataStreamsEstimatedDataInBytes = decodeOrThrow(
+      getDataStreamsEstimatedDataInBytesResponseRt,
+      (message: string) =>
+        new GetDataStreamsStatsError(
+          `Failed to decode data streams estimated data in bytes response: ${message}"`
+        )
+    )(response);
+
+    return dataStreamsEstimatedDataInBytes;
+  }
 }

x-pack/plugins/dataset_quality/public/services/data_streams_stats/types.ts

@@ -13,6 +13,8 @@ import {
   GetDataStreamsStatsQuery,
   GetDataStreamDetailsParams,
   DataStreamDetails,
+  GetDataStreamsEstimatedDataInBytesParams,
+  GetDataStreamsEstimatedDataInBytesResponse,
 } from '../../../common/data_streams_stats';
 
 export type DataStreamsStatsServiceSetup = void;
@@ -31,4 +33,7 @@ export interface IDataStreamsStatsClient {
     params?: GetDataStreamsDegradedDocsStatsQuery
   ): Promise<DataStreamDegradedDocsStatServiceResponse>;
   getDataStreamDetails(params: GetDataStreamDetailsParams): Promise<DataStreamDetails>;
+  getDataStreamsEstimatedDataInBytes(
+    params: GetDataStreamsEstimatedDataInBytesParams
+  ): Promise<GetDataStreamsEstimatedDataInBytesResponse>;
 }

x-pack/plugins/dataset_quality/public/state_machines/summary_panel/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './src';

x-pack/plugins/dataset_quality/public/state_machines/summary_panel/src/defaults.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DefaultDatasetsSummaryPanelContext } from './types';
+
+export const MAX_RETRIES = 1;
+export const RETRY_DELAY_IN_MS = 5000;
+
+export const defaultContext: DefaultDatasetsSummaryPanelContext = {
+  datasetsQuality: {
+    percentages: [],
+  },
+  datasetsActivity: {
+    total: 0,
+    active: 0,
+  },
+  estimatedData: {
+    estimatedDataInBytes: 0,
+  },
+  retries: {
+    datasetsQualityRetries: 0,
+    datasetsActivityRetries: 0,
+    estimatedDataRetries: 0,
+  },
+};

x-pack/plugins/dataset_quality/public/state_machines/summary_panel/src/index.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './state_machine';
+export * from './types';
+export * from './defaults';
+export * from './notifications';

x-pack/plugins/dataset_quality/public/state_machines/summary_panel/src/notifications.ts

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IToasts } from '@kbn/core/public';
+import { i18n } from '@kbn/i18n';
+
+export const fetchDatasetsQualityFailedNotifier = (toasts: IToasts, error: Error) => {
+  toasts.addDanger({
+    title: i18n.translate('xpack.datasetQuality.fetchDatasetsQualityDetailsFailed', {
+      defaultMessage: "We couldn't get your datasets quality details. Default values are shown.",
+    }),
+    text: error.message,
+  });
+};
+
+export const fetchDatasetsActivityFailedNotifier = (toasts: IToasts, error: Error) => {
+  toasts.addDanger({
+    title: i18n.translate('xpack.datasetQuality.fetchDatasetsActivityFailed', {
+      defaultMessage:
+        "We couldn't get your active/inactive datasets details. Default values are shown.",
+    }),
+    text: error.message,
+  });
+};
+
+export const fetchDatasetsEstimatedDataFailedNotifier = (toasts: IToasts, error: Error) => {
+  toasts.addDanger({
+    title: i18n.translate('xpack.datasetQuality.fetchDatasetsEstimatedDataFailed', {
+      defaultMessage: "We couldn't get your datasets estimated data. Default values are shown.",
+    }),
+    text: error.message,
+  });
+};

x-pack/plugins/dataset_quality/public/state_machines/summary_panel/src/state_machine.ts

@@ -0,0 +1,241 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IToasts } from '@kbn/core/public';
+import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
+import { getDefaultTimeRange } from '../../../utils';
+import { filterInactiveDatasets } from '../../../utils/filter_inactive_datasets';
+import { IDataStreamsStatsClient } from '../../../services/data_streams_stats';
+import { defaultContext, MAX_RETRIES, RETRY_DELAY_IN_MS } from './defaults';
+import {
+  DatasetsActivityDetails,
+  DatasetsQuality,
+  DatasetsSummaryPanelContext,
+  DatasetsSummaryPanelState,
+  DatasetSummaryPanelEvent,
+  DefaultDatasetsSummaryPanelContext,
+  EstimatedDataDetails,
+  Retries,
+} from './types';
+import {
+  fetchDatasetsEstimatedDataFailedNotifier,
+  fetchDatasetsActivityFailedNotifier,
+  fetchDatasetsQualityFailedNotifier,
+} from './notifications';
+
+export const createPureDatasetsSummaryPanelStateMachine = (
+  initialContext: DatasetsSummaryPanelContext
+) =>
+  /** @xstate-layout N4IgpgJg5mDOIC5QGUCuBbdBDATgTwAUsA7MAGwDoIsAXLWMG2ARVSzIEsa8KAzRgMYALDsSgBiCAHtSFUQDcpAazAU0mXIRLkqteoxZtO3PoJFiECqQNocZAbQAMAXSfPEoAA5TYXO8Q8QAA9EABZQgCYKAHYAVgBmADZQ6IAOUNjQgE5YgEZYgBoQPEQAWlTcilyUx1q6lNSI0IBfZqL1bHwiUkpqOgYmVnYuHn4aYVEJMBwcKRwKTzJaXjn0NQxOrR7dfoMh41GzSctiRRsafzc3QO9fC5lAkIRI1IoK3IrE1LzonMSikoICJZChZL5NUKOTJfZJZFptEAdTTdHR9fSDIwjUzjcxTGZzBZLGgrHBrJFdbS9PQDQzDExjCYWKznS4uey5dxIEC3PwPLlPSKxCjAxJNRzxVKJLJw1IAxAfUIxdK5XLRaK5Rzg2KpVrtDbIyk7dGwACCAgu8ixDNxkhkqisKnWGgp2zRNLNFqtRyZp2stgcbJcNx8vIC-LCsSi2u+EVyEXiWUc0SlhWKiATlWqwNjwNSqSy8VisV1iP1LtR1IMHo4lvp3rxs3mi2WqydmxRVN2TGrtcOOOOzP9xCuQa5PPuYdAAviitCqWi4oSsUThdCcueWUqsca0ay0Xi+9CiRL5K2Fa7pvNNa9-bE4mmjcJLdJbYNrsr3avvexjKgJzOQ5XBywZ3P4jxhDOFBzguhaFiumTrokCRKtUjhgrk0qNNEJ5lmelBwBc2A0JAAAieg-ja0iyA6qinh2FAERwRGkeR1oDr6LIBq4o5eCGE7gc8hYULE0SRAejh5jkcLrpKiQUIk8QSbkimOLkSHxLkOHOnhDGwIRtAsXQFGTPe+JNkSJJkrh9GMcxEBkUZbE+gBE4jpyvGgXyU5hEm8kSvm8SKTBXzrlk+ZVKJEpgpKmTAlp7aGrZBn2ax9amY+zbEq2dGJXpTHJQ5WDGc5fquWywFjnxYHhs8jiKnG8QRKkdXpPGM7riqdWghh4SSrUcQZK0CLEFIEBwIEOU9CBoYCQuMQJI4EQRNEESxOKkTROupSrUKvxNcCjVQgWXzxW+57GvsIzTfxNVxlEFQ-JqMLigpW1qa8R7SotETfeqKSneWnYXZida3lA13Vd5CCqo4bz5DDiTPUk8TrnkwlFsuiYLgWinpADOlunsIM8DgjD4JMENecEiCI7DSFhdqSTLYkSYdQeFCBb8qpZE1PORvCerafRhMYnSPBkFIWBjRAlOTtTCBpJmy0ibG8MY4hsNQtkR5QvkB348LH6Xp63CywJKogtqLNLZkjSJpK66KfEcPShESGQk0arHgik3ne6X43r+Zu3Yk0TCX1NtzsCEn-GmG5yd8cbqhJc5wd7gsJe+F49lipM0OTYjB1DofO3kGluy8GShCjcfLnJyS2wu8aM1kBuGiLxvXiYEtS5ARfy2kiqFqHPM5NU+6x4Ca1CoWmqxizi0Hq3PvWbl+nESldD908KqvHEilLSta0zst65qiCQWJmFvxFqtbfbElG+FcV4OVZ5cs74j80H8tq3rafcc9xCjnIjJCzNGqBXvjoR+hkip5wLq-DyM0aorRBC9cIa1q7JmUh1I63V3r5Fkn9KB+E8p2Wfj3aW29EB2zhitZMjUmiBVlIA-cbxQhxm1GhcIqoIhDWaEAA */
+  createMachine<DatasetsSummaryPanelContext, DatasetSummaryPanelEvent, DatasetsSummaryPanelState>(
+    {
+      context: initialContext,
+      predictableActionArguments: true,
+      id: 'DatasetsQualitySummaryPanel',
+      type: 'parallel',
+      states: {
+        datasetsQuality: {
+          initial: 'fetching',
+          states: {
+            fetching: {
+              invoke: {
+                src: 'loadDatasetsQuality',
+                onDone: {
+                  target: 'loaded',
+                  actions: ['storeDatasetsQuality'],
+                },
+                onError: [
+                  {
+                    target: 'retrying',
+                    cond: {
+                      type: 'canRetry',
+                      counter: 'datasetsQualityRetries',
+                    },
+                    actions: ['incrementDatasetsQualityRetries'],
+                  },
+                  {
+                    target: 'loaded',
+                    actions: ['notifyFetchDatasetsQualityFailed'],
+                  },
+                ],
+              },
+            },
+            retrying: {
+              after: {
+                [RETRY_DELAY_IN_MS]: 'fetching',
+              },
+            },
+            loaded: {
+              type: 'final',
+            },
+          },
+        },
+        datasetsActivity: {
+          initial: 'fetching',
+          states: {
+            fetching: {
+              invoke: {
+                src: 'loadDatasetsActivity',
+                onDone: {
+                  target: 'loaded',
+                  actions: ['storeDatasetsActivity'],
+                },
+                onError: [
+                  {
+                    target: 'retrying',
+                    cond: {
+                      type: 'canRetry',
+                      counter: 'datasetsActivityRetries',
+                    },
+                    actions: ['incrementDatasetsActivityRetries'],
+                  },
+                  {
+                    target: 'loaded',
+                    actions: ['notifyFetchDatasetsActivityFailed'],
+                  },
+                ],
+              },
+            },
+            retrying: {
+              after: {
+                [RETRY_DELAY_IN_MS]: 'fetching',
+              },
+            },
+            loaded: {
+              type: 'final',
+            },
+          },
+        },
+        estimatedData: {
+          initial: 'fetching',
+          states: {
+            fetching: {
+              invoke: {
+                src: 'loadEstimatedData',
+                onDone: {
+                  target: 'loaded',
+                  actions: ['storeEstimatedData'],
+                },
+                onError: [
+                  {
+                    target: 'retrying',
+                    cond: {
+                      type: 'canRetry',
+                      counter: 'estimatedDataRetries',
+                    },
+                    actions: ['incrementEstimatedDataRetries'],
+                  },
+                  {
+                    target: 'loaded',
+                    actions: ['notifyFetchEstimatedDataFailed'],
+                  },
+                ],
+              },
+            },
+            retrying: {
+              after: {
+                [RETRY_DELAY_IN_MS]: 'fetching',
+              },
+            },
+            loaded: {
+              type: 'final',
+            },
+          },
+        },
+      },
+    },
+    {
+      actions: {
+        storeDatasetsQuality: assign((_context, event) =>
+          'data' in event ? { datasetsQuality: event.data as DatasetsQuality } : {}
+        ),
+        storeDatasetsActivity: assign((_context, event) =>
+          'data' in event ? { datasetsActivity: event.data as DatasetsActivityDetails } : {}
+        ),
+        storeEstimatedData: assign((_context, event) =>
+          'data' in event
+            ? {
+                estimatedData: event.data as EstimatedDataDetails,
+              }
+            : {}
+        ),
+        incrementDatasetsQualityRetries: assign(({ retries }, _event) => ({
+          retries: { ...retries, datasetsQualityRetries: retries.datasetsQualityRetries + 1 },
+        })),
+        incrementDatasetsActivityRetries: assign(({ retries }, _event) => ({
+          retries: { ...retries, datasetsActivityRetries: retries.datasetsActivityRetries + 1 },
+        })),
+        incrementEstimatedDataRetries: assign(({ retries }, _event) => ({
+          retries: { ...retries, estimatedDataRetries: retries.estimatedDataRetries + 1 },
+        })),
+      },
+      guards: {
+        canRetry: (context, event, { cond }) => {
+          if ('counter' in cond && cond.counter in context.retries) {
+            const retriesKey = cond.counter as keyof Retries;
+            return context.retries[retriesKey] < MAX_RETRIES;
+          }
+          return false;
+        },
+      },
+    }
+  );
+
+export interface DatasetsSummaryPanelStateMachineDependencies {
+  initialContext?: DefaultDatasetsSummaryPanelContext;
+  toasts: IToasts;
+  dataStreamStatsClient: IDataStreamsStatsClient;
+}
+
+export const createDatasetsSummaryPanelStateMachine = ({
+  initialContext = defaultContext,
+  toasts,
+  dataStreamStatsClient,
+}: DatasetsSummaryPanelStateMachineDependencies) =>
+  createPureDatasetsSummaryPanelStateMachine(initialContext).withConfig({
+    actions: {
+      notifyFetchDatasetsQualityFailed: (_context, event: DoneInvokeEvent<Error>) =>
+        fetchDatasetsQualityFailedNotifier(toasts, event.data),
+      notifyFetchDatasetsActivityFailed: (_context, event: DoneInvokeEvent<Error>) =>
+        fetchDatasetsActivityFailedNotifier(toasts, event.data),
+      notifyFetchEstimatedDataFailed: (_context, event: DoneInvokeEvent<Error>) =>
+        fetchDatasetsEstimatedDataFailedNotifier(toasts, event.data),
+    },
+    services: {
+      loadDatasetsQuality: async (_context) => {
+        const dataStreamsStats = await dataStreamStatsClient.getDataStreamsDegradedStats();
+        const percentages = dataStreamsStats.map((stat) => stat.percentage);
+        return { percentages };
+      },
+      loadDatasetsActivity: async (_context) => {
+        const dataStreamsStats = await dataStreamStatsClient.getDataStreamsStats();
+        const activeDataStreams = filterInactiveDatasets({ datasets: dataStreamsStats });
+        return {
+          total: dataStreamsStats.length,
+          active: activeDataStreams.length,
+        };
+      },
+      loadEstimatedData: async (_context) => {
+        const { from: start, to: end } = getDefaultTimeRange();
+        return dataStreamStatsClient.getDataStreamsEstimatedDataInBytes({
+          query: {
+            type: 'logs',
+            start,
+            end,
+          },
+        });
+      },
+    },
+  });
+
+export type DatasetsSummaryPanelStateService = InterpreterFrom<
+  typeof createDatasetsSummaryPanelStateMachine
+>;
+
+export type DatasetsSummaryPanelStateMachine = ReturnType<
+  typeof createDatasetsSummaryPanelStateMachine
+>;

x-pack/plugins/dataset_quality/public/state_machines/summary_panel/src/types.ts

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DoneInvokeEvent } from 'xstate';
+import { GetDataStreamsEstimatedDataInBytesResponse } from '../../../../common/data_streams_stats';
+
+export interface Retries {
+  datasetsQualityRetries: number;
+  datasetsActivityRetries: number;
+  estimatedDataRetries: number;
+}
+
+export interface DatasetsQuality {
+  percentages: number[];
+}
+
+export interface DatasetsActivityDetails {
+  total: number;
+  active: number;
+}
+
+export interface EstimatedDataDetails {
+  estimatedDataInBytes: number;
+}
+
+export interface WithDatasetsQuality {
+  datasetsQuality: DatasetsQuality;
+}
+
+export interface WithActiveDatasets {
+  datasetsActivity: DatasetsActivityDetails;
+}
+
+export interface WithEstimatedData {
+  estimatedData: EstimatedDataDetails;
+}
+
+export interface WithRetries {
+  retries: Retries;
+}
+
+export type DefaultDatasetsSummaryPanelContext = WithDatasetsQuality &
+  WithActiveDatasets &
+  WithEstimatedData &
+  WithRetries;
+
+export type DatasetsSummaryPanelState =
+  | {
+      value: 'datasetsQuality.fetching';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'datasetsQuality.loaded';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'datasetsQuality.retrying';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'datasetsActivity.fetching';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'datasetsActivity.loaded';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'datasetsActivity.retrying';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'estimatedData.fetching';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'estimatedData.loaded';
+      context: DefaultDatasetsSummaryPanelContext;
+    }
+  | {
+      value: 'estimatedData.retrying';
+      context: DefaultDatasetsSummaryPanelContext;
+    };
+
+export type DatasetSummaryPanelEvent =
+  | DoneInvokeEvent<Retries>
+  | DoneInvokeEvent<DatasetsQuality>
+  | DoneInvokeEvent<DatasetsActivityDetails>
+  | DoneInvokeEvent<GetDataStreamsEstimatedDataInBytesResponse>
+  | DoneInvokeEvent<Error>;
+
+export type DatasetsSummaryPanelContext = DatasetsSummaryPanelState['context'];

x-pack/plugins/dataset_quality/public/utils/filter_inactive_datasets.ts

@@ -6,20 +6,21 @@
  */
 
 import { DataStreamStat } from '../../common/data_streams_stats';
+import { getDefaultTimeRange } from './default_timerange';
 
 interface FilterInactiveDatasetsOptions {
   datasets: DataStreamStat[];
-  timeRange: {
+  timeRange?: {
     from: string;
     to: string;
   };
 }
 
-export const filterInactiveDatasets = (options: FilterInactiveDatasetsOptions) => {
-  const {
-    datasets,
-    timeRange: { from, to },
-  } = options;
+export const filterInactiveDatasets = ({
+  datasets,
+  timeRange = getDefaultTimeRange(),
+}: FilterInactiveDatasetsOptions) => {
+  const { from, to } = timeRange;
 
   const startDate = new Date(from).getTime();
   const endDate = new Date(to).getTime();

x-pack/plugins/dataset_quality/server/routes/data_streams/get_degraded_docs.ts

@@ -21,8 +21,8 @@ import { createDatasetQualityESClient, wildcardQuery } from '../../utils';
 export async function getDegradedDocsPaginated(options: {
   esClient: ElasticsearchClient;
   type?: DataStreamType;
-  start: number;
-  end: number;
+  start?: number;
+  end?: number;
   datasetQuery?: string;
   after?: {
     dataset: string;

x-pack/plugins/dataset_quality/server/routes/data_streams/get_estimated_data_in_bytes/index.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core/server';
+import { DEFAULT_DATASET_TYPE } from '../../../../common/constants';
+import { DataStreamType } from '../../../../common/types';
+import { indexStatsService } from '../../../services';
+
+export async function getEstimatedDataInBytes(args: {
+  esClient: ElasticsearchClient;
+  type?: DataStreamType;
+  start: number;
+  end: number;
+}) {
+  const { esClient, type = DEFAULT_DATASET_TYPE, start, end } = args;
+
+  const [{ doc_count: docCount, size_in_bytes: docSize }, indexDocCountInTimeRange] =
+    await Promise.all([
+      indexStatsService.getIndexStats(esClient, type),
+      indexStatsService.getIndexDocCount(esClient, type, start, end),
+    ]);
+
+  if (!docCount) return 0;
+
+  const avgDocSize = docSize / docCount;
+  const estimatedDataInBytes = Math.round(indexDocCountInTimeRange * avgDocSize);
+
+  return estimatedDataInBytes;
+}

x-pack/plugins/dataset_quality/server/routes/data_streams/routes.ts

@@ -21,6 +21,7 @@ import { getDataStreams } from './get_data_streams';
 import { getDataStreamsStats } from './get_data_streams_stats';
 import { getDegradedDocsPaginated } from './get_degraded_docs';
 import { getIntegrations } from './get_integrations';
+import { getEstimatedDataInBytes } from './get_estimated_data_in_bytes';
 
 const statsRoute = createDatasetQualityServerRoute({
   endpoint: 'GET /internal/dataset_quality/data_streams/stats',
@@ -70,7 +71,7 @@ const degradedDocsRoute = createDatasetQualityServerRoute({
   endpoint: 'GET /internal/dataset_quality/data_streams/degraded_docs',
   params: t.type({
     query: t.intersection([
-      rangeRt,
+      t.partial(rangeRt.props),
       typeRt,
       t.partial({
         datasetQuery: t.string,
@@ -135,8 +136,36 @@ const dataStreamDetailsRoute = createDatasetQualityServerRoute({
   },
 });
 
+const estimatedDataInBytesRoute = createDatasetQualityServerRoute({
+  endpoint: 'GET /internal/dataset_quality/data_streams/estimated_data',
+  params: t.type({
+    query: t.intersection([typeRt, rangeRt]),
+  }),
+  options: {
+    tags: [],
+  },
+  async handler(resources): Promise<{
+    estimatedDataInBytes: number;
+  }> {
+    const { context, params } = resources;
+    const coreContext = await context.core;
+
+    const esClient = coreContext.elasticsearch.client.asCurrentUser;
+
+    const estimatedDataInBytes = await getEstimatedDataInBytes({
+      esClient,
+      ...params.query,
+    });
+
+    return {
+      estimatedDataInBytes,
+    };
+  },
+});
+
 export const dataStreamsRouteRepository = {
   ...statsRoute,
   ...degradedDocsRoute,
   ...dataStreamDetailsRoute,
+  ...estimatedDataInBytesRoute,
 };

x-pack/plugins/dataset_quality/server/services/index.ts

@@ -6,3 +6,4 @@
  */
 
 export { dataStreamService } from './data_stream';
+export { indexStatsService } from './index_stats';

x-pack/plugins/dataset_quality/server/services/index_stats.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core/server';
+import { rangeQuery } from '@kbn/observability-plugin/server';
+import { DataStreamType } from '../../common/types';
+
+class IndexStatsService {
+  public async getIndexStats(
+    esClient: ElasticsearchClient,
+    type: DataStreamType
+  ): Promise<{
+    doc_count: number;
+    size_in_bytes: number;
+  }> {
+    try {
+      const index = `${type}-*-*`;
+
+      const indexStats = await esClient.indices.stats({ index });
+      return {
+        doc_count: indexStats._all.total?.docs ? indexStats._all.total?.docs?.count : 0,
+        size_in_bytes: indexStats._all.total?.store
+          ? indexStats._all.total?.store.size_in_bytes
+          : 0,
+      };
+    } catch (e) {
+      if (e.statusCode === 404) {
+        return { doc_count: 0, size_in_bytes: 0 };
+      }
+      throw e;
+    }
+  }
+
+  public async getIndexDocCount(
+    esClient: ElasticsearchClient,
+    type: DataStreamType,
+    start: number,
+    end: number
+  ): Promise<number> {
+    try {
+      const index = `${type}-*-*`;
+
+      const query = rangeQuery(start, end)[0];
+      const docCount = await esClient.count({
+        index,
+        query,
+      });
+
+      return docCount.count;
+    } catch (e) {
+      if (e.statusCode === 404) {
+        return 0;
+      }
+      throw e;
+    }
+  }
+}
+
+export const indexStatsService = new IndexStatsService();

x-pack/plugins/dataset_quality/tsconfig.json

@@ -29,6 +29,9 @@
     "@kbn/router-utils",
     "@kbn/xstate-utils",
     "@kbn/shared-ux-utility",
+    "@kbn/ui-theme",
+    "@kbn/core-notifications-browser",
+    "@kbn/formatters"
   ],
   "exclude": [
     "target/**/*",

x-pack/test/dataset_quality_api_integration/tests/data_streams/estimated_data.spec.ts

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { log, timerange } from '@kbn/apm-synthtrace-client';
+import expect from '@kbn/expect';
+import { DatasetQualityApiClientKey } from '../../common/config';
+import { FtrProviderContext } from '../../common/ftr_provider_context';
+
+export default function ApiTest({ getService }: FtrProviderContext) {
+  const registry = getService('registry');
+  const synthtrace = getService('logSynthtraceEsClient');
+  const datasetQualityApiClient = getService('datasetQualityApiClient');
+  const start = '2023-12-11T18:00:00.000Z';
+  const oneDayEnd = '2023-12-12T18:00:00.000Z';
+  const oneWeekEnd = '2023-12-18T18:00:00.000Z';
+  const dataset = 'nginx.access';
+  const namespace = 'default';
+
+  async function callApiAs(type: 'logs' | 'metrics', end: string) {
+    const user = 'datasetQualityLogsUser' as DatasetQualityApiClientKey;
+    return await datasetQualityApiClient[user]({
+      endpoint: 'GET /internal/dataset_quality/data_streams/estimated_data',
+      params: {
+        query: {
+          type,
+          start,
+          end,
+        },
+      },
+    });
+  }
+
+  registry.when('Estimated Data Details', { config: 'basic' }, () => {
+    describe('gets the data streams estimated data', () => {
+      before(async () => {
+        await synthtrace.index([
+          timerange(start, oneWeekEnd)
+            .interval('1h')
+            .rate(1)
+            .generator((timestamp) =>
+              log
+                .create()
+                .message('This is a log message')
+                .timestamp(timestamp)
+                .dataset(dataset)
+                .namespace(namespace)
+                .defaults({
+                  'log.file.path': '/my-service.log',
+                })
+            ),
+        ]);
+      });
+
+      it('returns a non-empty body', async () => {
+        const resp = await callApiAs('logs', oneDayEnd);
+        expect(resp.body).not.empty();
+      });
+
+      it('returns correct estimated data for 1 day of logs', async () => {
+        const resp = await callApiAs('logs', oneDayEnd);
+        expect(resp.body.estimatedDataInBytes).to.be.lessThan(2500).greaterThan(1000);
+      });
+
+      it('returns correct estimated data for 1 week of logs', async () => {
+        const resp = await callApiAs('logs', oneWeekEnd);
+        expect(resp.body.estimatedDataInBytes).to.be.lessThan(20000).greaterThan(10000);
+      });
+
+      it('returns correct estimated data for no data index', async () => {
+        const resp = await callApiAs('metrics', oneWeekEnd);
+        expect(resp.body.estimatedDataInBytes).to.equal(0);
+      });
+
+      after(async () => {
+        await synthtrace.clean();
+      });
+    });
+  });
+}

yarn.lock

@@ -4760,6 +4760,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/formatters@link:packages/kbn-formatters":
+  version "0.0.0"
+  uid ""
+
 "@kbn/ftr-apis-plugin@link:src/plugins/ftr_apis":
   version "0.0.0"
   uid ""