[SecuritySolution] Metering/stats api integration (#179666)

## Summary

Serverless pending on
https://github.com/elastic/elasticsearch-serverless/pull/1482

1. Adding a data fatcher for metering/stats api - This is for Serverless
Kibana especially.
2. Given that the response for metering/stats api is simpler. I amended
the response of original stats api to align with the metering/stats.

Both apis now return:
e.g.:

```
{
      ".ds-my-datastream-03-02-2024-00001": {
          "name": ".ds-my-datastream-03-02-2024-00001",
          "num_docs": 3,
          "size_in_bytes": 15785, // ---> null on serverless
          "datastream": "my-datastream" // ---> ds name, null on ESS
      },
     "my-test-index": {
        "name": "my-test-index",
        "num_docs": 2,
        "size_in_bytes": 11462
      },
}
```

**Note**: 
When an index health is `green` , its primaries.docs.count and
total.docs.count have different values after deployed to the cloud.

However, the doc count of both health yellow and green indices were all
coming from mockStatsGreenIndex[indexName].primaries?.docs?.count. So
when we are normalising the api formate, we should just use
`primaries.docs.count` to retrieve the doc count.

Example:
https://github.com/elastic/kibana/pull/179666/files#diff-49d86730fb4dd8bff8fca0ca72013703759075a4a81694349db881ebb1173188R60

**Steps to verify (serverless):**


https://p.elstc.co/paste/5R+Z497w#oT-1mqcTLxIEwYNtZFnv4BMYclGU7HPS1By+XQAjeIR

0.
x-pack/plugins/security_solution/public/app/solution_navigation/links/app_links.ts
remove L30-33 to reveal the entry of dq dashboard on serverless.
1. Mock the response in
https://github.com/elastic/kibana/pull/179666/files#diff-24b3062cb373e7849bda6cf4be35466545f00333b558d2047fcdaaf272a9ba4cR44
to return
```
Promise.resolve({
    _shards: {
      total: 4,
      successful: 2,
      failed: 0,
    },
    indices: [
      {
        name: '.ds-my-datastream-03-02-2024-00001',
        num_docs: 3,
        datastream: 'my-datastream', // ---> ds name
      },
      {
        name: 'my-test-index',
        num_docs: 2,
      },
    ],
    datastreams: [
      {
        name: 'my-datastream',
        num_docs: 6,
      },
    ],
    total: {
      num_docs: 8,
    },
  });
```
2. Create an index with docs containing same-family field
```
PUT my-test-index

PUT my-test-index/_mapping
{
"properties": {
  "@timestamp": {
    "type": "date"
  },
  "agent.type": {
    "type": "constant_keyword"
  },
  "event.category": {
    "type": "constant_keyword"
  }
}
}

POST my-test-index/_doc
{
"@timestamp": "2024-04-08T09:41:49.668Z",
"host": {
  "name": "foo"
},
"event": {
  "category": "an_invalid_category"
},
"some.field": "this",
"source": {
  "port": 90210,
  "ip": "10.1.2.3"
}
}

POST my-test-index/_doc
{
"@timestamp": "2024-02-08T09:42:22.123Z",
"host": {
  "name": "bar"
},
"event": {
  "category": "an_invalid_category"
},
"some.field": "space",
"source": {
  "port": 867,
  "ip": "10.9.8.7"
}
}
```
3. Add my-test-* into security-default-dataview from stack management.

**Expected result (Serverless)** (with the mock api response as above to
simulate the result):
1. `doc size` is hidden on the Serverless dq dashboard.
2. It shows `doc count` in the treemap for each index on the Serverless
dq dashboard.


<img width="2560" alt="Screenshot 2024-04-15 at 11 28 33"
src="1d3b7d3e-4650-4711-b76b-6215de9e1a59">




70edb22c-15c9-4f2e-b740-5bf2bd630da5









**Expected result (ESS)**: Should remain the same


https://p.elstc.co/paste/yRGlJc2z#mfABwGRSjk7dbxhzQ3bn+jqCtSSBT5UVK4W46CsVoBh

<img width="2560" alt="Screenshot 2024-04-10 at 09 31 58"
src="b6194cc8-8975-4ac4-b157-64d1e39ecb28">
 




### Checklist

Delete any items that are not applicable to this PR.

- [x] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Sergi Massaneda <sergi.massaneda@gmail.com>

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/index.tsx

@@ -102,6 +102,7 @@ const DataQualityDetailsComponent: React.FC<Props> = ({
     <>
       <StorageDetails
         formatBytes={formatBytes}
+        formatNumber={formatNumber}
         ilmPhases={ilmPhases}
         onIndexSelected={onIndexSelected}
         patterns={patterns}

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/helpers.test.ts

@@ -70,16 +70,16 @@ describe('helpers', () => {
       );
     });
 
-    test('it returns zero when the pattern exists in the rollup, but does not have a sizeInBytes', () => {
+    test('it returns undefined when the pattern exists in the rollup, but does not have a sizeInBytes', () => {
       const pattern = 'valid-*';
 
-      expect(getPatternSizeInBytes({ pattern, patternRollups: noSizeInBytes })).toEqual(0);
+      expect(getPatternSizeInBytes({ pattern, patternRollups: noSizeInBytes })).toBeUndefined();
     });
 
-    test('it returns zero when the pattern does NOT exist in the rollup', () => {
+    test('it returns undefined when the pattern does NOT exist in the rollup', () => {
       const pattern = 'does-not-exist-*';
 
-      expect(getPatternSizeInBytes({ pattern, patternRollups })).toEqual(0);
+      expect(getPatternSizeInBytes({ pattern, patternRollups })).toBeUndefined();
     });
   });
 
@@ -93,6 +93,7 @@ describe('helpers', () => {
         index: null,
         pattern,
         sizeInBytes: auditbeatWithAllResults.sizeInBytes,
+        docsCount: auditbeatWithAllResults.docsCount,
       });
     });
   });
@@ -113,6 +114,7 @@ describe('helpers', () => {
           index: '.ds-auditbeat-8.6.1-2023.02.07-000001',
           pattern: 'auditbeat-*',
           sizeInBytes: 18791790,
+          docsCount: 19123,
         },
         {
           color: euiThemeVars.euiColorDanger,
@@ -120,6 +122,7 @@ describe('helpers', () => {
           index: 'auditbeat-custom-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 28409,
+          docsCount: 4,
         },
         {
           color: euiThemeVars.euiColorDanger,
@@ -127,6 +130,7 @@ describe('helpers', () => {
           index: 'auditbeat-custom-empty-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 247,
+          docsCount: 0,
         },
       ]);
     });
@@ -145,6 +149,7 @@ describe('helpers', () => {
           index: '.ds-auditbeat-8.6.1-2023.02.07-000001',
           pattern: 'auditbeat-*',
           sizeInBytes: 18791790,
+          docsCount: 19123,
         },
         {
           color: euiThemeVars.euiColorDanger,
@@ -152,6 +157,7 @@ describe('helpers', () => {
           index: 'auditbeat-custom-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 28409,
+          docsCount: 4,
         },
         {
           color: euiThemeVars.euiColorDanger,
@@ -159,6 +165,7 @@ describe('helpers', () => {
           index: 'auditbeat-custom-empty-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 247,
+          docsCount: 0,
         },
       ]);
     });
@@ -179,6 +186,7 @@ describe('helpers', () => {
           index: null,
           pattern: '.alerts-security.alerts-default',
           sizeInBytes: 29717961631,
+          docsCount: 26093,
         },
         {
           color: euiThemeVars.euiColorSuccess,
@@ -186,14 +194,23 @@ describe('helpers', () => {
           index: '.internal.alerts-security.alerts-default-000001',
           pattern: '.alerts-security.alerts-default',
           sizeInBytes: 0,
+          docsCount: 26093,
+        },
+        {
+          color: null,
+          ilmPhase: null,
+          index: null,
+          pattern: 'auditbeat-*',
+          sizeInBytes: 18820446,
+          docsCount: 19127,
         },
-        { color: null, ilmPhase: null, index: null, pattern: 'auditbeat-*', sizeInBytes: 18820446 },
         {
           color: euiThemeVars.euiColorSuccess,
           ilmPhase: 'hot',
           index: '.ds-auditbeat-8.6.1-2023.02.07-000001',
           pattern: 'auditbeat-*',
           sizeInBytes: 18791790,
+          docsCount: 19123,
         },
         {
           color: euiThemeVars.euiColorDanger,
@@ -201,6 +218,7 @@ describe('helpers', () => {
           index: 'auditbeat-custom-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 28409,
+          docsCount: 4,
         },
         {
           color: euiThemeVars.euiColorDanger,
@@ -208,6 +226,7 @@ describe('helpers', () => {
           index: 'auditbeat-custom-empty-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 247,
+          docsCount: 0,
         },
         {
           color: null,
@@ -215,6 +234,7 @@ describe('helpers', () => {
           index: null,
           pattern: 'packetbeat-*',
           sizeInBytes: 1096520898,
+          docsCount: 3258632,
         },
         {
           color: euiThemeVars.euiColorPrimary,
@@ -222,6 +242,7 @@ describe('helpers', () => {
           index: '.ds-packetbeat-8.5.3-2023.02.04-000001',
           pattern: 'packetbeat-*',
           sizeInBytes: 584326147,
+          docsCount: 1630289,
         },
         {
           color: euiThemeVars.euiColorPrimary,
@@ -229,6 +250,7 @@ describe('helpers', () => {
           index: '.ds-packetbeat-8.6.1-2023.02.04-000001',
           pattern: 'packetbeat-*',
           sizeInBytes: 512194751,
+          docsCount: 1628343,
         },
       ]);
     });
@@ -249,6 +271,7 @@ describe('helpers', () => {
           indexName: '.internal.alerts-security.alerts-default-000001',
           pattern: '.alerts-security.alerts-default',
           sizeInBytes: 0,
+          docsCount: 26093,
         },
         {
           ilmPhase: 'hot',
@@ -256,6 +279,7 @@ describe('helpers', () => {
           indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001',
           pattern: 'auditbeat-*',
           sizeInBytes: 18791790,
+          docsCount: 19123,
         },
         {
           ilmPhase: 'unmanaged',
@@ -263,6 +287,7 @@ describe('helpers', () => {
           indexName: 'auditbeat-custom-empty-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 247,
+          docsCount: 0,
         },
         {
           ilmPhase: 'unmanaged',
@@ -270,18 +295,21 @@ describe('helpers', () => {
           indexName: 'auditbeat-custom-index-1',
           pattern: 'auditbeat-*',
           sizeInBytes: 28409,
+          docsCount: 4,
         },
         {
           ilmPhase: 'hot',
           indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001',
           pattern: 'packetbeat-*',
           sizeInBytes: 512194751,
+          docsCount: 1628343,
         },
         {
           ilmPhase: 'hot',
           indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
           pattern: 'packetbeat-*',
           sizeInBytes: 584326147,
+          docsCount: 1630289,
         },
       ]);
     });
@@ -295,6 +323,7 @@ describe('helpers', () => {
         })
       ).toEqual([
         {
+          docsCount: 26093,
           ilmPhase: undefined,
           incompatible: 0,
           indexName: '.internal.alerts-security.alerts-default-000001',
@@ -302,6 +331,7 @@ describe('helpers', () => {
           sizeInBytes: 0,
         },
         {
+          docsCount: 19123,
           ilmPhase: undefined,
           incompatible: 0,
           indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001',
@@ -309,6 +339,7 @@ describe('helpers', () => {
           sizeInBytes: 18791790,
         },
         {
+          docsCount: 0,
           ilmPhase: undefined,
           incompatible: 1,
           indexName: 'auditbeat-custom-empty-index-1',
@@ -316,6 +347,7 @@ describe('helpers', () => {
           sizeInBytes: 247,
         },
         {
+          docsCount: 4,
           ilmPhase: undefined,
           incompatible: 3,
           indexName: 'auditbeat-custom-index-1',
@@ -323,12 +355,14 @@ describe('helpers', () => {
           sizeInBytes: 28409,
         },
         {
+          docsCount: 1628343,
           ilmPhase: undefined,
           indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001',
           pattern: 'packetbeat-*',
           sizeInBytes: 512194751,
         },
         {
+          docsCount: 1630289,
           ilmPhase: undefined,
           indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
           pattern: 'packetbeat-*',
@@ -373,6 +407,7 @@ describe('helpers', () => {
           ilmPhase: 'hot',
           incompatible: 0,
           sizeInBytes: 0,
+          docsCount: 26093,
         },
         'auditbeat-*.ds-auditbeat-8.6.1-2023.02.07-000001': {
           pattern: 'auditbeat-*',
@@ -380,6 +415,7 @@ describe('helpers', () => {
           ilmPhase: 'hot',
           incompatible: 0,
           sizeInBytes: 18791790,
+          docsCount: 19123,
         },
         'auditbeat-*auditbeat-custom-empty-index-1': {
           pattern: 'auditbeat-*',
@@ -387,6 +423,7 @@ describe('helpers', () => {
           ilmPhase: 'unmanaged',
           incompatible: 1,
           sizeInBytes: 247,
+          docsCount: 0,
         },
         'auditbeat-*auditbeat-custom-index-1': {
           pattern: 'auditbeat-*',
@@ -394,14 +431,17 @@ describe('helpers', () => {
           ilmPhase: 'unmanaged',
           incompatible: 3,
           sizeInBytes: 28409,
+          docsCount: 4,
         },
         'packetbeat-*.ds-packetbeat-8.6.1-2023.02.04-000001': {
           pattern: 'packetbeat-*',
           indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001',
           ilmPhase: 'hot',
           sizeInBytes: 512194751,
+          docsCount: 1628343,
         },
         'packetbeat-*.ds-packetbeat-8.5.3-2023.02.04-000001': {
+          docsCount: 1630289,
           pattern: 'packetbeat-*',
           indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
           ilmPhase: 'hot',
@@ -457,14 +497,20 @@ describe('helpers', () => {
 
     it('returns the expected number of layers', () => {
       expect(
-        getLayersMultiDimensional({ formatBytes, layer0FillColor, pathToFlattenedBucketMap }).length
+        getLayersMultiDimensional({
+          valueFormatter: formatBytes,
+          layer0FillColor,
+          pathToFlattenedBucketMap,
+        }).length
       ).toEqual(2);
     });
 
     it('returns the expected fillLabel valueFormatter function', () => {
-      getLayersMultiDimensional({ formatBytes, layer0FillColor, pathToFlattenedBucketMap }).forEach(
+      getLayersMultiDimensional({
-        (x) => expect(x.fillLabel.valueFormatter(123)).toEqual('123B')
+        valueFormatter: formatBytes,
-      );
+        layer0FillColor,
+        pathToFlattenedBucketMap,
+      }).forEach((x) => expect(x.fillLabel.valueFormatter(123)).toEqual('123B'));
     });
   });
 });

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/helpers.ts

@@ -9,7 +9,7 @@ import type { Datum, Key, ArrayNode } from '@elastic/charts';
 import { euiThemeVars } from '@kbn/ui-theme';
 import { orderBy } from 'lodash/fp';
 
-import { getSizeInBytes } from '../../../../helpers';
+import { getDocsCount, getSizeInBytes } from '../../../../helpers';
 import { getIlmPhase } from '../../../pattern/helpers';
 import { PatternRollup } from '../../../../types';
 
@@ -18,7 +18,8 @@ export interface LegendItem {
   ilmPhase: string | null;
   index: string | null;
   pattern: string;
-  sizeInBytes: number;
+  sizeInBytes: number | undefined;
+  docsCount: number;
 }
 
 export interface FlattenedBucket {
@@ -26,7 +27,8 @@ export interface FlattenedBucket {
   incompatible: number | undefined;
   indexName: string | undefined;
   pattern: string;
-  sizeInBytes: number;
+  sizeInBytes: number | undefined;
+  docsCount: number;
 }
 
 export const getPatternSizeInBytes = ({
@@ -35,9 +37,23 @@ export const getPatternSizeInBytes = ({
 }: {
   pattern: string;
   patternRollups: Record<string, PatternRollup>;
+}): number | undefined => {
+  if (patternRollups[pattern] != null) {
+    return patternRollups[pattern].sizeInBytes;
+  } else {
+    return undefined;
+  }
+};
+
+export const getPatternDocsCount = ({
+  pattern,
+  patternRollups,
+}: {
+  pattern: string;
+  patternRollups: Record<string, PatternRollup>;
 }): number => {
   if (patternRollups[pattern] != null) {
-    return patternRollups[pattern].sizeInBytes ?? 0;
+    return patternRollups[pattern].docsCount ?? 0;
   } else {
     return 0;
   }
@@ -55,6 +71,7 @@ export const getPatternLegendItem = ({
   index: null,
   pattern,
   sizeInBytes: getPatternSizeInBytes({ pattern, patternRollups }),
+  docsCount: getPatternDocsCount({ pattern, patternRollups }),
 });
 
 export const getLegendItemsForPattern = ({
@@ -74,7 +91,8 @@ export const getLegendItemsForPattern = ({
         ilmPhase: flattenedBucket.ilmPhase ?? null,
         index: flattenedBucket.indexName ?? null,
         pattern: flattenedBucket.pattern,
-        sizeInBytes: flattenedBucket.sizeInBytes ?? 0,
+        sizeInBytes: flattenedBucket.sizeInBytes,
+        docsCount: flattenedBucket.docsCount,
       }))
   );
 
@@ -129,7 +147,7 @@ export const getFlattenedBuckets = ({
                   ? results[indexName].incompatible
                   : undefined;
               const sizeInBytes = getSizeInBytes({ indexName, stats });
-
+              const docsCount = getDocsCount({ stats, indexName });
               return [
                 ...validStats,
                 {
@@ -138,6 +156,7 @@ export const getFlattenedBuckets = ({
                   indexName,
                   pattern,
                   sizeInBytes,
+                  docsCount,
                 },
               ];
             } else {
@@ -187,16 +206,14 @@ export const getGroupFromPath = (path: ArrayNode['path']): string | undefined =>
 };
 
 export const getLayersMultiDimensional = ({
-  formatBytes,
+  valueFormatter,
   layer0FillColor,
   pathToFlattenedBucketMap,
 }: {
-  formatBytes: (value: number | undefined) => string;
+  valueFormatter: (value: number) => string;
   layer0FillColor: string;
   pathToFlattenedBucketMap: Record<string, FlattenedBucket | undefined>;
 }) => {
-  const valueFormatter = (d: number) => formatBytes(d);
-
   return [
     {
       fillLabel: {

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.test.tsx

@@ -22,6 +22,10 @@ const defaultBytesFormat = '0,0.[0]b';
 const formatBytes = (value: number | undefined) =>
   value != null ? numeral(value).format(defaultBytesFormat) : EMPTY_STAT;
 
+const defaultNumberFormat = '0,0.[000]';
+const formatNumber = (value: number | undefined) =>
+  value != null ? numeral(value).format(defaultNumberFormat) : EMPTY_STAT;
+
 const ilmPhases = ['hot', 'warm', 'unmanaged'];
 const patterns = ['.alerts-security.alerts-default', 'auditbeat-*', 'packetbeat-*'];
 
@@ -35,6 +39,7 @@ const onIndexSelected = jest.fn();
 
 const defaultProps: Props = {
   formatBytes,
+  formatNumber,
   ilmPhases,
   onIndexSelected,
   patternRollups,

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.tsx

@@ -6,16 +6,18 @@
  */
 
 import type { PartialTheme, Theme } from '@elastic/charts';
-import React, { useMemo } from 'react';
+import React, { useCallback, useMemo } from 'react';
 
 import { getFlattenedBuckets } from './helpers';
 import { StorageTreemap } from '../../../storage_treemap';
 import { DEFAULT_MAX_CHART_HEIGHT, StorageTreemapContainer } from '../../../tabs/styles';
 import { PatternRollup, SelectedIndex } from '../../../../types';
 import { useDataQualityContext } from '../../../data_quality_context';
+import { DOCS_UNIT } from './translations';
 
 export interface Props {
   formatBytes: (value: number | undefined) => string;
+  formatNumber: (value: number | undefined) => string;
   ilmPhases: string[];
   onIndexSelected: ({ indexName, pattern }: SelectedIndex) => void;
   patternRollups: Record<string, PatternRollup>;
@@ -26,6 +28,7 @@ export interface Props {
 
 const StorageDetailsComponent: React.FC<Props> = ({
   formatBytes,
+  formatNumber,
   ilmPhases,
   onIndexSelected,
   patternRollups,
@@ -44,18 +47,25 @@ const StorageDetailsComponent: React.FC<Props> = ({
       }),
     [ilmPhases, isILMAvailable, patternRollups]
   );
+  const accessor = flattenedBuckets[0]?.sizeInBytes != null ? 'sizeInBytes' : 'docsCount';
+  const valueFormatter = useCallback(
+    (d: number) =>
+      accessor === 'sizeInBytes' ? formatBytes(d) : `${formatNumber(d)} ${DOCS_UNIT(d)}`,
+    [accessor, formatBytes, formatNumber]
+  );
 
   return (
     <StorageTreemapContainer data-test-subj="storageDetails">
       <StorageTreemap
+        accessor={accessor}
+        baseTheme={baseTheme}
         flattenedBuckets={flattenedBuckets}
-        formatBytes={formatBytes}
         maxChartHeight={DEFAULT_MAX_CHART_HEIGHT}
         onIndexSelected={onIndexSelected}
-        patterns={patterns}
         patternRollups={patternRollups}
+        patterns={patterns}
         theme={theme}
-        baseTheme={baseTheme}
+        valueFormatter={valueFormatter}
       />
     </StorageTreemapContainer>
   );

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/translations.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export const DOCS_UNIT = (totalCount: number) =>
+  i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.storage.docs.unit', {
+    values: { totalCount },
+    defaultMessage: `{totalCount, plural, =1 {Doc} other {Docs}}`,
+  });

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts

@@ -5,11 +5,10 @@
  * 2.0.
  */
 
-import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
 import { orderBy } from 'lodash/fp';
 
 import { getDocsCount } from '../../../../helpers';
-import type { IndexToCheck, PatternRollup } from '../../../../types';
+import type { IndexToCheck, MeteringStatsIndex, PatternRollup } from '../../../../types';
 
 export const getIndexToCheck = ({
   indexName,
@@ -54,7 +53,7 @@ export const getIndexDocsCountFromRollup = ({
   indexName: string;
   patternRollup: PatternRollup;
 }): number => {
-  const stats: Record<string, IndicesStatsIndicesStats> | null = patternRollup?.stats ?? null;
+  const stats: Record<string, MeteringStatsIndex> | null = patternRollup?.stats ?? null;
 
   return getDocsCount({
     indexName,

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/index.tsx

@@ -74,6 +74,7 @@ export const getAllMarkdownCommentsFromResults = ({
   const summaryTableMarkdownRows: string[] = summaryTableItems.map((item) => {
     const result: DataQualityCheckResult | undefined =
       patternRollup.results != null ? patternRollup.results[item.indexName] : undefined;
+    const sizeInBytes = getSizeInBytes({ indexName: item.indexName, stats: patternRollup.stats });
 
     return getSummaryTableMarkdownRow({
       docsCount: item.docsCount,
@@ -84,7 +85,7 @@ export const getAllMarkdownCommentsFromResults = ({
       incompatible: result?.incompatible,
       isILMAvailable,
       patternDocsCount: patternRollup.docsCount ?? 0,
-      sizeInBytes: getSizeInBytes({ indexName: item.indexName, stats: patternRollup.stats }),
+      sizeInBytes,
     }).trim();
   });
 

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts

@@ -432,7 +432,14 @@ describe('helpers', () => {
     test('it returns the expected header when isILMAvailable is false', () => {
       const isILMAvailable = false;
       expect(getSummaryTableMarkdownHeader(isILMAvailable)).toEqual(
-        '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|'
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|'
+      );
+    });
+
+    test('it returns the expected header when displayDocSize is false', () => {
+      const isILMAvailable = false;
+      expect(getSummaryTableMarkdownHeader(isILMAvailable)).toEqual(
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|'
       );
     });
   });
@@ -481,9 +488,25 @@ describe('helpers', () => {
           indexName: 'auditbeat-custom-index-1',
           isILMAvailable: false,
           patternDocsCount: 57410,
-          sizeInBytes: 28413,
+          sizeInBytes: undefined,
         })
-      ).toEqual('| -- | auditbeat-custom-index-1 | 4 (0.0%) | -- | 27.7KB |\n');
+      ).toEqual('| -- | auditbeat-custom-index-1 | 4 (0.0%) | -- |\n');
+    });
+
+    test('it returns the expected row when sizeInBytes is undefined', () => {
+      expect(
+        getSummaryTableMarkdownRow({
+          docsCount: 4,
+          formatBytes,
+          formatNumber,
+          incompatible: undefined, // <--
+          ilmPhase: undefined, // <--
+          indexName: 'auditbeat-custom-index-1',
+          isILMAvailable: false,
+          patternDocsCount: 57410,
+          sizeInBytes: undefined,
+        })
+      ).toEqual('| -- | auditbeat-custom-index-1 | 4 (0.0%) | -- |\n');
     });
   });
 
@@ -517,10 +540,28 @@ describe('helpers', () => {
           isILMAvailable: false,
           partitionedFieldMetadata: mockPartitionedFieldMetadata,
           patternDocsCount: 57410,
-          sizeInBytes: 28413,
+          sizeInBytes: undefined,
         })
       ).toEqual(
-        '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | 27.7KB |\n\n'
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n'
+      );
+    });
+
+    test('it returns the expected comment when sizeInBytes is undefined', () => {
+      expect(
+        getSummaryTableMarkdownComment({
+          docsCount: 4,
+          formatBytes,
+          formatNumber,
+          ilmPhase: 'unmanaged',
+          indexName: 'auditbeat-custom-index-1',
+          isILMAvailable: false,
+          partitionedFieldMetadata: mockPartitionedFieldMetadata,
+          patternDocsCount: 57410,
+          sizeInBytes: undefined,
+        })
+      ).toEqual(
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n'
       );
     });
   });
@@ -554,7 +595,7 @@ describe('helpers', () => {
           sizeInBytes: undefined,
         })
       ).toEqual(
-        '| Incompatible fields | Indices checked | Indices | Size | Docs |\n|---------------------|-----------------|---------|------|------|\n| -- | -- | -- | -- | 0 |\n'
+        '| Incompatible fields | Indices checked | Indices | Docs |\n|---------------------|-----------------|---------|------|\n| -- | -- | -- | 0 |\n'
       );
     });
   });
@@ -588,7 +629,7 @@ describe('helpers', () => {
           sizeInBytes: undefined,
         })
       ).toEqual(
-        '# Data quality\n\n| Incompatible fields | Indices checked | Indices | Size | Docs |\n|---------------------|-----------------|---------|------|------|\n| -- | -- | -- | -- | 0 |\n\n'
+        '# Data quality\n\n| Incompatible fields | Indices checked | Indices | Docs |\n|---------------------|-----------------|---------|------|\n| -- | -- | -- | 0 |\n\n'
       );
     });
   });

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts

@@ -218,18 +218,18 @@ export const getResultEmoji = (incompatible: number | undefined): string => {
   }
 };
 
-export const getSummaryTableMarkdownHeader = (isILMAvailable: boolean): string =>
+export const getSummaryTableMarkdownHeader = (includeDocSize: boolean): string =>
-  isILMAvailable
+  includeDocSize
     ? `| ${RESULT} | ${INDEX} | ${DOCS} | ${INCOMPATIBLE_FIELDS} | ${ILM_PHASE} | ${SIZE} |
 |${getHeaderSeparator(RESULT)}|${getHeaderSeparator(INDEX)}|${getHeaderSeparator(
         DOCS
       )}|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator(
         ILM_PHASE
       )}|${getHeaderSeparator(SIZE)}|`
-    : `| ${RESULT} | ${INDEX} | ${DOCS} | ${INCOMPATIBLE_FIELDS} | ${SIZE} |
+    : `| ${RESULT} | ${INDEX} | ${DOCS} | ${INCOMPATIBLE_FIELDS} |
 |${getHeaderSeparator(RESULT)}|${getHeaderSeparator(INDEX)}|${getHeaderSeparator(
         DOCS
-      )}|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator(SIZE)}|`;
+      )}|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|`;
 
 export const getSummaryTableMarkdownRow = ({
   docsCount,
@@ -252,7 +252,7 @@ export const getSummaryTableMarkdownRow = ({
   patternDocsCount: number;
   sizeInBytes: number | undefined;
 }): string =>
-  isILMAvailable
+  isILMAvailable && Number.isInteger(sizeInBytes)
     ? `| ${getResultEmoji(incompatible)} | ${escape(indexName)} | ${formatNumber(
         docsCount
       )} (${getDocsCountPercent({
@@ -267,7 +267,7 @@ export const getSummaryTableMarkdownRow = ({
       )} (${getDocsCountPercent({
         docsCount,
         patternDocsCount,
-      })}) | ${incompatible ?? EMPTY_PLACEHOLDER} | ${formatBytes(sizeInBytes)} |
+      })}) | ${incompatible ?? EMPTY_PLACEHOLDER} |
 `;
 
 export const getSummaryTableMarkdownComment = ({
@@ -322,13 +322,22 @@ export const getStatsRollupMarkdownComment = ({
   indicesChecked: number | undefined;
   sizeInBytes: number | undefined;
 }): string =>
-  `| ${INCOMPATIBLE_FIELDS} | ${INDICES_CHECKED} | ${INDICES} | ${SIZE} | ${DOCS} |
+  Number.isInteger(sizeInBytes)
+    ? `| ${INCOMPATIBLE_FIELDS} | ${INDICES_CHECKED} | ${INDICES} | ${SIZE} | ${DOCS} |
 |${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator(
-    INDICES_CHECKED
+        INDICES_CHECKED
-  )}|${getHeaderSeparator(INDICES)}|${getHeaderSeparator(SIZE)}|${getHeaderSeparator(DOCS)}|
+      )}|${getHeaderSeparator(INDICES)}|${getHeaderSeparator(SIZE)}|${getHeaderSeparator(DOCS)}|
 | ${incompatible ?? EMPTY_STAT} | ${indicesChecked ?? EMPTY_STAT} | ${
-    indices ?? EMPTY_STAT
+        indices ?? EMPTY_STAT
-  } | ${formatBytes(sizeInBytes)} | ${formatNumber(docsCount)} |
+      } | ${formatBytes(sizeInBytes)} | ${formatNumber(docsCount)} |
+`
+    : `| ${INCOMPATIBLE_FIELDS} | ${INDICES_CHECKED} | ${INDICES} | ${DOCS} |
+|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator(
+        INDICES_CHECKED
+      )}|${getHeaderSeparator(INDICES)}|${getHeaderSeparator(DOCS)}|
+| ${incompatible ?? EMPTY_STAT} | ${indicesChecked ?? EMPTY_STAT} | ${
+        indices ?? EMPTY_STAT
+      } | ${formatNumber(docsCount)} |
 `;
 
 export const getDataQualitySummaryMarkdownComment = ({

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts

@@ -441,7 +441,7 @@ describe('helpers', () => {
           indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001',
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
-          sizeInBytes: 0,
+          sizeInBytes: undefined,
           checkedAt: undefined,
         },
         {
@@ -451,7 +451,7 @@ describe('helpers', () => {
           indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
-          sizeInBytes: 0,
+          sizeInBytes: undefined,
           checkedAt: undefined,
         },
         {
@@ -461,7 +461,7 @@ describe('helpers', () => {
           indexName: 'auditbeat-custom-index-1',
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
-          sizeInBytes: 0,
+          sizeInBytes: undefined,
           checkedAt: undefined,
         },
       ]);

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts

@@ -5,10 +5,7 @@
  * 2.0.
  */
 
-import type {
+import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types';
-  IlmExplainLifecycleLifecycleExplain,
-  IndicesStatsIndicesStats,
-} from '@elastic/elasticsearch/lib/api/types';
 import { isEqual, orderBy } from 'lodash/fp';
 
 import type { IndexSummaryTableItem } from '../summary_table/helpers';
@@ -18,6 +15,7 @@ import type {
   DataQualityCheckResult,
   PatternRollup,
   SortConfig,
+  MeteringStatsIndex,
 } from '../../types';
 import { getDocsCount, getSizeInBytes } from '../../helpers';
 
@@ -159,7 +157,7 @@ export const getSummaryTableItems = ({
   results: Record<string, DataQualityCheckResult> | undefined;
   sortByColumn: string;
   sortByDirection: 'desc' | 'asc';
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }): IndexSummaryTableItem[] => {
   const summaryTableItems = indexNames.map((indexName) => ({
     docsCount: getDocsCount({ stats, indexName }),
@@ -189,7 +187,7 @@ export const shouldCreateIndexNames = ({
   indexNames: string[] | undefined;
   isILMAvailable: boolean;
   newIndexNames: string[];
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }): boolean => {
   return (
     !isEqual(newIndexNames, indexNames) &&
@@ -211,7 +209,7 @@ export const shouldCreatePatternRollup = ({
   isILMAvailable: boolean;
   newDocsCount: number;
   patternRollup: PatternRollup | undefined;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }): boolean => {
   if (patternRollup?.docsCount === newDocsCount) {
     return false;

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/index.tsx

@@ -272,10 +272,12 @@ const PatternComponent: React.FC<Props> = ({
         indices: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }).length,
         pattern,
         results: undefined,
-        sizeInBytes: getTotalSizeInBytes({
+        sizeInBytes: isILMAvailable
-          indexNames: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }),
+          ? getTotalSizeInBytes({
-          stats,
+              indexNames: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }),
-        }),
+              stats,
+            }) ?? 0
+          : undefined,
         stats,
       });
     }
@@ -335,7 +337,7 @@ const PatternComponent: React.FC<Props> = ({
             ilmExplainPhaseCounts={ilmExplainPhaseCounts}
             pattern={pattern}
             patternDocsCount={patternRollup?.docsCount ?? 0}
-            patternSizeInBytes={patternRollup?.sizeInBytes ?? 0}
+            patternSizeInBytes={patternRollup?.sizeInBytes}
           />
           <EuiSpacer />
         </EuiFlexItem>

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/index.tsx

@@ -21,7 +21,7 @@ interface Props {
   indicesChecked: number | undefined;
   pattern: string;
   patternDocsCount: number;
-  patternSizeInBytes: number;
+  patternSizeInBytes: number | undefined;
 }
 
 const PatternSummaryComponent: React.FC<Props> = ({

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/stats_rollup/index.tsx

@@ -120,23 +120,25 @@ const StatsRollupComponent: React.FC<Props> = ({
         </IndicesStatContainer>
       </EuiFlexItem>
 
-      <EuiFlexItem grow={false}>
+      {sizeInBytes != null && (
-        <IndicesStatContainer>
+        <EuiFlexItem grow={false}>
-          <EuiToolTip
+          <IndicesStatContainer>
-            content={
+            <EuiToolTip
-              pattern != null
+              content={
-                ? i18n.INDICES_SIZE_PATTERN_TOOL_TIP(pattern)
+                pattern != null
-                : i18n.TOTAL_SIZE_TOOL_TIP
+                  ? i18n.INDICES_SIZE_PATTERN_TOOL_TIP(pattern)
-            }
+                  : i18n.TOTAL_SIZE_TOOL_TIP
-          >
+              }
-            <EuiStat
+            >
-              description={sizeDescription}
+              <EuiStat
-              title={sizeInBytes != null ? formatBytes(sizeInBytes) : EMPTY_STAT}
+                description={sizeDescription}
-              titleSize={STAT_TITLE_SIZE}
+                title={sizeInBytes != null ? formatBytes(sizeInBytes) : EMPTY_STAT}
-            />
+                titleSize={STAT_TITLE_SIZE}
-          </EuiToolTip>
+              />
-        </IndicesStatContainer>
+            </EuiToolTip>
-      </EuiFlexItem>
+          </IndicesStatContainer>
+        </EuiFlexItem>
+      )}
 
       <EuiFlexItem grow={false}>
         <DocsContainer>

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.test.tsx

@@ -49,13 +49,14 @@ const flattenedBuckets = getFlattenedBuckets({
 const onIndexSelected = jest.fn();
 
 const defaultProps: Props = {
+  accessor: 'sizeInBytes',
   flattenedBuckets,
-  formatBytes,
   maxChartHeight: DEFAULT_MAX_CHART_HEIGHT,
   onIndexSelected,
   patternRollups,
   patterns,
   baseTheme: DARK_THEME,
+  valueFormatter: formatBytes,
 };
 
 jest.mock('@elastic/charts', () => {

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.tsx

@@ -39,15 +39,16 @@ export const LEGEND_WIDTH = 220; // px
 export const LEGEND_TEXT_WITH = 120; // px
 
 export interface Props {
+  accessor: 'sizeInBytes' | 'docsCount';
+  baseTheme: Theme;
   flattenedBuckets: FlattenedBucket[];
-  formatBytes: (value: number | undefined) => string;
   maxChartHeight?: number;
   minChartHeight?: number;
   onIndexSelected: ({ indexName, pattern }: SelectedIndex) => void;
   patternRollups: Record<string, PatternRollup>;
   patterns: string[];
   theme?: PartialTheme;
-  baseTheme: Theme;
+  valueFormatter: (value: number) => string;
 }
 
 interface GetGroupByFieldsResult {
@@ -84,15 +85,16 @@ export const getGroupByFieldsOnClick = (
 };
 
 const StorageTreemapComponent: React.FC<Props> = ({
+  accessor,
+  baseTheme,
   flattenedBuckets,
-  formatBytes,
   maxChartHeight,
   minChartHeight = DEFAULT_MIN_CHART_HEIGHT,
   onIndexSelected,
   patternRollups,
   patterns,
   theme = {},
-  baseTheme,
+  valueFormatter,
 }: Props) => {
   const fillColor = useMemo(
     () => theme?.background?.color ?? baseTheme.background.color,
@@ -129,14 +131,14 @@ const StorageTreemapComponent: React.FC<Props> = ({
   const layers = useMemo(
     () =>
       getLayersMultiDimensional({
-        formatBytes,
+        valueFormatter,
         layer0FillColor: fillColor,
         pathToFlattenedBucketMap,
       }),
-    [fillColor, formatBytes, pathToFlattenedBucketMap]
+    [fillColor, valueFormatter, pathToFlattenedBucketMap]
   );
 
-  const valueAccessor = useCallback(({ sizeInBytes }: Datum) => sizeInBytes, []);
+  const valueAccessor = useCallback((d: Datum) => d[accessor], [accessor]);
 
   const legendItems = useMemo(
     () => getLegendItems({ patterns, flattenedBuckets, patternRollups }),
@@ -167,7 +169,7 @@ const StorageTreemapComponent: React.FC<Props> = ({
               layers={layers}
               layout={PartitionLayout.treemap}
               valueAccessor={valueAccessor}
-              valueFormatter={(d: number) => formatBytes(d)}
+              valueFormatter={valueFormatter}
             />
           </Chart>
         )}
@@ -180,10 +182,10 @@ const StorageTreemapComponent: React.FC<Props> = ({
           className="eui-yScroll"
           $width={LEGEND_WIDTH}
         >
-          {legendItems.map(({ color, ilmPhase, index, pattern, sizeInBytes }) => (
+          {legendItems.map(({ color, ilmPhase, index, pattern, sizeInBytes, docsCount }) => (
             <ChartLegendItem
               color={color}
-              count={formatBytes(sizeInBytes)}
+              count={valueFormatter(accessor === 'sizeInBytes' ? sizeInBytes ?? 0 : docsCount)}
               dataTestSubj={`chart-legend-item-${ilmPhase}${pattern}${index}`}
               key={`${ilmPhase}${pattern}${index}`}
               onClick={

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx

@@ -565,6 +565,30 @@ describe('helpers', () => {
 
         expect(screen.getByTestId('sizeInBytes')).toHaveTextContent('98.6MB');
       });
+
+      test('it should not render sizeInBytes if it is not a number', () => {
+        const testIndexSummaryTableItem = { ...indexSummaryTableItem, sizeInBytes: undefined };
+        const columns = getSummaryTableColumns({
+          formatBytes,
+          formatNumber,
+          itemIdToExpandedRowMap: {},
+          isILMAvailable,
+          pattern: 'auditbeat-*',
+          toggleExpanded: jest.fn(),
+        });
+
+        const sizeInBytesRender = (columns[6] as EuiTableFieldDataColumnType<IndexSummaryTableItem>)
+          .render;
+
+        render(
+          <TestProviders>
+            {sizeInBytesRender != null &&
+              sizeInBytesRender(testIndexSummaryTableItem, testIndexSummaryTableItem)}
+          </TestProviders>
+        );
+
+        expect(screen.queryByTestId('sizeInBytes')).toBeNull();
+      });
     });
   });
 

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx

@@ -42,7 +42,7 @@ export interface IndexSummaryTableItem {
   ilmPhase: IlmPhase | undefined;
   pattern: string;
   patternDocsCount: number;
-  sizeInBytes: number;
+  sizeInBytes: number | undefined;
   checkedAt: number | undefined;
 }
 
@@ -120,6 +120,24 @@ export const getSummaryTableILMPhaseColumn = (
       ]
     : [];
 
+export const getSummaryTableSizeInBytesColumn = ({
+  formatBytes,
+}: {
+  formatBytes: (value: number | undefined) => string;
+}): Array<EuiBasicTableColumn<IndexSummaryTableItem>> => [
+  {
+    field: 'sizeInBytes',
+    name: i18n.SIZE,
+    render: (_, { sizeInBytes }) =>
+      Number.isInteger(sizeInBytes) ? (
+        <EuiToolTip content={INDEX_SIZE_TOOLTIP}>
+          <span data-test-subj="sizeInBytes">{formatBytes(sizeInBytes)}</span>
+        </EuiToolTip>
+      ) : null,
+    sortable: true,
+    truncateText: false,
+  },
+];
 export const getSummaryTableColumns = ({
   formatBytes,
   formatNumber,
@@ -232,11 +250,12 @@ export const getSummaryTableColumns = ({
   {
     field: 'sizeInBytes',
     name: i18n.SIZE,
-    render: (_, { sizeInBytes }) => (
+    render: (_, { sizeInBytes }) =>
-      <EuiToolTip content={INDEX_SIZE_TOOLTIP}>
+      Number.isInteger(sizeInBytes) ? (
-        <span data-test-subj="sizeInBytes">{formatBytes(sizeInBytes)}</span>
+        <EuiToolTip content={INDEX_SIZE_TOOLTIP}>
-      </EuiToolTip>
+          <span data-test-subj="sizeInBytes">{formatBytes(sizeInBytes)}</span>
-    ),
+        </EuiToolTip>
+      ) : null,
     sortable: true,
     truncateText: false,
   },

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts

@@ -108,7 +108,29 @@ ${ECS_IS_A_PERMISSIVE_SCHEMA}
         })
       ).toEqual([
         '### auditbeat-custom-index-1\n',
-        '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | 27.7KB |\n\n',
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n',
+        '### **Incompatible fields** `3` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
+        `#### 4 Custom field mappings\n\nThese fields are not defined by the Elastic Common Schema (ECS), version ${EcsVersion}.\n\nECS is a permissive schema. If your events have additional data that cannot be mapped to ECS, you can simply add them to your events, using custom field names.\n`,
+        '#### Custom fields - auditbeat-custom-index-1\n\n\n| Field | Index mapping type | \n|-------|--------------------|\n| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |\n',
+      ]);
+    });
+
+    test('it returns the expected comment without Size when Size is undefined', () => {
+      expect(
+        getAllCustomMarkdownComments({
+          docsCount: 4,
+          formatBytes,
+          formatNumber,
+          ilmPhase: 'unmanaged',
+          indexName: 'auditbeat-custom-index-1',
+          isILMAvailable: false,
+          partitionedFieldMetadata: mockPartitionedFieldMetadata,
+          patternDocsCount: 57410,
+          sizeInBytes: undefined,
+        })
+      ).toEqual([
+        '### auditbeat-custom-index-1\n',
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n',
         '### **Incompatible fields** `3` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
         `#### 4 Custom field mappings\n\nThese fields are not defined by the Elastic Common Schema (ECS), version ${EcsVersion}.\n\nECS is a permissive schema. If your events have additional data that cannot be mapped to ECS, you can simply add them to your events, using custom field names.\n`,
         '#### Custom fields - auditbeat-custom-index-1\n\n\n| Field | Index mapping type | \n|-------|--------------------|\n| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |\n',

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx

@@ -15,7 +15,7 @@ import {
   timestamp,
 } from '../../mock/enriched_field_metadata/mock_enriched_field_metadata';
 import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
-import { mockStatsGreenIndex } from '../../mock/stats/mock_stats_green_index';
+import { mockStatsAuditbeatIndex } from '../../mock/stats/mock_stats_packetbeat_index';
 import {
   getEcsCompliantColor,
   getMissingTimestampComment,
@@ -83,7 +83,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 57410,
           setSelectedTabId: jest.fn(),
-          stats: mockStatsGreenIndex,
+          stats: mockStatsAuditbeatIndex,
           baseTheme: DARK_THEME,
         }).map((x) => omit(['append', 'content'], x))
       ).toEqual([

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx

@@ -15,7 +15,6 @@ import type {
   WordCloudElementEvent,
   XYChartElementEvent,
 } from '@elastic/charts';
-import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
 import { EuiBadge } from '@elastic/eui';
 import { euiThemeVars } from '@kbn/ui-theme';
 import React from 'react';
@@ -39,7 +38,12 @@ import * as i18n from '../index_properties/translations';
 import { SameFamilyTab } from './same_family_tab';
 import { SummaryTab } from './summary_tab';
 import { getFillColor } from './summary_tab/helpers';
-import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../types';
+import type {
+  EnrichedFieldMetadata,
+  IlmPhase,
+  MeteringStatsIndex,
+  PartitionedFieldMetadata,
+} from '../../types';
 
 export const getMissingTimestampComment = (): string =>
   getMarkdownComment({
@@ -105,7 +109,7 @@ export const getTabs = ({
   pattern: string;
   patternDocsCount: number;
   setSelectedTabId: (tabId: string) => void;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
   theme?: PartialTheme;
   baseTheme: Theme;
 }) => [

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.test.ts

@@ -403,11 +403,37 @@ ${MAPPINGS_THAT_CONFLICT_WITH_ECS}
           isILMAvailable: false,
           partitionedFieldMetadata: emptyIncompatible,
           patternDocsCount: 57410,
-          sizeInBytes: 28413,
+          sizeInBytes: undefined,
         })
       ).toEqual([
         '### auditbeat-custom-index-1\n',
-        '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|\n| ✅ | auditbeat-custom-index-1 | 4 (0.0%) | 0 | 27.7KB |\n\n',
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ✅ | auditbeat-custom-index-1 | 4 (0.0%) | 0 |\n\n',
+        '### **Incompatible fields** `0` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
+        '\n\n\n',
+      ]);
+    });
+
+    test('it returns the expected comment when `sizeInBytes` is not an integer', () => {
+      const emptyIncompatible: PartitionedFieldMetadata = {
+        ...mockPartitionedFieldMetadata,
+        incompatible: [], // <-- empty
+      };
+
+      expect(
+        getAllIncompatibleMarkdownComments({
+          docsCount: 4,
+          formatBytes,
+          formatNumber,
+          ilmPhase: 'unmanaged',
+          indexName: 'auditbeat-custom-index-1',
+          isILMAvailable: false,
+          partitionedFieldMetadata: emptyIncompatible,
+          patternDocsCount: 57410,
+          sizeInBytes: undefined,
+        })
+      ).toEqual([
+        '### auditbeat-custom-index-1\n',
+        '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ✅ | auditbeat-custom-index-1 | 4 (0.0%) | 0 |\n\n',
         '### **Incompatible fields** `0` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
         '\n\n\n',
       ]);

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts

@@ -62,8 +62,8 @@ import {
   auditbeatWithAllResults,
 } from './mock/pattern_rollup/mock_auditbeat_pattern_rollup';
 import { mockStats } from './mock/stats/mock_stats';
-import { mockStatsGreenIndex } from './mock/stats/mock_stats_green_index';
+import { mockStatsAuditbeatIndex } from './mock/stats/mock_stats_packetbeat_index';
-import { mockStatsYellowIndex } from './mock/stats/mock_stats_yellow_index';
+import { mockStatsPacketbeatIndex } from './mock/stats/mock_stats_auditbeat_index';
 import {
   COLD_DESCRIPTION,
   FROZEN_DESCRIPTION,
@@ -751,12 +751,12 @@ describe('helpers', () => {
   describe('getDocsCount', () => {
     test('it returns the expected docs count when `stats` contains the `indexName`', () => {
       const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
-      const expectedCount = mockStatsYellowIndex[indexName].primaries?.docs?.count;
+      const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs;
 
       expect(
         getDocsCount({
           indexName,
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(expectedCount);
     });
@@ -767,7 +767,7 @@ describe('helpers', () => {
       expect(
         getDocsCount({
           indexName,
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(0);
     });
@@ -789,37 +789,37 @@ describe('helpers', () => {
       expect(
         getDocsCount({
           indexName,
-          stats: mockStatsGreenIndex,
+          stats: mockStatsAuditbeatIndex,
         })
-      ).toEqual(mockStatsGreenIndex[indexName].primaries?.docs?.count);
+      ).toEqual(mockStatsAuditbeatIndex[indexName].num_docs);
     });
   });
 
   describe('getSizeInBytes', () => {
     test('it returns the expected size when `stats` contains the `indexName`', () => {
       const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
-      const expectedCount = mockStatsYellowIndex[indexName].primaries?.store?.size_in_bytes;
+      const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes;
 
       expect(
         getSizeInBytes({
           indexName,
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(expectedCount);
     });
 
-    test('it returns zero when `stats` does NOT contain the `indexName`', () => {
+    test('it returns undefined when `stats` does NOT contain the `indexName`', () => {
       const indexName = 'not-gonna-find-it';
 
       expect(
         getSizeInBytes({
           indexName,
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
-      ).toEqual(0);
+      ).toBeUndefined();
     });
 
-    test('it returns zero when `stats` is null', () => {
+    test('it returns undefined when `stats` is null', () => {
       const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
 
       expect(
@@ -827,7 +827,7 @@ describe('helpers', () => {
           indexName,
           stats: null,
         })
-      ).toEqual(0);
+      ).toBeUndefined();
     });
 
     test('it returns the expected size for a green index, where `primaries.store.size_in_bytes` and `total.store.size_in_bytes` have different values', () => {
@@ -836,21 +836,21 @@ describe('helpers', () => {
       expect(
         getSizeInBytes({
           indexName,
-          stats: mockStatsGreenIndex,
+          stats: mockStatsAuditbeatIndex,
         })
-      ).toEqual(mockStatsGreenIndex[indexName].primaries?.store?.size_in_bytes);
+      ).toEqual(mockStatsAuditbeatIndex[indexName].size_in_bytes);
     });
   });
 
   describe('getTotalDocsCount', () => {
     test('it returns the expected total given a subset of index names in the stats', () => {
       const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001';
-      const expectedCount = mockStatsYellowIndex[indexName].primaries?.docs?.count;
+      const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs;
 
       expect(
         getTotalDocsCount({
           indexNames: [indexName],
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(expectedCount);
     });
@@ -864,7 +864,7 @@ describe('helpers', () => {
       expect(
         getTotalDocsCount({
           indexNames: allIndexNamesInStats,
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(3258632);
     });
@@ -873,19 +873,19 @@ describe('helpers', () => {
       expect(
         getTotalDocsCount({
           indexNames: [], // <-- empty
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(0);
     });
 
     test('it returns the expected total for a green index', () => {
       const indexName = 'auditbeat-custom-index-1';
-      const expectedCount = mockStatsGreenIndex[indexName].primaries?.docs?.count;
+      const expectedCount = mockStatsAuditbeatIndex[indexName].num_docs;
 
       expect(
         getTotalDocsCount({
           indexNames: [indexName],
-          stats: mockStatsGreenIndex,
+          stats: mockStatsAuditbeatIndex,
         })
       ).toEqual(expectedCount);
     });
@@ -894,12 +894,12 @@ describe('helpers', () => {
   describe('getTotalSizeInBytes', () => {
     test('it returns the expected total given a subset of index names in the stats', () => {
       const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001';
-      const expectedCount = mockStatsYellowIndex[indexName].primaries?.store?.size_in_bytes;
+      const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes;
 
       expect(
         getTotalSizeInBytes({
           indexNames: [indexName],
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(expectedCount);
     });
@@ -913,28 +913,58 @@ describe('helpers', () => {
       expect(
         getTotalSizeInBytes({
           indexNames: allIndexNamesInStats,
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(1464758182);
     });
 
-    test('it returns zero given an empty collection of index names', () => {
+    test('it returns undefined given an empty collection of index names', () => {
       expect(
         getTotalSizeInBytes({
           indexNames: [], // <-- empty
-          stats: mockStatsYellowIndex,
+          stats: mockStatsPacketbeatIndex,
         })
-      ).toEqual(0);
+      ).toBeUndefined();
     });
 
-    test('it returns the expected total for a green index', () => {
+    test('it returns undefined if sizeInByte in not an integer', () => {
       const indexName = 'auditbeat-custom-index-1';
-      const expectedCount = mockStatsGreenIndex[indexName].primaries?.store?.size_in_bytes;
 
       expect(
         getTotalSizeInBytes({
           indexNames: [indexName],
-          stats: mockStatsGreenIndex,
+          stats: { [indexName]: { ...mockStatsAuditbeatIndex[indexName], size_in_bytes: null } },
+        })
+      ).toBeUndefined();
+    });
+
+    test('it returns the expected total for an index', () => {
+      const indexName = 'auditbeat-custom-index-1';
+      const expectedCount = mockStatsAuditbeatIndex[indexName].size_in_bytes;
+
+      expect(
+        getTotalSizeInBytes({
+          indexNames: [indexName],
+          stats: mockStatsAuditbeatIndex,
+        })
+      ).toEqual(expectedCount);
+    });
+
+    test('it returns the expected total for indices', () => {
+      const expectedCount = Object.values(mockStatsPacketbeatIndex).reduce(
+        (acc, { size_in_bytes: sizeInBytes }) => {
+          return acc + (sizeInBytes ?? 0);
+        },
+        0
+      );
+
+      expect(
+        getTotalSizeInBytes({
+          indexNames: [
+            '.ds-packetbeat-8.6.1-2023.02.04-000001',
+            '.ds-packetbeat-8.5.3-2023.02.04-000001',
+          ],
+          stats: mockStatsPacketbeatIndex,
         })
       ).toEqual(expectedCount);
     });

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts

@@ -6,10 +6,7 @@
  */
 
 import type { HttpHandler } from '@kbn/core-http-browser';
-import type {
+import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types';
-  IlmExplainLifecycleLifecycleExplain,
-  IndicesStatsIndicesStats,
-} from '@elastic/elasticsearch/lib/api/types';
 import { has, sortBy } from 'lodash/fp';
 import { IToasts } from '@kbn/core-notifications-browser';
 import { getIlmPhase } from './data_quality_panel/pattern/helpers';
@@ -24,6 +21,7 @@ import type {
   EnrichedFieldMetadata,
   ErrorSummary,
   IlmPhase,
+  MeteringStatsIndex,
   PartitionedFieldMetadata,
   PartitionedFieldMetadataStats,
   PatternRollup,
@@ -42,7 +40,7 @@ export const getIndexNames = ({
   ilmExplain: Record<string, IlmExplainLifecycleLifecycleExplain> | null;
   ilmPhases: string[];
   isILMAvailable: boolean;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }): string[] => {
   if (((isILMAvailable && ilmExplain != null) || !isILMAvailable) && stats != null) {
     const allIndexNames = Object.keys(stats);
@@ -270,15 +268,15 @@ export const getDocsCount = ({
   stats,
 }: {
   indexName: string;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
-}): number => (stats && stats[indexName]?.primaries?.docs?.count) ?? 0;
+}): number => (stats && stats[indexName]?.num_docs) ?? 0;
 
 export const getIndexId = ({
   indexName,
   stats,
 }: {
   indexName: string;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }): string | null | undefined => stats && stats[indexName]?.uuid;
 
 export const getSizeInBytes = ({
@@ -286,15 +284,15 @@ export const getSizeInBytes = ({
   stats,
 }: {
   indexName: string;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
-}): number => (stats && stats[indexName]?.primaries?.store?.total_data_set_size_in_bytes) ?? 0;
+}): number | undefined => (stats && stats[indexName]?.size_in_bytes) ?? undefined;
 
 export const getTotalDocsCount = ({
   indexNames,
   stats,
 }: {
   indexNames: string[];
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }): number =>
   indexNames.reduce(
     (acc: number, indexName: string) => acc + getDocsCount({ stats, indexName }),
@@ -306,12 +304,22 @@ export const getTotalSizeInBytes = ({
   stats,
 }: {
   indexNames: string[];
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
-}): number =>
+}): number | undefined => {
-  indexNames.reduce(
+  let sum;
-    (acc: number, indexName: string) => acc + getSizeInBytes({ stats, indexName }),
+  for (let i = 0; i < indexNames.length; i++) {
-    0
+    const currentSizeInBytes = getSizeInBytes({ stats, indexName: indexNames[i] });
-  );
+    if (currentSizeInBytes != null) {
+      if (sum == null) {
+        sum = 0;
+      }
+      sum += currentSizeInBytes;
+    } else {
+      return undefined;
+    }
+  }
+  return sum;
+};
 
 export const EMPTY_STAT = '--';
 
@@ -498,7 +506,7 @@ export const formatStorageResult = ({
   ilmPhase: result.ilmPhase,
   markdownComments: result.markdownComments,
   ecsVersion: report.ecsVersion,
-  indexId: report.indexId,
+  indexId: report.indexId ?? '', // ---> we don't have this field when isILMAvailable is false
   error: result.error,
 });
 

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts

@@ -36,14 +36,9 @@ export const alertIndexNoResults: PatternRollup = {
   sizeInBytes: 6423408623,
   stats: {
     '.internal.alerts-security.alerts-default-000001': {
-      health: 'green',
+      name: '.internal.alerts-security.alerts-default-000001',
-      status: 'open',
+      num_docs: 25914,
-      total: {
+      size_in_bytes: 6423408623,
-        docs: {
-          count: 25914,
-          deleted: 0,
-        },
-      },
     },
   },
 };
@@ -89,14 +84,9 @@ export const alertIndexWithAllResults: PatternRollup = {
   sizeInBytes: 29717961631,
   stats: {
     '.internal.alerts-security.alerts-default-000001': {
-      health: 'green',
+      name: '.internal.alerts-security.alerts-default-000001',
-      status: 'open',
+      num_docs: 26093,
-      total: {
+      size_in_bytes: 0,
-        docs: {
-          count: 26093,
-          deleted: 0,
-        },
-      },
     },
   },
 };

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts

@@ -49,57 +49,21 @@ export const auditbeatNoResults: PatternRollup = {
   stats: {
     '.ds-auditbeat-8.6.1-2023.02.07-000001': {
       uuid: 'YpxavlUVTw2x_E_QtADrpg',
-      health: 'yellow',
+      size_in_bytes: 18791790,
-      primaries: {
+      num_docs: 19123,
-        store: {
+      name: '.ds-auditbeat-8.6.1-2023.02.07-000001',
-          size_in_bytes: 18791790,
-          total_data_set_size_in_bytes: 18791790,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 19123,
-          deleted: 0,
-        },
-      },
     },
     'auditbeat-custom-empty-index-1': {
       uuid: 'Iz5FJjsLQla34mD6kBAQBw',
-      health: 'yellow',
+      size_in_bytes: 247,
-      primaries: {
+      num_docs: 0,
-        store: {
+      name: 'auditbeat-custom-empty-index-1',
-          size_in_bytes: 247,
-          total_data_set_size_in_bytes: 247,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 0,
-          deleted: 0,
-        },
-      },
     },
     'auditbeat-custom-index-1': {
       uuid: 'xJvgb2QCQPSjlr7UnW8tFA',
-      health: 'yellow',
+      size_in_bytes: 28409,
-      primaries: {
+      num_docs: 4,
-        store: {
+      name: 'auditbeat-custom-index-1',
-          size_in_bytes: 28409,
-          total_data_set_size_in_bytes: 28409,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 4,
-          deleted: 0,
-        },
-      },
     },
   },
 };
@@ -184,57 +148,21 @@ export const auditbeatWithAllResults: PatternRollup = {
   stats: {
     '.ds-auditbeat-8.6.1-2023.02.07-000001': {
       uuid: 'YpxavlUVTw2x_E_QtADrpg',
-      health: 'yellow',
+      size_in_bytes: 18791790,
-      primaries: {
+      num_docs: 19123,
-        store: {
+      name: '.ds-auditbeat-8.6.1-2023.02.07-000001',
-          size_in_bytes: 18791790,
-          total_data_set_size_in_bytes: 18791790,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 19123,
-          deleted: 0,
-        },
-      },
     },
     'auditbeat-custom-empty-index-1': {
       uuid: 'Iz5FJjsLQla34mD6kBAQBw',
-      health: 'yellow',
+      size_in_bytes: 247,
-      primaries: {
+      num_docs: 0,
-        store: {
+      name: 'auditbeat-custom-empty-index-1',
-          size_in_bytes: 247,
-          total_data_set_size_in_bytes: 247,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 0,
-          deleted: 0,
-        },
-      },
     },
     'auditbeat-custom-index-1': {
       uuid: 'xJvgb2QCQPSjlr7UnW8tFA',
-      health: 'yellow',
+      size_in_bytes: 28409,
-      primaries: {
+      num_docs: 4,
-        store: {
+      name: 'auditbeat-custom-index-1',
-          size_in_bytes: 28409,
-          total_data_set_size_in_bytes: 28409,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 4,
-          deleted: 0,
-        },
-      },
     },
   },
 };

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts

@@ -47,39 +47,15 @@ export const packetbeatNoResults: PatternRollup = {
   stats: {
     '.ds-packetbeat-8.6.1-2023.02.04-000001': {
       uuid: 'x5Uuw4j4QM2YidHLNixCwg',
-      health: 'yellow',
+      name: '.ds-packetbeat-8.6.1-2023.02.04-000001',
-      primaries: {
+      num_docs: 1628343,
-        store: {
+      size_in_bytes: 512194751,
-          size_in_bytes: 512194751,
-          total_data_set_size_in_bytes: 512194751,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 1628343,
-          deleted: 0,
-        },
-      },
     },
     '.ds-packetbeat-8.5.3-2023.02.04-000001': {
       uuid: 'we0vNWm2Q6iz6uHubyHS6Q',
-      health: 'yellow',
+      size_in_bytes: 584326147,
-      primaries: {
+      name: '.ds-packetbeat-8.5.3-2023.02.04-000001',
-        store: {
+      num_docs: 1630289,
-          size_in_bytes: 584326147,
-          total_data_set_size_in_bytes: 584326147,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 1630289,
-          deleted: 0,
-        },
-      },
     },
   },
 };
@@ -150,37 +126,15 @@ export const packetbeatWithSomeErrors: PatternRollup = {
   stats: {
     '.ds-packetbeat-8.6.1-2023.02.04-000001': {
       uuid: 'x5Uuw4j4QM2YidHLNixCwg',
-      health: 'yellow',
+      size_in_bytes: 731583142,
-      primaries: {
+      name: '.ds-packetbeat-8.6.1-2023.02.04-000001',
-        store: {
+      num_docs: 1628343,
-          size_in_bytes: 512194751,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 1628343,
-          deleted: 0,
-        },
-      },
     },
     '.ds-packetbeat-8.5.3-2023.02.04-000001': {
       uuid: 'we0vNWm2Q6iz6uHubyHS6Q',
-      health: 'yellow',
+      size_in_bytes: 584326147,
-      primaries: {
+      name: '.ds-packetbeat-8.5.3-2023.02.04-000001',
-        store: {
+      num_docs: 1630289,
-          size_in_bytes: 584326147,
-          reserved_in_bytes: 0,
-        },
-      },
-      status: 'open',
-      total: {
-        docs: {
-          count: 1630289,
-          deleted: 0,
-        },
-      },
     },
   },
 };
@@ -244,553 +198,15 @@ export const mockPacketbeatPatternRollup: PatternRollup = {
   stats: {
     '.ds-packetbeat-8.6.1-2023.02.04-000001': {
       uuid: 'x5Uuw4j4QM2YidHLNixCwg',
-      health: 'yellow',
+      size_in_bytes: 731583142,
-      status: 'open',
+      name: '.ds-packetbeat-8.6.1-2023.02.04-000001',
-      primaries: {
+      num_docs: 1628343,
-        docs: {
-          count: 1628343,
-          deleted: 0,
-        },
-        shard_stats: {
-          total_count: 1,
-        },
-        store: {
-          size_in_bytes: 731583142,
-          total_data_set_size_in_bytes: 731583142,
-          reserved_in_bytes: 0,
-        },
-        indexing: {
-          index_total: 0,
-          index_time_in_millis: 0,
-          index_current: 0,
-          index_failed: 0,
-          delete_total: 0,
-          delete_time_in_millis: 0,
-          delete_current: 0,
-          noop_update_total: 0,
-          is_throttled: false,
-          throttle_time_in_millis: 0,
-        },
-        get: {
-          total: 0,
-          time_in_millis: 0,
-          exists_total: 0,
-          exists_time_in_millis: 0,
-          missing_total: 0,
-          missing_time_in_millis: 0,
-          current: 0,
-        },
-        search: {
-          open_contexts: 0,
-          query_total: 120726,
-          query_time_in_millis: 234865,
-          query_current: 0,
-          fetch_total: 109324,
-          fetch_time_in_millis: 500584,
-          fetch_current: 0,
-          scroll_total: 10432,
-          scroll_time_in_millis: 3874632,
-          scroll_current: 0,
-          suggest_total: 0,
-          suggest_time_in_millis: 0,
-          suggest_current: 0,
-        },
-        merges: {
-          current: 0,
-          current_docs: 0,
-          current_size_in_bytes: 0,
-          total: 0,
-          total_time_in_millis: 0,
-          total_docs: 0,
-          total_size_in_bytes: 0,
-          total_stopped_time_in_millis: 0,
-          total_throttled_time_in_millis: 0,
-          total_auto_throttle_in_bytes: 20971520,
-        },
-        refresh: {
-          total: 2,
-          total_time_in_millis: 0,
-          external_total: 2,
-          external_total_time_in_millis: 1,
-          listeners: 0,
-        },
-        flush: {
-          total: 1,
-          periodic: 1,
-          total_time_in_millis: 0,
-        },
-        warmer: {
-          current: 0,
-          total: 1,
-          total_time_in_millis: 1,
-        },
-        query_cache: {
-          memory_size_in_bytes: 8316098,
-          total_count: 34248343,
-          hit_count: 3138879,
-          miss_count: 31109464,
-          cache_size: 4585,
-          cache_count: 4585,
-          evictions: 0,
-        },
-        fielddata: {
-          memory_size_in_bytes: 12424,
-          evictions: 0,
-        },
-        completion: {
-          size_in_bytes: 0,
-        },
-        segments: {
-          count: 19,
-          memory_in_bytes: 0,
-          terms_memory_in_bytes: 0,
-          stored_fields_memory_in_bytes: 0,
-          term_vectors_memory_in_bytes: 0,
-          norms_memory_in_bytes: 0,
-          points_memory_in_bytes: 0,
-          doc_values_memory_in_bytes: 0,
-          index_writer_memory_in_bytes: 0,
-          version_map_memory_in_bytes: 0,
-          fixed_bit_set_memory_in_bytes: 304,
-          max_unsafe_auto_id_timestamp: -1,
-          file_sizes: {},
-        },
-        translog: {
-          operations: 0,
-          size_in_bytes: 55,
-          uncommitted_operations: 0,
-          uncommitted_size_in_bytes: 55,
-          earliest_last_modified_age: 606298841,
-        },
-        request_cache: {
-          memory_size_in_bytes: 89216,
-          evictions: 0,
-          hit_count: 704,
-          miss_count: 38,
-        },
-        recovery: {
-          current_as_source: 0,
-          current_as_target: 0,
-          throttle_time_in_millis: 0,
-        },
-        bulk: {
-          total_operations: 0,
-          total_time_in_millis: 0,
-          total_size_in_bytes: 0,
-          avg_time_in_millis: 0,
-          avg_size_in_bytes: 0,
-        },
-      },
-      total: {
-        docs: {
-          count: 1628343,
-          deleted: 0,
-        },
-        shard_stats: {
-          total_count: 1,
-        },
-        store: {
-          size_in_bytes: 731583142,
-          total_data_set_size_in_bytes: 731583142,
-          reserved_in_bytes: 0,
-        },
-        indexing: {
-          index_total: 0,
-          index_time_in_millis: 0,
-          index_current: 0,
-          index_failed: 0,
-          delete_total: 0,
-          delete_time_in_millis: 0,
-          delete_current: 0,
-          noop_update_total: 0,
-          is_throttled: false,
-          throttle_time_in_millis: 0,
-        },
-        get: {
-          total: 0,
-          time_in_millis: 0,
-          exists_total: 0,
-          exists_time_in_millis: 0,
-          missing_total: 0,
-          missing_time_in_millis: 0,
-          current: 0,
-        },
-        search: {
-          open_contexts: 0,
-          query_total: 120726,
-          query_time_in_millis: 234865,
-          query_current: 0,
-          fetch_total: 109324,
-          fetch_time_in_millis: 500584,
-          fetch_current: 0,
-          scroll_total: 10432,
-          scroll_time_in_millis: 3874632,
-          scroll_current: 0,
-          suggest_total: 0,
-          suggest_time_in_millis: 0,
-          suggest_current: 0,
-        },
-        merges: {
-          current: 0,
-          current_docs: 0,
-          current_size_in_bytes: 0,
-          total: 0,
-          total_time_in_millis: 0,
-          total_docs: 0,
-          total_size_in_bytes: 0,
-          total_stopped_time_in_millis: 0,
-          total_throttled_time_in_millis: 0,
-          total_auto_throttle_in_bytes: 20971520,
-        },
-        refresh: {
-          total: 2,
-          total_time_in_millis: 0,
-          external_total: 2,
-          external_total_time_in_millis: 1,
-          listeners: 0,
-        },
-        flush: {
-          total: 1,
-          periodic: 1,
-          total_time_in_millis: 0,
-        },
-        warmer: {
-          current: 0,
-          total: 1,
-          total_time_in_millis: 1,
-        },
-        query_cache: {
-          memory_size_in_bytes: 8316098,
-          total_count: 34248343,
-          hit_count: 3138879,
-          miss_count: 31109464,
-          cache_size: 4585,
-          cache_count: 4585,
-          evictions: 0,
-        },
-        fielddata: {
-          memory_size_in_bytes: 12424,
-          evictions: 0,
-        },
-        completion: {
-          size_in_bytes: 0,
-        },
-        segments: {
-          count: 19,
-          memory_in_bytes: 0,
-          terms_memory_in_bytes: 0,
-          stored_fields_memory_in_bytes: 0,
-          term_vectors_memory_in_bytes: 0,
-          norms_memory_in_bytes: 0,
-          points_memory_in_bytes: 0,
-          doc_values_memory_in_bytes: 0,
-          index_writer_memory_in_bytes: 0,
-          version_map_memory_in_bytes: 0,
-          fixed_bit_set_memory_in_bytes: 304,
-          max_unsafe_auto_id_timestamp: -1,
-          file_sizes: {},
-        },
-        translog: {
-          operations: 0,
-          size_in_bytes: 55,
-          uncommitted_operations: 0,
-          uncommitted_size_in_bytes: 55,
-          earliest_last_modified_age: 606298841,
-        },
-        request_cache: {
-          memory_size_in_bytes: 89216,
-          evictions: 0,
-          hit_count: 704,
-          miss_count: 38,
-        },
-        recovery: {
-          current_as_source: 0,
-          current_as_target: 0,
-          throttle_time_in_millis: 0,
-        },
-        bulk: {
-          total_operations: 0,
-          total_time_in_millis: 0,
-          total_size_in_bytes: 0,
-          avg_time_in_millis: 0,
-          avg_size_in_bytes: 0,
-        },
-      },
     },
     '.ds-packetbeat-8.5.3-2023.02.04-000001': {
       uuid: 'we0vNWm2Q6iz6uHubyHS6Q',
-      health: 'yellow',
+      size_in_bytes: 733175040,
-      status: 'open',
+      name: '.ds-packetbeat-8.5.3-2023.02.04-000001',
-      primaries: {
+      num_docs: 1630289,
-        docs: {
-          count: 1630289,
-          deleted: 0,
-        },
-        shard_stats: {
-          total_count: 1,
-        },
-        store: {
-          size_in_bytes: 733175040,
-          total_data_set_size_in_bytes: 733175040,
-          reserved_in_bytes: 0,
-        },
-        indexing: {
-          index_total: 0,
-          index_time_in_millis: 0,
-          index_current: 0,
-          index_failed: 0,
-          delete_total: 0,
-          delete_time_in_millis: 0,
-          delete_current: 0,
-          noop_update_total: 0,
-          is_throttled: false,
-          throttle_time_in_millis: 0,
-        },
-        get: {
-          total: 0,
-          time_in_millis: 0,
-          exists_total: 0,
-          exists_time_in_millis: 0,
-          missing_total: 0,
-          missing_time_in_millis: 0,
-          current: 0,
-        },
-        search: {
-          open_contexts: 0,
-          query_total: 120726,
-          query_time_in_millis: 248138,
-          query_current: 0,
-          fetch_total: 109484,
-          fetch_time_in_millis: 500514,
-          fetch_current: 0,
-          scroll_total: 10432,
-          scroll_time_in_millis: 3871379,
-          scroll_current: 0,
-          suggest_total: 0,
-          suggest_time_in_millis: 0,
-          suggest_current: 0,
-        },
-        merges: {
-          current: 0,
-          current_docs: 0,
-          current_size_in_bytes: 0,
-          total: 0,
-          total_time_in_millis: 0,
-          total_docs: 0,
-          total_size_in_bytes: 0,
-          total_stopped_time_in_millis: 0,
-          total_throttled_time_in_millis: 0,
-          total_auto_throttle_in_bytes: 20971520,
-        },
-        refresh: {
-          total: 2,
-          total_time_in_millis: 0,
-          external_total: 2,
-          external_total_time_in_millis: 2,
-          listeners: 0,
-        },
-        flush: {
-          total: 1,
-          periodic: 1,
-          total_time_in_millis: 0,
-        },
-        warmer: {
-          current: 0,
-          total: 1,
-          total_time_in_millis: 1,
-        },
-        query_cache: {
-          memory_size_in_bytes: 5387543,
-          total_count: 24212135,
-          hit_count: 2223357,
-          miss_count: 21988778,
-          cache_size: 3275,
-          cache_count: 3275,
-          evictions: 0,
-        },
-        fielddata: {
-          memory_size_in_bytes: 12336,
-          evictions: 0,
-        },
-        completion: {
-          size_in_bytes: 0,
-        },
-        segments: {
-          count: 20,
-          memory_in_bytes: 0,
-          terms_memory_in_bytes: 0,
-          stored_fields_memory_in_bytes: 0,
-          term_vectors_memory_in_bytes: 0,
-          norms_memory_in_bytes: 0,
-          points_memory_in_bytes: 0,
-          doc_values_memory_in_bytes: 0,
-          index_writer_memory_in_bytes: 0,
-          version_map_memory_in_bytes: 0,
-          fixed_bit_set_memory_in_bytes: 320,
-          max_unsafe_auto_id_timestamp: -1,
-          file_sizes: {},
-        },
-        translog: {
-          operations: 0,
-          size_in_bytes: 55,
-          uncommitted_operations: 0,
-          uncommitted_size_in_bytes: 55,
-          earliest_last_modified_age: 606298805,
-        },
-        request_cache: {
-          memory_size_in_bytes: 89320,
-          evictions: 0,
-          hit_count: 704,
-          miss_count: 38,
-        },
-        recovery: {
-          current_as_source: 0,
-          current_as_target: 0,
-          throttle_time_in_millis: 0,
-        },
-        bulk: {
-          total_operations: 0,
-          total_time_in_millis: 0,
-          total_size_in_bytes: 0,
-          avg_time_in_millis: 0,
-          avg_size_in_bytes: 0,
-        },
-      },
-      total: {
-        docs: {
-          count: 1630289,
-          deleted: 0,
-        },
-        shard_stats: {
-          total_count: 1,
-        },
-        store: {
-          size_in_bytes: 733175040,
-          total_data_set_size_in_bytes: 733175040,
-          reserved_in_bytes: 0,
-        },
-        indexing: {
-          index_total: 0,
-          index_time_in_millis: 0,
-          index_current: 0,
-          index_failed: 0,
-          delete_total: 0,
-          delete_time_in_millis: 0,
-          delete_current: 0,
-          noop_update_total: 0,
-          is_throttled: false,
-          throttle_time_in_millis: 0,
-        },
-        get: {
-          total: 0,
-          time_in_millis: 0,
-          exists_total: 0,
-          exists_time_in_millis: 0,
-          missing_total: 0,
-          missing_time_in_millis: 0,
-          current: 0,
-        },
-        search: {
-          open_contexts: 0,
-          query_total: 120726,
-          query_time_in_millis: 248138,
-          query_current: 0,
-          fetch_total: 109484,
-          fetch_time_in_millis: 500514,
-          fetch_current: 0,
-          scroll_total: 10432,
-          scroll_time_in_millis: 3871379,
-          scroll_current: 0,
-          suggest_total: 0,
-          suggest_time_in_millis: 0,
-          suggest_current: 0,
-        },
-        merges: {
-          current: 0,
-          current_docs: 0,
-          current_size_in_bytes: 0,
-          total: 0,
-          total_time_in_millis: 0,
-          total_docs: 0,
-          total_size_in_bytes: 0,
-          total_stopped_time_in_millis: 0,
-          total_throttled_time_in_millis: 0,
-          total_auto_throttle_in_bytes: 20971520,
-        },
-        refresh: {
-          total: 2,
-          total_time_in_millis: 0,
-          external_total: 2,
-          external_total_time_in_millis: 2,
-          listeners: 0,
-        },
-        flush: {
-          total: 1,
-          periodic: 1,
-          total_time_in_millis: 0,
-        },
-        warmer: {
-          current: 0,
-          total: 1,
-          total_time_in_millis: 1,
-        },
-        query_cache: {
-          memory_size_in_bytes: 5387543,
-          total_count: 24212135,
-          hit_count: 2223357,
-          miss_count: 21988778,
-          cache_size: 3275,
-          cache_count: 3275,
-          evictions: 0,
-        },
-        fielddata: {
-          memory_size_in_bytes: 12336,
-          evictions: 0,
-        },
-        completion: {
-          size_in_bytes: 0,
-        },
-        segments: {
-          count: 20,
-          memory_in_bytes: 0,
-          terms_memory_in_bytes: 0,
-          stored_fields_memory_in_bytes: 0,
-          term_vectors_memory_in_bytes: 0,
-          norms_memory_in_bytes: 0,
-          points_memory_in_bytes: 0,
-          doc_values_memory_in_bytes: 0,
-          index_writer_memory_in_bytes: 0,
-          version_map_memory_in_bytes: 0,
-          fixed_bit_set_memory_in_bytes: 320,
-          max_unsafe_auto_id_timestamp: -1,
-          file_sizes: {},
-        },
-        translog: {
-          operations: 0,
-          size_in_bytes: 55,
-          uncommitted_operations: 0,
-          uncommitted_size_in_bytes: 55,
-          earliest_last_modified_age: 606298805,
-        },
-        request_cache: {
-          memory_size_in_bytes: 89320,
-          evictions: 0,
-          hit_count: 704,
-          miss_count: 38,
-        },
-        recovery: {
-          current_as_source: 0,
-          current_as_target: 0,
-          throttle_time_in_millis: 0,
-        },
-        bulk: {
-          total_operations: 0,
-          total_time_in_millis: 0,
-          total_size_in_bytes: 0,
-          avg_time_in_millis: 0,
-          avg_size_in_bytes: 0,
-        },
-      },
     },
   },
 };

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats.tsx

@@ -5,832 +5,25 @@
  * 2.0.
  */
 
-import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
+import { MeteringStatsIndex } from '../../types';
 
-export const mockStats: Record<string, IndicesStatsIndicesStats> = {
+export const mockStats: Record<string, MeteringStatsIndex> = {
   '.ds-packetbeat-8.6.1-2023.02.04-000001': {
     uuid: 'x5Uuw4j4QM2YidHLNixCwg',
-    health: 'yellow',
+    num_docs: 1628343,
-    status: 'open',
+    size_in_bytes: 731583142,
-    primaries: {
+    name: '.ds-packetbeat-8.6.1-2023.02.04-000001',
-      docs: {
-        count: 1628343,
-        deleted: 0,
-      },
-      shard_stats: {
-        total_count: 1,
-      },
-      store: {
-        size_in_bytes: 731583142,
-        total_data_set_size_in_bytes: 731583142,
-        reserved_in_bytes: 0,
-      },
-      indexing: {
-        index_total: 0,
-        index_time_in_millis: 0,
-        index_current: 0,
-        index_failed: 0,
-        delete_total: 0,
-        delete_time_in_millis: 0,
-        delete_current: 0,
-        noop_update_total: 0,
-        is_throttled: false,
-        throttle_time_in_millis: 0,
-      },
-      get: {
-        total: 0,
-        time_in_millis: 0,
-        exists_total: 0,
-        exists_time_in_millis: 0,
-        missing_total: 0,
-        missing_time_in_millis: 0,
-        current: 0,
-      },
-      search: {
-        open_contexts: 0,
-        query_total: 32,
-        query_time_in_millis: 111,
-        query_current: 0,
-        fetch_total: 32,
-        fetch_time_in_millis: 0,
-        fetch_current: 0,
-        scroll_total: 0,
-        scroll_time_in_millis: 0,
-        scroll_current: 0,
-        suggest_total: 0,
-        suggest_time_in_millis: 0,
-        suggest_current: 0,
-      },
-      merges: {
-        current: 0,
-        current_docs: 0,
-        current_size_in_bytes: 0,
-        total: 0,
-        total_time_in_millis: 0,
-        total_docs: 0,
-        total_size_in_bytes: 0,
-        total_stopped_time_in_millis: 0,
-        total_throttled_time_in_millis: 0,
-        total_auto_throttle_in_bytes: 20971520,
-      },
-      refresh: {
-        total: 2,
-        total_time_in_millis: 0,
-        external_total: 2,
-        external_total_time_in_millis: 15,
-        listeners: 0,
-      },
-      flush: {
-        total: 1,
-        periodic: 1,
-        total_time_in_millis: 0,
-      },
-      warmer: {
-        current: 0,
-        total: 1,
-        total_time_in_millis: 15,
-      },
-      query_cache: {
-        memory_size_in_bytes: 0,
-        total_count: 301,
-        hit_count: 0,
-        miss_count: 301,
-        cache_size: 0,
-        cache_count: 0,
-        evictions: 0,
-      },
-      fielddata: {
-        memory_size_in_bytes: 1080,
-        evictions: 0,
-      },
-      completion: {
-        size_in_bytes: 0,
-      },
-      segments: {
-        count: 19,
-        memory_in_bytes: 0,
-        terms_memory_in_bytes: 0,
-        stored_fields_memory_in_bytes: 0,
-        term_vectors_memory_in_bytes: 0,
-        norms_memory_in_bytes: 0,
-        points_memory_in_bytes: 0,
-        doc_values_memory_in_bytes: 0,
-        index_writer_memory_in_bytes: 0,
-        version_map_memory_in_bytes: 0,
-        fixed_bit_set_memory_in_bytes: 304,
-        max_unsafe_auto_id_timestamp: -1,
-        file_sizes: {},
-      },
-      translog: {
-        operations: 0,
-        size_in_bytes: 55,
-        uncommitted_operations: 0,
-        uncommitted_size_in_bytes: 55,
-        earliest_last_modified_age: 136482466,
-      },
-      request_cache: {
-        memory_size_in_bytes: 3680,
-        evictions: 0,
-        hit_count: 28,
-        miss_count: 4,
-      },
-      recovery: {
-        current_as_source: 0,
-        current_as_target: 0,
-        throttle_time_in_millis: 0,
-      },
-      bulk: {
-        total_operations: 0,
-        total_time_in_millis: 0,
-        total_size_in_bytes: 0,
-        avg_time_in_millis: 0,
-        avg_size_in_bytes: 0,
-      },
-    },
-    total: {
-      docs: {
-        count: 1628343,
-        deleted: 0,
-      },
-      shard_stats: {
-        total_count: 1,
-      },
-      store: {
-        size_in_bytes: 731583142,
-        total_data_set_size_in_bytes: 731583142,
-        reserved_in_bytes: 0,
-      },
-      indexing: {
-        index_total: 0,
-        index_time_in_millis: 0,
-        index_current: 0,
-        index_failed: 0,
-        delete_total: 0,
-        delete_time_in_millis: 0,
-        delete_current: 0,
-        noop_update_total: 0,
-        is_throttled: false,
-        throttle_time_in_millis: 0,
-      },
-      get: {
-        total: 0,
-        time_in_millis: 0,
-        exists_total: 0,
-        exists_time_in_millis: 0,
-        missing_total: 0,
-        missing_time_in_millis: 0,
-        current: 0,
-      },
-      search: {
-        open_contexts: 0,
-        query_total: 32,
-        query_time_in_millis: 111,
-        query_current: 0,
-        fetch_total: 32,
-        fetch_time_in_millis: 0,
-        fetch_current: 0,
-        scroll_total: 0,
-        scroll_time_in_millis: 0,
-        scroll_current: 0,
-        suggest_total: 0,
-        suggest_time_in_millis: 0,
-        suggest_current: 0,
-      },
-      merges: {
-        current: 0,
-        current_docs: 0,
-        current_size_in_bytes: 0,
-        total: 0,
-        total_time_in_millis: 0,
-        total_docs: 0,
-        total_size_in_bytes: 0,
-        total_stopped_time_in_millis: 0,
-        total_throttled_time_in_millis: 0,
-        total_auto_throttle_in_bytes: 20971520,
-      },
-      refresh: {
-        total: 2,
-        total_time_in_millis: 0,
-        external_total: 2,
-        external_total_time_in_millis: 15,
-        listeners: 0,
-      },
-      flush: {
-        total: 1,
-        periodic: 1,
-        total_time_in_millis: 0,
-      },
-      warmer: {
-        current: 0,
-        total: 1,
-        total_time_in_millis: 15,
-      },
-      query_cache: {
-        memory_size_in_bytes: 0,
-        total_count: 301,
-        hit_count: 0,
-        miss_count: 301,
-        cache_size: 0,
-        cache_count: 0,
-        evictions: 0,
-      },
-      fielddata: {
-        memory_size_in_bytes: 1080,
-        evictions: 0,
-      },
-      completion: {
-        size_in_bytes: 0,
-      },
-      segments: {
-        count: 19,
-        memory_in_bytes: 0,
-        terms_memory_in_bytes: 0,
-        stored_fields_memory_in_bytes: 0,
-        term_vectors_memory_in_bytes: 0,
-        norms_memory_in_bytes: 0,
-        points_memory_in_bytes: 0,
-        doc_values_memory_in_bytes: 0,
-        index_writer_memory_in_bytes: 0,
-        version_map_memory_in_bytes: 0,
-        fixed_bit_set_memory_in_bytes: 304,
-        max_unsafe_auto_id_timestamp: -1,
-        file_sizes: {},
-      },
-      translog: {
-        operations: 0,
-        size_in_bytes: 55,
-        uncommitted_operations: 0,
-        uncommitted_size_in_bytes: 55,
-        earliest_last_modified_age: 136482466,
-      },
-      request_cache: {
-        memory_size_in_bytes: 3680,
-        evictions: 0,
-        hit_count: 28,
-        miss_count: 4,
-      },
-      recovery: {
-        current_as_source: 0,
-        current_as_target: 0,
-        throttle_time_in_millis: 0,
-      },
-      bulk: {
-        total_operations: 0,
-        total_time_in_millis: 0,
-        total_size_in_bytes: 0,
-        avg_time_in_millis: 0,
-        avg_size_in_bytes: 0,
-      },
-    },
   },
   '.ds-packetbeat-8.5.3-2023.02.04-000001': {
     uuid: 'we0vNWm2Q6iz6uHubyHS6Q',
-    health: 'yellow',
+    num_docs: 1630289,
-    status: 'open',
+    size_in_bytes: 733175040,
-    primaries: {
+    name: '.ds-packetbeat-8.5.3-2023.02.04-000001',
-      docs: {
-        count: 1630289,
-        deleted: 0,
-      },
-      shard_stats: {
-        total_count: 1,
-      },
-      store: {
-        size_in_bytes: 733175040,
-        total_data_set_size_in_bytes: 733175040,
-        reserved_in_bytes: 0,
-      },
-      indexing: {
-        index_total: 0,
-        index_time_in_millis: 0,
-        index_current: 0,
-        index_failed: 0,
-        delete_total: 0,
-        delete_time_in_millis: 0,
-        delete_current: 0,
-        noop_update_total: 0,
-        is_throttled: false,
-        throttle_time_in_millis: 0,
-      },
-      get: {
-        total: 0,
-        time_in_millis: 0,
-        exists_total: 0,
-        exists_time_in_millis: 0,
-        missing_total: 0,
-        missing_time_in_millis: 0,
-        current: 0,
-      },
-      search: {
-        open_contexts: 0,
-        query_total: 32,
-        query_time_in_millis: 111,
-        query_current: 0,
-        fetch_total: 32,
-        fetch_time_in_millis: 0,
-        fetch_current: 0,
-        scroll_total: 0,
-        scroll_time_in_millis: 0,
-        scroll_current: 0,
-        suggest_total: 0,
-        suggest_time_in_millis: 0,
-        suggest_current: 0,
-      },
-      merges: {
-        current: 0,
-        current_docs: 0,
-        current_size_in_bytes: 0,
-        total: 0,
-        total_time_in_millis: 0,
-        total_docs: 0,
-        total_size_in_bytes: 0,
-        total_stopped_time_in_millis: 0,
-        total_throttled_time_in_millis: 0,
-        total_auto_throttle_in_bytes: 20971520,
-      },
-      refresh: {
-        total: 2,
-        total_time_in_millis: 0,
-        external_total: 2,
-        external_total_time_in_millis: 2,
-        listeners: 0,
-      },
-      flush: {
-        total: 1,
-        periodic: 1,
-        total_time_in_millis: 0,
-      },
-      warmer: {
-        current: 0,
-        total: 1,
-        total_time_in_millis: 2,
-      },
-      query_cache: {
-        memory_size_in_bytes: 0,
-        total_count: 203,
-        hit_count: 0,
-        miss_count: 203,
-        cache_size: 0,
-        cache_count: 0,
-        evictions: 0,
-      },
-      fielddata: {
-        memory_size_in_bytes: 1168,
-        evictions: 0,
-      },
-      completion: {
-        size_in_bytes: 0,
-      },
-      segments: {
-        count: 20,
-        memory_in_bytes: 0,
-        terms_memory_in_bytes: 0,
-        stored_fields_memory_in_bytes: 0,
-        term_vectors_memory_in_bytes: 0,
-        norms_memory_in_bytes: 0,
-        points_memory_in_bytes: 0,
-        doc_values_memory_in_bytes: 0,
-        index_writer_memory_in_bytes: 0,
-        version_map_memory_in_bytes: 0,
-        fixed_bit_set_memory_in_bytes: 320,
-        max_unsafe_auto_id_timestamp: -1,
-        file_sizes: {},
-      },
-      translog: {
-        operations: 0,
-        size_in_bytes: 55,
-        uncommitted_operations: 0,
-        uncommitted_size_in_bytes: 55,
-        earliest_last_modified_age: 136482425,
-      },
-      request_cache: {
-        memory_size_in_bytes: 3688,
-        evictions: 0,
-        hit_count: 28,
-        miss_count: 4,
-      },
-      recovery: {
-        current_as_source: 0,
-        current_as_target: 0,
-        throttle_time_in_millis: 0,
-      },
-      bulk: {
-        total_operations: 0,
-        total_time_in_millis: 0,
-        total_size_in_bytes: 0,
-        avg_time_in_millis: 0,
-        avg_size_in_bytes: 0,
-      },
-    },
-    total: {
-      docs: {
-        count: 1630289,
-        deleted: 0,
-      },
-      shard_stats: {
-        total_count: 1,
-      },
-      store: {
-        size_in_bytes: 733175040,
-        total_data_set_size_in_bytes: 733175040,
-        reserved_in_bytes: 0,
-      },
-      indexing: {
-        index_total: 0,
-        index_time_in_millis: 0,
-        index_current: 0,
-        index_failed: 0,
-        delete_total: 0,
-        delete_time_in_millis: 0,
-        delete_current: 0,
-        noop_update_total: 0,
-        is_throttled: false,
-        throttle_time_in_millis: 0,
-      },
-      get: {
-        total: 0,
-        time_in_millis: 0,
-        exists_total: 0,
-        exists_time_in_millis: 0,
-        missing_total: 0,
-        missing_time_in_millis: 0,
-        current: 0,
-      },
-      search: {
-        open_contexts: 0,
-        query_total: 32,
-        query_time_in_millis: 111,
-        query_current: 0,
-        fetch_total: 32,
-        fetch_time_in_millis: 0,
-        fetch_current: 0,
-        scroll_total: 0,
-        scroll_time_in_millis: 0,
-        scroll_current: 0,
-        suggest_total: 0,
-        suggest_time_in_millis: 0,
-        suggest_current: 0,
-      },
-      merges: {
-        current: 0,
-        current_docs: 0,
-        current_size_in_bytes: 0,
-        total: 0,
-        total_time_in_millis: 0,
-        total_docs: 0,
-        total_size_in_bytes: 0,
-        total_stopped_time_in_millis: 0,
-        total_throttled_time_in_millis: 0,
-        total_auto_throttle_in_bytes: 20971520,
-      },
-      refresh: {
-        total: 2,
-        total_time_in_millis: 0,
-        external_total: 2,
-        external_total_time_in_millis: 2,
-        listeners: 0,
-      },
-      flush: {
-        total: 1,
-        periodic: 1,
-        total_time_in_millis: 0,
-      },
-      warmer: {
-        current: 0,
-        total: 1,
-        total_time_in_millis: 2,
-      },
-      query_cache: {
-        memory_size_in_bytes: 0,
-        total_count: 203,
-        hit_count: 0,
-        miss_count: 203,
-        cache_size: 0,
-        cache_count: 0,
-        evictions: 0,
-      },
-      fielddata: {
-        memory_size_in_bytes: 1168,
-        evictions: 0,
-      },
-      completion: {
-        size_in_bytes: 0,
-      },
-      segments: {
-        count: 20,
-        memory_in_bytes: 0,
-        terms_memory_in_bytes: 0,
-        stored_fields_memory_in_bytes: 0,
-        term_vectors_memory_in_bytes: 0,
-        norms_memory_in_bytes: 0,
-        points_memory_in_bytes: 0,
-        doc_values_memory_in_bytes: 0,
-        index_writer_memory_in_bytes: 0,
-        version_map_memory_in_bytes: 0,
-        fixed_bit_set_memory_in_bytes: 320,
-        max_unsafe_auto_id_timestamp: -1,
-        file_sizes: {},
-      },
-      translog: {
-        operations: 0,
-        size_in_bytes: 55,
-        uncommitted_operations: 0,
-        uncommitted_size_in_bytes: 55,
-        earliest_last_modified_age: 136482425,
-      },
-      request_cache: {
-        memory_size_in_bytes: 3688,
-        evictions: 0,
-        hit_count: 28,
-        miss_count: 4,
-      },
-      recovery: {
-        current_as_source: 0,
-        current_as_target: 0,
-        throttle_time_in_millis: 0,
-      },
-      bulk: {
-        total_operations: 0,
-        total_time_in_millis: 0,
-        total_size_in_bytes: 0,
-        avg_time_in_millis: 0,
-        avg_size_in_bytes: 0,
-      },
-    },
   },
   'auditbeat-custom-index-1': {
     uuid: 'uyJDDqGrRQqdBTN0mCF-iw',
-    health: 'yellow',
+    num_docs: 4,
-    status: 'open',
+    size_in_bytes: 28413,
-    primaries: {
+    name: 'auditbeat-custom-index-1',
-      docs: {
-        count: 4,
-        deleted: 0,
-      },
-      shard_stats: {
-        total_count: 1,
-      },
-      store: {
-        size_in_bytes: 28413,
-        total_data_set_size_in_bytes: 28413,
-        reserved_in_bytes: 0,
-      },
-      indexing: {
-        index_total: 0,
-        index_time_in_millis: 0,
-        index_current: 0,
-        index_failed: 0,
-        delete_total: 0,
-        delete_time_in_millis: 0,
-        delete_current: 0,
-        noop_update_total: 0,
-        is_throttled: false,
-        throttle_time_in_millis: 0,
-      },
-      get: {
-        total: 0,
-        time_in_millis: 0,
-        exists_total: 0,
-        exists_time_in_millis: 0,
-        missing_total: 0,
-        missing_time_in_millis: 0,
-        current: 0,
-      },
-      search: {
-        open_contexts: 0,
-        query_total: 24,
-        query_time_in_millis: 5,
-        query_current: 0,
-        fetch_total: 24,
-        fetch_time_in_millis: 0,
-        fetch_current: 0,
-        scroll_total: 0,
-        scroll_time_in_millis: 0,
-        scroll_current: 0,
-        suggest_total: 0,
-        suggest_time_in_millis: 0,
-        suggest_current: 0,
-      },
-      merges: {
-        current: 0,
-        current_docs: 0,
-        current_size_in_bytes: 0,
-        total: 0,
-        total_time_in_millis: 0,
-        total_docs: 0,
-        total_size_in_bytes: 0,
-        total_stopped_time_in_millis: 0,
-        total_throttled_time_in_millis: 0,
-        total_auto_throttle_in_bytes: 20971520,
-      },
-      refresh: {
-        total: 2,
-        total_time_in_millis: 0,
-        external_total: 2,
-        external_total_time_in_millis: 0,
-        listeners: 0,
-      },
-      flush: {
-        total: 1,
-        periodic: 1,
-        total_time_in_millis: 0,
-      },
-      warmer: {
-        current: 0,
-        total: 1,
-        total_time_in_millis: 0,
-      },
-      query_cache: {
-        memory_size_in_bytes: 58,
-        total_count: 0,
-        hit_count: 0,
-        miss_count: 0,
-        cache_size: 0,
-        cache_count: 0,
-        evictions: 0,
-      },
-      fielddata: {
-        memory_size_in_bytes: 608,
-        evictions: 0,
-      },
-      completion: {
-        size_in_bytes: 0,
-      },
-      segments: {
-        count: 4,
-        memory_in_bytes: 0,
-        terms_memory_in_bytes: 0,
-        stored_fields_memory_in_bytes: 0,
-        term_vectors_memory_in_bytes: 0,
-        norms_memory_in_bytes: 0,
-        points_memory_in_bytes: 0,
-        doc_values_memory_in_bytes: 0,
-        index_writer_memory_in_bytes: 0,
-        version_map_memory_in_bytes: 0,
-        fixed_bit_set_memory_in_bytes: 0,
-        max_unsafe_auto_id_timestamp: -1,
-        file_sizes: {},
-      },
-      translog: {
-        operations: 0,
-        size_in_bytes: 55,
-        uncommitted_operations: 0,
-        uncommitted_size_in_bytes: 55,
-        earliest_last_modified_age: 79289897,
-      },
-      request_cache: {
-        memory_size_in_bytes: 3760,
-        evictions: 0,
-        hit_count: 20,
-        miss_count: 4,
-      },
-      recovery: {
-        current_as_source: 0,
-        current_as_target: 0,
-        throttle_time_in_millis: 0,
-      },
-      bulk: {
-        total_operations: 0,
-        total_time_in_millis: 0,
-        total_size_in_bytes: 0,
-        avg_time_in_millis: 0,
-        avg_size_in_bytes: 0,
-      },
-    },
-    total: {
-      docs: {
-        count: 4,
-        deleted: 0,
-      },
-      shard_stats: {
-        total_count: 1,
-      },
-      store: {
-        size_in_bytes: 28413,
-        total_data_set_size_in_bytes: 28413,
-        reserved_in_bytes: 0,
-      },
-      indexing: {
-        index_total: 0,
-        index_time_in_millis: 0,
-        index_current: 0,
-        index_failed: 0,
-        delete_total: 0,
-        delete_time_in_millis: 0,
-        delete_current: 0,
-        noop_update_total: 0,
-        is_throttled: false,
-        throttle_time_in_millis: 0,
-      },
-      get: {
-        total: 0,
-        time_in_millis: 0,
-        exists_total: 0,
-        exists_time_in_millis: 0,
-        missing_total: 0,
-        missing_time_in_millis: 0,
-        current: 0,
-      },
-      search: {
-        open_contexts: 0,
-        query_total: 24,
-        query_time_in_millis: 5,
-        query_current: 0,
-        fetch_total: 24,
-        fetch_time_in_millis: 0,
-        fetch_current: 0,
-        scroll_total: 0,
-        scroll_time_in_millis: 0,
-        scroll_current: 0,
-        suggest_total: 0,
-        suggest_time_in_millis: 0,
-        suggest_current: 0,
-      },
-      merges: {
-        current: 0,
-        current_docs: 0,
-        current_size_in_bytes: 0,
-        total: 0,
-        total_time_in_millis: 0,
-        total_docs: 0,
-        total_size_in_bytes: 0,
-        total_stopped_time_in_millis: 0,
-        total_throttled_time_in_millis: 0,
-        total_auto_throttle_in_bytes: 20971520,
-      },
-      refresh: {
-        total: 2,
-        total_time_in_millis: 0,
-        external_total: 2,
-        external_total_time_in_millis: 0,
-        listeners: 0,
-      },
-      flush: {
-        total: 1,
-        periodic: 1,
-        total_time_in_millis: 0,
-      },
-      warmer: {
-        current: 0,
-        total: 1,
-        total_time_in_millis: 0,
-      },
-      query_cache: {
-        memory_size_in_bytes: 58,
-        total_count: 0,
-        hit_count: 0,
-        miss_count: 0,
-        cache_size: 0,
-        cache_count: 0,
-        evictions: 0,
-      },
-      fielddata: {
-        memory_size_in_bytes: 608,
-        evictions: 0,
-      },
-      completion: {
-        size_in_bytes: 0,
-      },
-      segments: {
-        count: 4,
-        memory_in_bytes: 0,
-        terms_memory_in_bytes: 0,
-        stored_fields_memory_in_bytes: 0,
-        term_vectors_memory_in_bytes: 0,
-        norms_memory_in_bytes: 0,
-        points_memory_in_bytes: 0,
-        doc_values_memory_in_bytes: 0,
-        index_writer_memory_in_bytes: 0,
-        version_map_memory_in_bytes: 0,
-        fixed_bit_set_memory_in_bytes: 0,
-        max_unsafe_auto_id_timestamp: -1,
-        file_sizes: {},
-      },
-      translog: {
-        operations: 0,
-        size_in_bytes: 55,
-        uncommitted_operations: 0,
-        uncommitted_size_in_bytes: 55,
-        earliest_last_modified_age: 79289897,
-      },
-      request_cache: {
-        memory_size_in_bytes: 3760,
-        evictions: 0,
-        hit_count: 20,
-        miss_count: 4,
-      },
-      recovery: {
-        current_as_source: 0,
-        current_as_target: 0,
-        throttle_time_in_millis: 0,
-      },
-      bulk: {
-        total_operations: 0,
-        total_time_in_millis: 0,
-        total_size_in_bytes: 0,
-        avg_time_in_millis: 0,
-        avg_size_in_bytes: 0,
-      },
-    },
   },
 };

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_auditbeat_index.tsx

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MeteringStatsIndex } from '../../types';
+
+export const mockStatsPacketbeatIndex: Record<string, MeteringStatsIndex> = {
+  '.ds-packetbeat-8.6.1-2023.02.04-000001': {
+    uuid: 'x5Uuw4j4QM2YidHLNixCwg',
+    num_docs: 1628343,
+    size_in_bytes: 731583142,
+    name: '.ds-packetbeat-8.6.1-2023.02.04-000001',
+  },
+  '.ds-packetbeat-8.5.3-2023.02.04-000001': {
+    uuid: 'we0vNWm2Q6iz6uHubyHS6Q',
+    num_docs: 1630289,
+    size_in_bytes: 733175040,
+    name: '.ds-packetbeat-8.5.3-2023.02.04-000001',
+  },
+};

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_packetbeat_index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MeteringStatsIndex } from '../../types';
+
+export const mockStatsAuditbeatIndex: Record<string, MeteringStatsIndex> = {
+  'auditbeat-custom-index-1': {
+    uuid: 'jRlr6H_jSAysOLZ6KynoCQ',
+    size_in_bytes: 28425,
+    name: 'auditbeat-custom-index-1',
+    num_docs: 4,
+  },
+};

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts

@@ -8,7 +8,6 @@
 import type {
   IlmExplainLifecycleLifecycleExplain,
   IndicesGetMappingIndexMappingRecord,
-  IndicesStatsIndicesStats,
 } from '@elastic/elasticsearch/lib/api/types';
 import type { Direction } from '@elastic/eui';
 
@@ -119,7 +118,7 @@ export interface PatternRollup {
   pattern: string;
   results: Record<string, DataQualityCheckResult> | undefined;
   sizeInBytes: number | undefined;
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
 }
 
 export interface CheckIndexRequest {
@@ -175,10 +174,32 @@ export interface SelectedIndex {
   pattern: string;
 }
 
+export interface MeteringStatsIndex {
+  uuid?: string;
+  name: string;
+  num_docs: number | null;
+  size_in_bytes: number | null;
+  data_stream?: string;
+}
+
+export interface MeteringIndicesStatsResponse {
+  _shards: {
+    total: number;
+    successful: number;
+    failed: number;
+  };
+  indices: MeteringStatsIndex[];
+  datastreams: Array<{ name: string; num_docs: number; size_in_bytes: number }>;
+  total: {
+    num_docs: number;
+    size_in_bytes: number;
+  };
+}
+
 export type DataQualityIndexCheckedParams = DataQualityCheckAllCompletedParams & {
   errorCount?: number;
   ilmPhase?: string;
-  indexId: string;
+  indexId?: string;
   indexName: string;
   sameFamilyFields?: string[];
   unallowedMappingFields?: string[];

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts

@@ -21,9 +21,8 @@ import {
   packetbeatNoResults,
   packetbeatWithSomeErrors,
 } from '../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
-import { DataQualityCheckResult, PatternRollup } from '../types';
+import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../types';
 import { EMPTY_STAT } from '../helpers';
-import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
 import { mockPartitionedFieldMetadata } from '../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
 import { alertIndexWithAllResults } from '../mock/pattern_rollup/mock_alerts_pattern_rollup';
 import { EcsVersion } from '@elastic/ecs';
@@ -168,14 +167,14 @@ describe('helpers', () => {
   });
 
   describe('updateResultOnCheckCompleted', () => {
-    const packetbeatStats861: IndicesStatsIndicesStats =
+    const packetbeatStats861: MeteringStatsIndex =
       mockPacketbeatPatternRollup.stats != null
         ? mockPacketbeatPatternRollup.stats['.ds-packetbeat-8.6.1-2023.02.04-000001']
-        : {};
+        : ({} as MeteringStatsIndex);
-    const packetbeatStats853: IndicesStatsIndicesStats =
+    const packetbeatStats853: MeteringStatsIndex =
       mockPacketbeatPatternRollup.stats != null
         ? mockPacketbeatPatternRollup.stats['.ds-packetbeat-8.5.3-2023.02.04-000001']
-        : {};
+        : ({} as MeteringStatsIndex);
 
     test('it returns the updated rollups', () => {
       expect(

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx

@@ -9,7 +9,7 @@ import { renderHook } from '@testing-library/react-hooks';
 import React from 'react';
 
 import { DataQualityProvider } from '../data_quality_panel/data_quality_context';
-import { mockStatsGreenIndex } from '../mock/stats/mock_stats_green_index';
+import { mockStatsAuditbeatIndex } from '../mock/stats/mock_stats_packetbeat_index';
 import { ERROR_LOADING_STATS } from '../translations';
 import { useStats, UseStats } from '.';
 import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
@@ -65,7 +65,7 @@ describe('useStats', () => {
     };
 
     beforeEach(async () => {
-      mockHttpFetch.mockResolvedValue(mockStatsGreenIndex);
+      mockHttpFetch.mockResolvedValue(mockStatsAuditbeatIndex);
 
       const { waitForNextUpdate } = renderHook(() => useStats({ pattern, startDate, endDate }), {
         wrapper: ContextWrapperILMNotAvailable,
@@ -81,7 +81,7 @@ describe('useStats', () => {
     let statsResult: UseStats | undefined;
 
     beforeEach(async () => {
-      mockHttpFetch.mockResolvedValue(mockStatsGreenIndex);
+      mockHttpFetch.mockResolvedValue(mockStatsAuditbeatIndex);
 
       const { result, waitForNextUpdate } = renderHook(() => useStats(params), {
         wrapper: ContextWrapper,
@@ -91,7 +91,7 @@ describe('useStats', () => {
     });
 
     test('it returns the expected stats', async () => {
-      expect(statsResult?.stats).toEqual(mockStatsGreenIndex);
+      expect(statsResult?.stats).toEqual(mockStatsAuditbeatIndex);
     });
 
     test('it returns loading: false, because the data has loaded', async () => {

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx

@@ -5,18 +5,18 @@
  * 2.0.
  */
 
-import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
 import { useEffect, useState } from 'react';
 import { HttpFetchQuery } from '@kbn/core/public';
 
 import { useDataQualityContext } from '../data_quality_panel/data_quality_context';
 import * as i18n from '../translations';
 import { INTERNAL_API_VERSION } from '../helpers';
+import { MeteringStatsIndex } from '../types';
 
 const STATS_ENDPOINT = '/internal/ecs_data_quality_dashboard/stats';
 
 export interface UseStats {
-  stats: Record<string, IndicesStatsIndicesStats> | null;
+  stats: Record<string, MeteringStatsIndex> | null;
   error: string | null;
   loading: boolean;
 }
@@ -31,7 +31,7 @@ export const useStats = ({
   startDate?: string | null;
 }): UseStats => {
   const { httpFetch, isILMAvailable } = useDataQualityContext();
-  const [stats, setStats] = useState<Record<string, IndicesStatsIndicesStats> | null>(null);
+  const [stats, setStats] = useState<Record<string, MeteringStatsIndex> | null>(null);
   const [error, setError] = useState<string | null>(null);
   const [loading, setLoading] = useState<boolean>(true);
 
@@ -51,7 +51,7 @@ export const useStats = ({
           }
         }
 
-        const response = await httpFetch<Record<string, IndicesStatsIndicesStats>>(
+        const response = await httpFetch<Record<string, MeteringStatsIndex>>(
           `${STATS_ENDPOINT}/${encodedIndexName}`,
           {
             version: INTERNAL_API_VERSION,

x-pack/plugins/ecs_data_quality_dashboard/common/types.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+export interface MeteringStatsIndex {
+  uuid?: string;
+  name: string;
+  num_docs: number | null;
+  size_in_bytes: number | null;
+  data_stream?: string;
+}
+
+export interface MeteringIndicesStatsResponse {
+  _shards: {
+    total: number;
+    successful: number;
+    failed: number;
+  };
+  indices: MeteringStatsIndex[] | null | undefined;
+  datastreams: Array<{ name: string; num_docs: number }>;
+  total: {
+    num_docs: number;
+  };
+}

x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_metering_stats_index.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const mockMeteringStatsIndex = {
+  _shards: {
+    total: 4,
+    successful: 2,
+    failed: 0,
+  },
+  indices: [
+    {
+      name: '.ds-my-datastream-03-02-2024-00001',
+      num_docs: 3,
+      size_in_bytes: 15785,
+      datastream: 'my-datastream',
+    },
+    {
+      name: 'my-index-000001',
+      num_docs: 2,
+      size_in_bytes: 11462,
+    },
+  ],
+  datastreams: [
+    {
+      name: 'my-datastream',
+      num_docs: 6,
+      size_in_bytes: 31752,
+    },
+  ],
+  total: {
+    num_docs: 8,
+    size_in_bytes: 47214,
+  },
+};

x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_green_index.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
+import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
 
 /**
  * In a deployment where indices have a `green` health status, because there

x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_yellow_index.ts

@@ -5,12 +5,11 @@
  * 2.0.
  */
 
-import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
+import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
 
 /**
  * In a deployment where indices have a `yellow` health status, the
- * [`_stats`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html)
+	@@ -14,555 +14,17 @@ import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
- * API returns, (for an arbitrary index), results where the index's
  * `primaries.docs.count` and `total.docs.count` have the same value, per this
  * mock `_stats` API output
  */

x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts

@@ -5,8 +5,12 @@
  * 2.0.
  */
 
-import type { IndicesStatsResponse } from '@elastic/elasticsearch/lib/api/types';
+import type {
+  IndicesStatsIndicesStats,
+  IndicesStatsResponse,
+} from '@elastic/elasticsearch/lib/api/types';
 import type { IScopedClusterClient } from '@kbn/core/server';
+import type { MeteringIndicesStatsResponse, MeteringStatsIndex } from '../../common/types';
 
 export const fetchStats = (
   client: IScopedClusterClient,
@@ -16,3 +20,54 @@ export const fetchStats = (
     expand_wildcards: ['open'],
     index: indexPattern,
   });
+
+export const parseIndicesStats = (
+  statsIndices: Record<string, IndicesStatsIndicesStats> | undefined
+) =>
+  Object.entries(statsIndices ?? {}).reduce<Record<string, MeteringStatsIndex>>(
+    (acc, [key, value]) => {
+      acc[key] = {
+        uuid: value.uuid,
+        name: key,
+        num_docs: value?.primaries?.docs?.count ?? null,
+        size_in_bytes: value?.primaries?.store?.size_in_bytes ?? null,
+      };
+      return acc;
+    },
+    {}
+  );
+
+export const fetchMeteringStats = (
+  client: IScopedClusterClient,
+  indexPattern: string,
+  secondaryAuthorization?: string | string[] | undefined
+): Promise<MeteringIndicesStatsResponse> =>
+  client.asInternalUser.transport.request(
+    {
+      method: 'GET',
+      path: `/_metering/stats/${indexPattern}`,
+    },
+    { headers: { 'es-secondary-authorization': secondaryAuthorization } }
+  );
+
+export const parseMeteringStats = (meteringStatsIndices: MeteringStatsIndex[]) =>
+  meteringStatsIndices.reduce<Record<string, MeteringStatsIndex>>((acc, curr) => {
+    acc[curr.name] = curr;
+    return acc;
+  }, {});
+
+export const pickAvailableMeteringStats = (
+  indicesBuckets: Array<{ key: string }>,
+  meteringStatsIndices: Record<string, MeteringStatsIndex>
+) =>
+  indicesBuckets.reduce((acc: Record<string, MeteringStatsIndex>, { key }: { key: string }) => {
+    if (meteringStatsIndices?.[key]) {
+      acc[key] = {
+        name: meteringStatsIndices?.[key].name,
+        num_docs: meteringStatsIndices?.[key].num_docs,
+        size_in_bytes: null, // We don't have size_in_bytes intentionally when ILM is not available
+        data_stream: meteringStatsIndices?.[key].data_stream,
+      };
+    }
+    return acc;
+  }, {});

x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts

@@ -6,7 +6,7 @@
  */
 import { GET_INDEX_STATS } from '../../common/constants';
 
-import { fetchAvailableIndices, fetchStats } from '../lib';
+import { fetchAvailableIndices, fetchMeteringStats, fetchStats } from '../lib';
 
 import { serverMock } from '../__mocks__/server';
 import { requestMock } from '../__mocks__/request';
@@ -14,11 +14,19 @@ import { requestContextMock } from '../__mocks__/request_context';
 import { getIndexStatsRoute } from './get_index_stats';
 import type { MockedLogger } from '@kbn/logging-mocks';
 import { loggerMock } from '@kbn/logging-mocks';
+import { mockStatsGreenIndex } from '../__mocks__/mock_stats_green_index';
+import { mockStatsYellowIndex } from '../__mocks__/mock_stats_yellow_index';
+import { mockMeteringStatsIndex } from '../__mocks__/mock_metering_stats_index';
 
-jest.mock('../lib', () => ({
+jest.mock('../lib', () => {
-  fetchStats: jest.fn(),
+  const originalModule = jest.requireActual('../lib');
-  fetchAvailableIndices: jest.fn(),
+  return {
-}));
+    ...originalModule,
+    fetchStats: jest.fn(),
+    fetchMeteringStats: jest.fn(),
+    fetchAvailableIndices: jest.fn(),
+  };
+});
 
 describe('getIndexStatsRoute route', () => {
   let server: ReturnType<typeof serverMock.create>;
@@ -49,10 +57,41 @@ describe('getIndexStatsRoute route', () => {
     getIndexStatsRoute(server.router, logger);
   });
 
-  test('Returns index stats', async () => {
+  test('Returns index stats when index health is green', async () => {
-    const mockIndices = { 'auditbeat-7.15.1-2022.12.06-000001': {} };
+    const mockIndices = {
+      'auditbeat-custom-index-1': {
+        name: 'auditbeat-custom-index-1',
+        num_docs: 4,
+        size_in_bytes: 28425,
+        uuid: 'jRlr6H_jSAysOLZ6KynoCQ',
+      },
+    };
     (fetchStats as jest.Mock).mockResolvedValue({
-      indices: mockIndices,
+      indices: mockStatsGreenIndex,
+    });
+
+    const response = await server.inject(req, requestContextMock.convertContext(context));
+    expect(response.status).toEqual(200);
+    expect(response.body).toEqual(mockIndices);
+  });
+
+  test('Returns index stats when index health is yellow', async () => {
+    const mockIndices = {
+      '.ds-packetbeat-8.6.1-2023.02.04-000001': {
+        name: '.ds-packetbeat-8.6.1-2023.02.04-000001',
+        num_docs: 1628343,
+        size_in_bytes: 731583142,
+        uuid: 'x5Uuw4j4QM2YidHLNixCwg',
+      },
+      '.ds-packetbeat-8.5.3-2023.02.04-000001': {
+        name: '.ds-packetbeat-8.5.3-2023.02.04-000001',
+        num_docs: 1630289,
+        size_in_bytes: 733175040,
+        uuid: 'we0vNWm2Q6iz6uHubyHS6Q',
+      },
+    };
+    (fetchStats as jest.Mock).mockResolvedValue({
+      indices: mockStatsYellowIndex,
     });
 
     const response = await server.inject(req, requestContextMock.convertContext(context));
@@ -81,9 +120,8 @@ describe('getIndexStatsRoute route', () => {
       },
     });
 
-    const mockIndices = { 'auditbeat-7.15.1-2022.12.06-000001': {} };
     (fetchStats as jest.Mock).mockResolvedValue({
-      indices: mockIndices,
+      indices: mockMeteringStatsIndex,
     });
 
     const response = await server.inject(request, requestContextMock.convertContext(context));
@@ -107,18 +145,19 @@ describe('getIndexStatsRoute route', () => {
     });
 
     const mockIndices = {
-      'auditbeat-7.15.1-2022.12.06-000001': {},
+      'my-index-000001': {
-      'auditbeat-7.15.1-2022.11.06-000001': {},
+        name: 'my-index-000001',
+        num_docs: 2,
+        size_in_bytes: null,
+      },
     };
-    (fetchStats as jest.Mock).mockResolvedValue({
+    (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex);
-      indices: mockIndices,
-    });
     (fetchAvailableIndices as jest.Mock).mockResolvedValue({
       aggregations: {
         index: {
           buckets: [
             {
-              key: 'auditbeat-7.15.1-2022.12.06-000001',
+              key: 'my-index-000001',
             },
           ],
         },
@@ -127,7 +166,64 @@ describe('getIndexStatsRoute route', () => {
 
     const response = await server.inject(request, requestContextMock.convertContext(context));
     expect(response.status).toEqual(200);
-    expect(response.body).toEqual({ 'auditbeat-7.15.1-2022.12.06-000001': {} });
+    expect(response.body).toEqual(mockIndices);
+  });
+
+  test('returns an empty object when "meteringStats indices" are not available', async () => {
+    const request = requestMock.create({
+      method: 'get',
+      path: GET_INDEX_STATS,
+      params: {
+        pattern: `auditbeat-*`,
+      },
+      query: {
+        isILMAvailable: false,
+        startDate: `now-7d`,
+        endDate: `now`,
+      },
+    });
+
+    const mockIndices = {};
+    (fetchMeteringStats as jest.Mock).mockResolvedValue({ indices: undefined });
+    (fetchAvailableIndices as jest.Mock).mockResolvedValue({
+      aggregations: undefined,
+    });
+
+    const response = await server.inject(request, requestContextMock.convertContext(context));
+    expect(response.status).toEqual(200);
+    expect(response.body).toEqual(mockIndices);
+    expect(fetchAvailableIndices).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalledWith(
+      `No metering stats indices found under pattern: auditbeat-*`
+    );
+  });
+
+  test('returns an empty object when "availableIndices" indices are not available', async () => {
+    const request = requestMock.create({
+      method: 'get',
+      path: GET_INDEX_STATS,
+      params: {
+        pattern: `auditbeat-*`,
+      },
+      query: {
+        isILMAvailable: false,
+        startDate: `now-7d`,
+        endDate: `now`,
+      },
+    });
+
+    const mockIndices = {};
+    (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex);
+    (fetchAvailableIndices as jest.Mock).mockResolvedValue({
+      aggregations: undefined,
+    });
+
+    const response = await server.inject(request, requestContextMock.convertContext(context));
+    expect(response.status).toEqual(200);
+    expect(response.body).toEqual(mockIndices);
+    expect(logger.warn).toHaveBeenCalledWith(
+      `No available indices found under pattern: auditbeat-*, in the given date range: now-7d - now`
+    );
   });
 });
 

x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts

@@ -7,8 +7,14 @@
 import { i18n } from '@kbn/i18n';
 import type { IRouter, Logger } from '@kbn/core/server';
 
-import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types';
+import {
-import { fetchStats, fetchAvailableIndices } from '../lib';
+  fetchStats,
+  fetchAvailableIndices,
+  fetchMeteringStats,
+  parseIndicesStats,
+  parseMeteringStats,
+  pickAvailableMeteringStats,
+} from '../lib';
 import { buildResponse } from '../lib/build_response';
 import { GET_INDEX_STATS, INTERNAL_API_VERSION } from '../../common/constants';
 import { buildRouteValidation } from '../schemas/common';
@@ -41,12 +47,14 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => {
 
           const decodedIndexName = decodeURIComponent(request.params.pattern);
 
-          const stats = await fetchStats(client, decodedIndexName);
           const { isILMAvailable, startDate, endDate } = request.query;
 
           if (isILMAvailable === true) {
+            const stats = await fetchStats(client, decodedIndexName);
+            const parsedIndices = parseIndicesStats(stats.indices);
+
             return response.ok({
-              body: stats.indices,
+              body: parsedIndices,
             });
           }
 
@@ -57,24 +65,43 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => {
           if (startDate && endDate) {
             const decodedStartDate = decodeURIComponent(startDate);
             const decodedEndDate = decodeURIComponent(endDate);
+            const meteringStats = await fetchMeteringStats(
+              client,
+              decodedIndexName,
+              request.headers.authorization
+            );
 
-            const indices = await fetchAvailableIndices(esClient, {
+            if (!meteringStats.indices) {
+              logger.warn(`No metering stats indices found under pattern: ${decodedIndexName}`);
+              return response.ok({
+                body: {},
+              });
+            }
+
+            const meteringStatsIndices = parseMeteringStats(meteringStats.indices);
+
+            const availableIndices = await fetchAvailableIndices(esClient, {
               indexPattern: decodedIndexName,
               startDate: decodedStartDate,
               endDate: decodedEndDate,
             });
-            const availableIndices = indices?.aggregations?.index?.buckets?.reduce(
+
-              (acc: Record<string, IndicesStatsIndicesStats>, { key }: { key: string }) => {
+            if (!availableIndices.aggregations?.index?.buckets) {
-                if (stats.indices?.[key]) {
+              logger.warn(
-                  acc[key] = stats.indices?.[key];
+                `No available indices found under pattern: ${decodedIndexName}, in the given date range: ${decodedStartDate} - ${decodedEndDate}`
-                }
+              );
-                return acc;
+              return response.ok({
-              },
+                body: {},
-              {}
+              });
+            }
+
+            const indices = pickAvailableMeteringStats(
+              availableIndices.aggregations.index.buckets,
+              meteringStatsIndices
             );
 
             return response.ok({
-              body: availableIndices,
+              body: indices,
             });
           } else {
             return resp.error({
@@ -89,7 +116,6 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => {
           }
         } catch (err) {
           logger.error(JSON.stringify(err));
-
           return resp.error({
             body: err.message ?? API_DEFAULT_ERROR_MESSAGE,
             statusCode: err.statusCode ?? 500,

x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts

@@ -25,7 +25,7 @@ export const dataQualityIndexCheckedEvent: DataQualityTelemetryIndexCheckedEvent
       type: 'keyword',
       _meta: {
         description: 'Index uuid',
-        optional: false,
+        optional: true,
       },
     },
     indexName: {

x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts

@@ -11,7 +11,7 @@ import type { TelemetryEventTypes } from '../../constants';
 export type ReportDataQualityIndexCheckedParams = ReportDataQualityCheckAllCompletedParams & {
   errorCount?: number;
   ilmPhase?: string;
-  indexId: string;
+  indexId?: string;
   indexName: string;
   sameFamilyFields?: string[];
   unallowedMappingFields?: string[];