diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/index.tsx index 4ce7801dbe8f..9b2dff03e30c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/index.tsx @@ -102,6 +102,7 @@ const DataQualityDetailsComponent: React.FC = ({ <> { ); }); - test('it returns zero when the pattern exists in the rollup, but does not have a sizeInBytes', () => { + test('it returns undefined when the pattern exists in the rollup, but does not have a sizeInBytes', () => { const pattern = 'valid-*'; - expect(getPatternSizeInBytes({ pattern, patternRollups: noSizeInBytes })).toEqual(0); + expect(getPatternSizeInBytes({ pattern, patternRollups: noSizeInBytes })).toBeUndefined(); }); - test('it returns zero when the pattern does NOT exist in the rollup', () => { + test('it returns undefined when the pattern does NOT exist in the rollup', () => { const pattern = 'does-not-exist-*'; - expect(getPatternSizeInBytes({ pattern, patternRollups })).toEqual(0); + expect(getPatternSizeInBytes({ pattern, patternRollups })).toBeUndefined(); }); }); @@ -93,6 +93,7 @@ describe('helpers', () => { index: null, pattern, sizeInBytes: auditbeatWithAllResults.sizeInBytes, + docsCount: auditbeatWithAllResults.docsCount, }); }); }); @@ -113,6 +114,7 @@ describe('helpers', () => { index: '.ds-auditbeat-8.6.1-2023.02.07-000001', pattern: 'auditbeat-*', sizeInBytes: 18791790, + docsCount: 19123, }, { color: euiThemeVars.euiColorDanger, @@ -120,6 +122,7 @@ describe('helpers', () => { index: 'auditbeat-custom-index-1', pattern: 'auditbeat-*', sizeInBytes: 28409, + docsCount: 4, }, { color: euiThemeVars.euiColorDanger, @@ -127,6 +130,7 @@ describe('helpers', () => { index: 'auditbeat-custom-empty-index-1', pattern: 'auditbeat-*', sizeInBytes: 247, + docsCount: 0, }, ]); }); @@ -145,6 +149,7 @@ describe('helpers', () => { index: '.ds-auditbeat-8.6.1-2023.02.07-000001', pattern: 'auditbeat-*', sizeInBytes: 18791790, + docsCount: 19123, }, { color: euiThemeVars.euiColorDanger, @@ -152,6 +157,7 @@ describe('helpers', () => { index: 'auditbeat-custom-index-1', pattern: 'auditbeat-*', sizeInBytes: 28409, + docsCount: 4, }, { color: euiThemeVars.euiColorDanger, @@ -159,6 +165,7 @@ describe('helpers', () => { index: 'auditbeat-custom-empty-index-1', pattern: 'auditbeat-*', sizeInBytes: 247, + docsCount: 0, }, ]); }); @@ -179,6 +186,7 @@ describe('helpers', () => { index: null, pattern: '.alerts-security.alerts-default', sizeInBytes: 29717961631, + docsCount: 26093, }, { color: euiThemeVars.euiColorSuccess, @@ -186,14 +194,23 @@ describe('helpers', () => { index: '.internal.alerts-security.alerts-default-000001', pattern: '.alerts-security.alerts-default', sizeInBytes: 0, + docsCount: 26093, + }, + { + color: null, + ilmPhase: null, + index: null, + pattern: 'auditbeat-*', + sizeInBytes: 18820446, + docsCount: 19127, }, - { color: null, ilmPhase: null, index: null, pattern: 'auditbeat-*', sizeInBytes: 18820446 }, { color: euiThemeVars.euiColorSuccess, ilmPhase: 'hot', index: '.ds-auditbeat-8.6.1-2023.02.07-000001', pattern: 'auditbeat-*', sizeInBytes: 18791790, + docsCount: 19123, }, { color: euiThemeVars.euiColorDanger, @@ -201,6 +218,7 @@ describe('helpers', () => { index: 'auditbeat-custom-index-1', pattern: 'auditbeat-*', sizeInBytes: 28409, + docsCount: 4, }, { color: euiThemeVars.euiColorDanger, @@ -208,6 +226,7 @@ describe('helpers', () => { index: 'auditbeat-custom-empty-index-1', pattern: 'auditbeat-*', sizeInBytes: 247, + docsCount: 0, }, { color: null, @@ -215,6 +234,7 @@ describe('helpers', () => { index: null, pattern: 'packetbeat-*', sizeInBytes: 1096520898, + docsCount: 3258632, }, { color: euiThemeVars.euiColorPrimary, @@ -222,6 +242,7 @@ describe('helpers', () => { index: '.ds-packetbeat-8.5.3-2023.02.04-000001', pattern: 'packetbeat-*', sizeInBytes: 584326147, + docsCount: 1630289, }, { color: euiThemeVars.euiColorPrimary, @@ -229,6 +250,7 @@ describe('helpers', () => { index: '.ds-packetbeat-8.6.1-2023.02.04-000001', pattern: 'packetbeat-*', sizeInBytes: 512194751, + docsCount: 1628343, }, ]); }); @@ -249,6 +271,7 @@ describe('helpers', () => { indexName: '.internal.alerts-security.alerts-default-000001', pattern: '.alerts-security.alerts-default', sizeInBytes: 0, + docsCount: 26093, }, { ilmPhase: 'hot', @@ -256,6 +279,7 @@ describe('helpers', () => { indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001', pattern: 'auditbeat-*', sizeInBytes: 18791790, + docsCount: 19123, }, { ilmPhase: 'unmanaged', @@ -263,6 +287,7 @@ describe('helpers', () => { indexName: 'auditbeat-custom-empty-index-1', pattern: 'auditbeat-*', sizeInBytes: 247, + docsCount: 0, }, { ilmPhase: 'unmanaged', @@ -270,18 +295,21 @@ describe('helpers', () => { indexName: 'auditbeat-custom-index-1', pattern: 'auditbeat-*', sizeInBytes: 28409, + docsCount: 4, }, { ilmPhase: 'hot', indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', pattern: 'packetbeat-*', sizeInBytes: 512194751, + docsCount: 1628343, }, { ilmPhase: 'hot', indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', pattern: 'packetbeat-*', sizeInBytes: 584326147, + docsCount: 1630289, }, ]); }); @@ -295,6 +323,7 @@ describe('helpers', () => { }) ).toEqual([ { + docsCount: 26093, ilmPhase: undefined, incompatible: 0, indexName: '.internal.alerts-security.alerts-default-000001', @@ -302,6 +331,7 @@ describe('helpers', () => { sizeInBytes: 0, }, { + docsCount: 19123, ilmPhase: undefined, incompatible: 0, indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001', @@ -309,6 +339,7 @@ describe('helpers', () => { sizeInBytes: 18791790, }, { + docsCount: 0, ilmPhase: undefined, incompatible: 1, indexName: 'auditbeat-custom-empty-index-1', @@ -316,6 +347,7 @@ describe('helpers', () => { sizeInBytes: 247, }, { + docsCount: 4, ilmPhase: undefined, incompatible: 3, indexName: 'auditbeat-custom-index-1', @@ -323,12 +355,14 @@ describe('helpers', () => { sizeInBytes: 28409, }, { + docsCount: 1628343, ilmPhase: undefined, indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', pattern: 'packetbeat-*', sizeInBytes: 512194751, }, { + docsCount: 1630289, ilmPhase: undefined, indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', pattern: 'packetbeat-*', @@ -373,6 +407,7 @@ describe('helpers', () => { ilmPhase: 'hot', incompatible: 0, sizeInBytes: 0, + docsCount: 26093, }, 'auditbeat-*.ds-auditbeat-8.6.1-2023.02.07-000001': { pattern: 'auditbeat-*', @@ -380,6 +415,7 @@ describe('helpers', () => { ilmPhase: 'hot', incompatible: 0, sizeInBytes: 18791790, + docsCount: 19123, }, 'auditbeat-*auditbeat-custom-empty-index-1': { pattern: 'auditbeat-*', @@ -387,6 +423,7 @@ describe('helpers', () => { ilmPhase: 'unmanaged', incompatible: 1, sizeInBytes: 247, + docsCount: 0, }, 'auditbeat-*auditbeat-custom-index-1': { pattern: 'auditbeat-*', @@ -394,14 +431,17 @@ describe('helpers', () => { ilmPhase: 'unmanaged', incompatible: 3, sizeInBytes: 28409, + docsCount: 4, }, 'packetbeat-*.ds-packetbeat-8.6.1-2023.02.04-000001': { pattern: 'packetbeat-*', indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', ilmPhase: 'hot', sizeInBytes: 512194751, + docsCount: 1628343, }, 'packetbeat-*.ds-packetbeat-8.5.3-2023.02.04-000001': { + docsCount: 1630289, pattern: 'packetbeat-*', indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', ilmPhase: 'hot', @@ -457,14 +497,20 @@ describe('helpers', () => { it('returns the expected number of layers', () => { expect( - getLayersMultiDimensional({ formatBytes, layer0FillColor, pathToFlattenedBucketMap }).length + getLayersMultiDimensional({ + valueFormatter: formatBytes, + layer0FillColor, + pathToFlattenedBucketMap, + }).length ).toEqual(2); }); it('returns the expected fillLabel valueFormatter function', () => { - getLayersMultiDimensional({ formatBytes, layer0FillColor, pathToFlattenedBucketMap }).forEach( - (x) => expect(x.fillLabel.valueFormatter(123)).toEqual('123B') - ); + getLayersMultiDimensional({ + valueFormatter: formatBytes, + layer0FillColor, + pathToFlattenedBucketMap, + }).forEach((x) => expect(x.fillLabel.valueFormatter(123)).toEqual('123B')); }); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/helpers.ts index 326488e707aa..5b3860af4492 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/helpers.ts @@ -9,7 +9,7 @@ import type { Datum, Key, ArrayNode } from '@elastic/charts'; import { euiThemeVars } from '@kbn/ui-theme'; import { orderBy } from 'lodash/fp'; -import { getSizeInBytes } from '../../../../helpers'; +import { getDocsCount, getSizeInBytes } from '../../../../helpers'; import { getIlmPhase } from '../../../pattern/helpers'; import { PatternRollup } from '../../../../types'; @@ -18,7 +18,8 @@ export interface LegendItem { ilmPhase: string | null; index: string | null; pattern: string; - sizeInBytes: number; + sizeInBytes: number | undefined; + docsCount: number; } export interface FlattenedBucket { @@ -26,7 +27,8 @@ export interface FlattenedBucket { incompatible: number | undefined; indexName: string | undefined; pattern: string; - sizeInBytes: number; + sizeInBytes: number | undefined; + docsCount: number; } export const getPatternSizeInBytes = ({ @@ -35,9 +37,23 @@ export const getPatternSizeInBytes = ({ }: { pattern: string; patternRollups: Record; +}): number | undefined => { + if (patternRollups[pattern] != null) { + return patternRollups[pattern].sizeInBytes; + } else { + return undefined; + } +}; + +export const getPatternDocsCount = ({ + pattern, + patternRollups, +}: { + pattern: string; + patternRollups: Record; }): number => { if (patternRollups[pattern] != null) { - return patternRollups[pattern].sizeInBytes ?? 0; + return patternRollups[pattern].docsCount ?? 0; } else { return 0; } @@ -55,6 +71,7 @@ export const getPatternLegendItem = ({ index: null, pattern, sizeInBytes: getPatternSizeInBytes({ pattern, patternRollups }), + docsCount: getPatternDocsCount({ pattern, patternRollups }), }); export const getLegendItemsForPattern = ({ @@ -74,7 +91,8 @@ export const getLegendItemsForPattern = ({ ilmPhase: flattenedBucket.ilmPhase ?? null, index: flattenedBucket.indexName ?? null, pattern: flattenedBucket.pattern, - sizeInBytes: flattenedBucket.sizeInBytes ?? 0, + sizeInBytes: flattenedBucket.sizeInBytes, + docsCount: flattenedBucket.docsCount, })) ); @@ -129,7 +147,7 @@ export const getFlattenedBuckets = ({ ? results[indexName].incompatible : undefined; const sizeInBytes = getSizeInBytes({ indexName, stats }); - + const docsCount = getDocsCount({ stats, indexName }); return [ ...validStats, { @@ -138,6 +156,7 @@ export const getFlattenedBuckets = ({ indexName, pattern, sizeInBytes, + docsCount, }, ]; } else { @@ -187,16 +206,14 @@ export const getGroupFromPath = (path: ArrayNode['path']): string | undefined => }; export const getLayersMultiDimensional = ({ - formatBytes, + valueFormatter, layer0FillColor, pathToFlattenedBucketMap, }: { - formatBytes: (value: number | undefined) => string; + valueFormatter: (value: number) => string; layer0FillColor: string; pathToFlattenedBucketMap: Record; }) => { - const valueFormatter = (d: number) => formatBytes(d); - return [ { fillLabel: { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.test.tsx index 17a64b43b5ef..a3ff3665cf6e 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.test.tsx @@ -22,6 +22,10 @@ const defaultBytesFormat = '0,0.[0]b'; const formatBytes = (value: number | undefined) => value != null ? numeral(value).format(defaultBytesFormat) : EMPTY_STAT; +const defaultNumberFormat = '0,0.[000]'; +const formatNumber = (value: number | undefined) => + value != null ? numeral(value).format(defaultNumberFormat) : EMPTY_STAT; + const ilmPhases = ['hot', 'warm', 'unmanaged']; const patterns = ['.alerts-security.alerts-default', 'auditbeat-*', 'packetbeat-*']; @@ -35,6 +39,7 @@ const onIndexSelected = jest.fn(); const defaultProps: Props = { formatBytes, + formatNumber, ilmPhases, onIndexSelected, patternRollups, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.tsx index 571bc6329920..a6a96e1b23aa 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/index.tsx @@ -6,16 +6,18 @@ */ import type { PartialTheme, Theme } from '@elastic/charts'; -import React, { useMemo } from 'react'; +import React, { useCallback, useMemo } from 'react'; import { getFlattenedBuckets } from './helpers'; import { StorageTreemap } from '../../../storage_treemap'; import { DEFAULT_MAX_CHART_HEIGHT, StorageTreemapContainer } from '../../../tabs/styles'; import { PatternRollup, SelectedIndex } from '../../../../types'; import { useDataQualityContext } from '../../../data_quality_context'; +import { DOCS_UNIT } from './translations'; export interface Props { formatBytes: (value: number | undefined) => string; + formatNumber: (value: number | undefined) => string; ilmPhases: string[]; onIndexSelected: ({ indexName, pattern }: SelectedIndex) => void; patternRollups: Record; @@ -26,6 +28,7 @@ export interface Props { const StorageDetailsComponent: React.FC = ({ formatBytes, + formatNumber, ilmPhases, onIndexSelected, patternRollups, @@ -44,18 +47,25 @@ const StorageDetailsComponent: React.FC = ({ }), [ilmPhases, isILMAvailable, patternRollups] ); + const accessor = flattenedBuckets[0]?.sizeInBytes != null ? 'sizeInBytes' : 'docsCount'; + const valueFormatter = useCallback( + (d: number) => + accessor === 'sizeInBytes' ? formatBytes(d) : `${formatNumber(d)} ${DOCS_UNIT(d)}`, + [accessor, formatBytes, formatNumber] + ); return ( ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/translations.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/translations.ts new file mode 100644 index 000000000000..82332bde4423 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/body/data_quality_details/storage_details/translations.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const DOCS_UNIT = (totalCount: number) => + i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.storage.docs.unit', { + values: { totalCount }, + defaultMessage: `{totalCount, plural, =1 {Doc} other {Docs}}`, + }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts index 30f314c73ea3..ede3184350e5 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts @@ -5,11 +5,10 @@ * 2.0. */ -import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; import { orderBy } from 'lodash/fp'; import { getDocsCount } from '../../../../helpers'; -import type { IndexToCheck, PatternRollup } from '../../../../types'; +import type { IndexToCheck, MeteringStatsIndex, PatternRollup } from '../../../../types'; export const getIndexToCheck = ({ indexName, @@ -54,7 +53,7 @@ export const getIndexDocsCountFromRollup = ({ indexName: string; patternRollup: PatternRollup; }): number => { - const stats: Record | null = patternRollup?.stats ?? null; + const stats: Record | null = patternRollup?.stats ?? null; return getDocsCount({ indexName, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/index.tsx index 76cd13d29134..974f44c4c664 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/index.tsx @@ -74,6 +74,7 @@ export const getAllMarkdownCommentsFromResults = ({ const summaryTableMarkdownRows: string[] = summaryTableItems.map((item) => { const result: DataQualityCheckResult | undefined = patternRollup.results != null ? patternRollup.results[item.indexName] : undefined; + const sizeInBytes = getSizeInBytes({ indexName: item.indexName, stats: patternRollup.stats }); return getSummaryTableMarkdownRow({ docsCount: item.docsCount, @@ -84,7 +85,7 @@ export const getAllMarkdownCommentsFromResults = ({ incompatible: result?.incompatible, isILMAvailable, patternDocsCount: patternRollup.docsCount ?? 0, - sizeInBytes: getSizeInBytes({ indexName: item.indexName, stats: patternRollup.stats }), + sizeInBytes, }).trim(); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts index 154d78a64f6c..098c66370f1c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts @@ -432,7 +432,14 @@ describe('helpers', () => { test('it returns the expected header when isILMAvailable is false', () => { const isILMAvailable = false; expect(getSummaryTableMarkdownHeader(isILMAvailable)).toEqual( - '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|' + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|' + ); + }); + + test('it returns the expected header when displayDocSize is false', () => { + const isILMAvailable = false; + expect(getSummaryTableMarkdownHeader(isILMAvailable)).toEqual( + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|' ); }); }); @@ -481,9 +488,25 @@ describe('helpers', () => { indexName: 'auditbeat-custom-index-1', isILMAvailable: false, patternDocsCount: 57410, - sizeInBytes: 28413, + sizeInBytes: undefined, }) - ).toEqual('| -- | auditbeat-custom-index-1 | 4 (0.0%) | -- | 27.7KB |\n'); + ).toEqual('| -- | auditbeat-custom-index-1 | 4 (0.0%) | -- |\n'); + }); + + test('it returns the expected row when sizeInBytes is undefined', () => { + expect( + getSummaryTableMarkdownRow({ + docsCount: 4, + formatBytes, + formatNumber, + incompatible: undefined, // <-- + ilmPhase: undefined, // <-- + indexName: 'auditbeat-custom-index-1', + isILMAvailable: false, + patternDocsCount: 57410, + sizeInBytes: undefined, + }) + ).toEqual('| -- | auditbeat-custom-index-1 | 4 (0.0%) | -- |\n'); }); }); @@ -517,10 +540,28 @@ describe('helpers', () => { isILMAvailable: false, partitionedFieldMetadata: mockPartitionedFieldMetadata, patternDocsCount: 57410, - sizeInBytes: 28413, + sizeInBytes: undefined, }) ).toEqual( - '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | 27.7KB |\n\n' + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n' + ); + }); + + test('it returns the expected comment when sizeInBytes is undefined', () => { + expect( + getSummaryTableMarkdownComment({ + docsCount: 4, + formatBytes, + formatNumber, + ilmPhase: 'unmanaged', + indexName: 'auditbeat-custom-index-1', + isILMAvailable: false, + partitionedFieldMetadata: mockPartitionedFieldMetadata, + patternDocsCount: 57410, + sizeInBytes: undefined, + }) + ).toEqual( + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n' ); }); }); @@ -554,7 +595,7 @@ describe('helpers', () => { sizeInBytes: undefined, }) ).toEqual( - '| Incompatible fields | Indices checked | Indices | Size | Docs |\n|---------------------|-----------------|---------|------|------|\n| -- | -- | -- | -- | 0 |\n' + '| Incompatible fields | Indices checked | Indices | Docs |\n|---------------------|-----------------|---------|------|\n| -- | -- | -- | 0 |\n' ); }); }); @@ -588,7 +629,7 @@ describe('helpers', () => { sizeInBytes: undefined, }) ).toEqual( - '# Data quality\n\n| Incompatible fields | Indices checked | Indices | Size | Docs |\n|---------------------|-----------------|---------|------|------|\n| -- | -- | -- | -- | 0 |\n\n' + '# Data quality\n\n| Incompatible fields | Indices checked | Indices | Docs |\n|---------------------|-----------------|---------|------|\n| -- | -- | -- | 0 |\n\n' ); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts index b1d8996fac95..850986e3959f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts @@ -218,18 +218,18 @@ export const getResultEmoji = (incompatible: number | undefined): string => { } }; -export const getSummaryTableMarkdownHeader = (isILMAvailable: boolean): string => - isILMAvailable +export const getSummaryTableMarkdownHeader = (includeDocSize: boolean): string => + includeDocSize ? `| ${RESULT} | ${INDEX} | ${DOCS} | ${INCOMPATIBLE_FIELDS} | ${ILM_PHASE} | ${SIZE} | |${getHeaderSeparator(RESULT)}|${getHeaderSeparator(INDEX)}|${getHeaderSeparator( DOCS )}|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator( ILM_PHASE )}|${getHeaderSeparator(SIZE)}|` - : `| ${RESULT} | ${INDEX} | ${DOCS} | ${INCOMPATIBLE_FIELDS} | ${SIZE} | + : `| ${RESULT} | ${INDEX} | ${DOCS} | ${INCOMPATIBLE_FIELDS} | |${getHeaderSeparator(RESULT)}|${getHeaderSeparator(INDEX)}|${getHeaderSeparator( DOCS - )}|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator(SIZE)}|`; + )}|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|`; export const getSummaryTableMarkdownRow = ({ docsCount, @@ -252,7 +252,7 @@ export const getSummaryTableMarkdownRow = ({ patternDocsCount: number; sizeInBytes: number | undefined; }): string => - isILMAvailable + isILMAvailable && Number.isInteger(sizeInBytes) ? `| ${getResultEmoji(incompatible)} | ${escape(indexName)} | ${formatNumber( docsCount )} (${getDocsCountPercent({ @@ -267,7 +267,7 @@ export const getSummaryTableMarkdownRow = ({ )} (${getDocsCountPercent({ docsCount, patternDocsCount, - })}) | ${incompatible ?? EMPTY_PLACEHOLDER} | ${formatBytes(sizeInBytes)} | + })}) | ${incompatible ?? EMPTY_PLACEHOLDER} | `; export const getSummaryTableMarkdownComment = ({ @@ -322,13 +322,22 @@ export const getStatsRollupMarkdownComment = ({ indicesChecked: number | undefined; sizeInBytes: number | undefined; }): string => - `| ${INCOMPATIBLE_FIELDS} | ${INDICES_CHECKED} | ${INDICES} | ${SIZE} | ${DOCS} | + Number.isInteger(sizeInBytes) + ? `| ${INCOMPATIBLE_FIELDS} | ${INDICES_CHECKED} | ${INDICES} | ${SIZE} | ${DOCS} | |${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator( - INDICES_CHECKED - )}|${getHeaderSeparator(INDICES)}|${getHeaderSeparator(SIZE)}|${getHeaderSeparator(DOCS)}| + INDICES_CHECKED + )}|${getHeaderSeparator(INDICES)}|${getHeaderSeparator(SIZE)}|${getHeaderSeparator(DOCS)}| | ${incompatible ?? EMPTY_STAT} | ${indicesChecked ?? EMPTY_STAT} | ${ - indices ?? EMPTY_STAT - } | ${formatBytes(sizeInBytes)} | ${formatNumber(docsCount)} | + indices ?? EMPTY_STAT + } | ${formatBytes(sizeInBytes)} | ${formatNumber(docsCount)} | +` + : `| ${INCOMPATIBLE_FIELDS} | ${INDICES_CHECKED} | ${INDICES} | ${DOCS} | +|${getHeaderSeparator(INCOMPATIBLE_FIELDS)}|${getHeaderSeparator( + INDICES_CHECKED + )}|${getHeaderSeparator(INDICES)}|${getHeaderSeparator(DOCS)}| +| ${incompatible ?? EMPTY_STAT} | ${indicesChecked ?? EMPTY_STAT} | ${ + indices ?? EMPTY_STAT + } | ${formatNumber(docsCount)} | `; export const getDataQualitySummaryMarkdownComment = ({ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts index cdf625111375..efc4f3a5a509 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts @@ -441,7 +441,7 @@ describe('helpers', () => { indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', pattern: 'auditbeat-*', patternDocsCount: 4, - sizeInBytes: 0, + sizeInBytes: undefined, checkedAt: undefined, }, { @@ -451,7 +451,7 @@ describe('helpers', () => { indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001', pattern: 'auditbeat-*', patternDocsCount: 4, - sizeInBytes: 0, + sizeInBytes: undefined, checkedAt: undefined, }, { @@ -461,7 +461,7 @@ describe('helpers', () => { indexName: 'auditbeat-custom-index-1', pattern: 'auditbeat-*', patternDocsCount: 4, - sizeInBytes: 0, + sizeInBytes: undefined, checkedAt: undefined, }, ]); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts index 1bddcd83e7a1..1c64e1eb0095 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts @@ -5,10 +5,7 @@ * 2.0. */ -import type { - IlmExplainLifecycleLifecycleExplain, - IndicesStatsIndicesStats, -} from '@elastic/elasticsearch/lib/api/types'; +import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; import { isEqual, orderBy } from 'lodash/fp'; import type { IndexSummaryTableItem } from '../summary_table/helpers'; @@ -18,6 +15,7 @@ import type { DataQualityCheckResult, PatternRollup, SortConfig, + MeteringStatsIndex, } from '../../types'; import { getDocsCount, getSizeInBytes } from '../../helpers'; @@ -159,7 +157,7 @@ export const getSummaryTableItems = ({ results: Record | undefined; sortByColumn: string; sortByDirection: 'desc' | 'asc'; - stats: Record | null; + stats: Record | null; }): IndexSummaryTableItem[] => { const summaryTableItems = indexNames.map((indexName) => ({ docsCount: getDocsCount({ stats, indexName }), @@ -189,7 +187,7 @@ export const shouldCreateIndexNames = ({ indexNames: string[] | undefined; isILMAvailable: boolean; newIndexNames: string[]; - stats: Record | null; + stats: Record | null; }): boolean => { return ( !isEqual(newIndexNames, indexNames) && @@ -211,7 +209,7 @@ export const shouldCreatePatternRollup = ({ isILMAvailable: boolean; newDocsCount: number; patternRollup: PatternRollup | undefined; - stats: Record | null; + stats: Record | null; }): boolean => { if (patternRollup?.docsCount === newDocsCount) { return false; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/index.tsx index 93e481fe309e..d54e2bf12fae 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/index.tsx @@ -272,10 +272,12 @@ const PatternComponent: React.FC = ({ indices: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }).length, pattern, results: undefined, - sizeInBytes: getTotalSizeInBytes({ - indexNames: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }), - stats, - }), + sizeInBytes: isILMAvailable + ? getTotalSizeInBytes({ + indexNames: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }), + stats, + }) ?? 0 + : undefined, stats, }); } @@ -335,7 +337,7 @@ const PatternComponent: React.FC = ({ ilmExplainPhaseCounts={ilmExplainPhaseCounts} pattern={pattern} patternDocsCount={patternRollup?.docsCount ?? 0} - patternSizeInBytes={patternRollup?.sizeInBytes ?? 0} + patternSizeInBytes={patternRollup?.sizeInBytes} /> diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/index.tsx index 1ef9443d7c9c..722f56ca721d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/index.tsx @@ -21,7 +21,7 @@ interface Props { indicesChecked: number | undefined; pattern: string; patternDocsCount: number; - patternSizeInBytes: number; + patternSizeInBytes: number | undefined; } const PatternSummaryComponent: React.FC = ({ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/stats_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/stats_rollup/index.tsx index 4a5188040454..32d157839351 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/stats_rollup/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/pattern_summary/stats_rollup/index.tsx @@ -120,23 +120,25 @@ const StatsRollupComponent: React.FC = ({ - - - - - - - + {sizeInBytes != null && ( + + + + + + + + )} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.test.tsx index f1ae399c4da2..c01e21e5d3dd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.test.tsx @@ -49,13 +49,14 @@ const flattenedBuckets = getFlattenedBuckets({ const onIndexSelected = jest.fn(); const defaultProps: Props = { + accessor: 'sizeInBytes', flattenedBuckets, - formatBytes, maxChartHeight: DEFAULT_MAX_CHART_HEIGHT, onIndexSelected, patternRollups, patterns, baseTheme: DARK_THEME, + valueFormatter: formatBytes, }; jest.mock('@elastic/charts', () => { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.tsx index a07d111eab8f..2edd59be93df 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/storage_treemap/index.tsx @@ -39,15 +39,16 @@ export const LEGEND_WIDTH = 220; // px export const LEGEND_TEXT_WITH = 120; // px export interface Props { + accessor: 'sizeInBytes' | 'docsCount'; + baseTheme: Theme; flattenedBuckets: FlattenedBucket[]; - formatBytes: (value: number | undefined) => string; maxChartHeight?: number; minChartHeight?: number; onIndexSelected: ({ indexName, pattern }: SelectedIndex) => void; patternRollups: Record; patterns: string[]; theme?: PartialTheme; - baseTheme: Theme; + valueFormatter: (value: number) => string; } interface GetGroupByFieldsResult { @@ -84,15 +85,16 @@ export const getGroupByFieldsOnClick = ( }; const StorageTreemapComponent: React.FC = ({ + accessor, + baseTheme, flattenedBuckets, - formatBytes, maxChartHeight, minChartHeight = DEFAULT_MIN_CHART_HEIGHT, onIndexSelected, patternRollups, patterns, theme = {}, - baseTheme, + valueFormatter, }: Props) => { const fillColor = useMemo( () => theme?.background?.color ?? baseTheme.background.color, @@ -129,14 +131,14 @@ const StorageTreemapComponent: React.FC = ({ const layers = useMemo( () => getLayersMultiDimensional({ - formatBytes, + valueFormatter, layer0FillColor: fillColor, pathToFlattenedBucketMap, }), - [fillColor, formatBytes, pathToFlattenedBucketMap] + [fillColor, valueFormatter, pathToFlattenedBucketMap] ); - const valueAccessor = useCallback(({ sizeInBytes }: Datum) => sizeInBytes, []); + const valueAccessor = useCallback((d: Datum) => d[accessor], [accessor]); const legendItems = useMemo( () => getLegendItems({ patterns, flattenedBuckets, patternRollups }), @@ -167,7 +169,7 @@ const StorageTreemapComponent: React.FC = ({ layers={layers} layout={PartitionLayout.treemap} valueAccessor={valueAccessor} - valueFormatter={(d: number) => formatBytes(d)} + valueFormatter={valueFormatter} /> )} @@ -180,10 +182,10 @@ const StorageTreemapComponent: React.FC = ({ className="eui-yScroll" $width={LEGEND_WIDTH} > - {legendItems.map(({ color, ilmPhase, index, pattern, sizeInBytes }) => ( + {legendItems.map(({ color, ilmPhase, index, pattern, sizeInBytes, docsCount }) => ( { expect(screen.getByTestId('sizeInBytes')).toHaveTextContent('98.6MB'); }); + + test('it should not render sizeInBytes if it is not a number', () => { + const testIndexSummaryTableItem = { ...indexSummaryTableItem, sizeInBytes: undefined }; + const columns = getSummaryTableColumns({ + formatBytes, + formatNumber, + itemIdToExpandedRowMap: {}, + isILMAvailable, + pattern: 'auditbeat-*', + toggleExpanded: jest.fn(), + }); + + const sizeInBytesRender = (columns[6] as EuiTableFieldDataColumnType) + .render; + + render( + + {sizeInBytesRender != null && + sizeInBytesRender(testIndexSummaryTableItem, testIndexSummaryTableItem)} + + ); + + expect(screen.queryByTestId('sizeInBytes')).toBeNull(); + }); }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx index 55e26200f0e4..6af80a1e0628 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx @@ -42,7 +42,7 @@ export interface IndexSummaryTableItem { ilmPhase: IlmPhase | undefined; pattern: string; patternDocsCount: number; - sizeInBytes: number; + sizeInBytes: number | undefined; checkedAt: number | undefined; } @@ -120,6 +120,24 @@ export const getSummaryTableILMPhaseColumn = ( ] : []; +export const getSummaryTableSizeInBytesColumn = ({ + formatBytes, +}: { + formatBytes: (value: number | undefined) => string; +}): Array> => [ + { + field: 'sizeInBytes', + name: i18n.SIZE, + render: (_, { sizeInBytes }) => + Number.isInteger(sizeInBytes) ? ( + + {formatBytes(sizeInBytes)} + + ) : null, + sortable: true, + truncateText: false, + }, +]; export const getSummaryTableColumns = ({ formatBytes, formatNumber, @@ -232,11 +250,12 @@ export const getSummaryTableColumns = ({ { field: 'sizeInBytes', name: i18n.SIZE, - render: (_, { sizeInBytes }) => ( - - {formatBytes(sizeInBytes)} - - ), + render: (_, { sizeInBytes }) => + Number.isInteger(sizeInBytes) ? ( + + {formatBytes(sizeInBytes)} + + ) : null, sortable: true, truncateText: false, }, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts index 1209f671fae2..30aae990fc7d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts @@ -108,7 +108,29 @@ ${ECS_IS_A_PERMISSIVE_SCHEMA} }) ).toEqual([ '### auditbeat-custom-index-1\n', - '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | 27.7KB |\n\n', + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n', + '### **Incompatible fields** `3` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n', + `#### 4 Custom field mappings\n\nThese fields are not defined by the Elastic Common Schema (ECS), version ${EcsVersion}.\n\nECS is a permissive schema. If your events have additional data that cannot be mapped to ECS, you can simply add them to your events, using custom field names.\n`, + '#### Custom fields - auditbeat-custom-index-1\n\n\n| Field | Index mapping type | \n|-------|--------------------|\n| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |\n', + ]); + }); + + test('it returns the expected comment without Size when Size is undefined', () => { + expect( + getAllCustomMarkdownComments({ + docsCount: 4, + formatBytes, + formatNumber, + ilmPhase: 'unmanaged', + indexName: 'auditbeat-custom-index-1', + isILMAvailable: false, + partitionedFieldMetadata: mockPartitionedFieldMetadata, + patternDocsCount: 57410, + sizeInBytes: undefined, + }) + ).toEqual([ + '### auditbeat-custom-index-1\n', + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 |\n\n', '### **Incompatible fields** `3` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n', `#### 4 Custom field mappings\n\nThese fields are not defined by the Elastic Common Schema (ECS), version ${EcsVersion}.\n\nECS is a permissive schema. If your events have additional data that cannot be mapped to ECS, you can simply add them to your events, using custom field names.\n`, '#### Custom fields - auditbeat-custom-index-1\n\n\n| Field | Index mapping type | \n|-------|--------------------|\n| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |\n', diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx index 7dca72046ab3..c6386d7710bf 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx @@ -15,7 +15,7 @@ import { timestamp, } from '../../mock/enriched_field_metadata/mock_enriched_field_metadata'; import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; -import { mockStatsGreenIndex } from '../../mock/stats/mock_stats_green_index'; +import { mockStatsAuditbeatIndex } from '../../mock/stats/mock_stats_packetbeat_index'; import { getEcsCompliantColor, getMissingTimestampComment, @@ -83,7 +83,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 57410, setSelectedTabId: jest.fn(), - stats: mockStatsGreenIndex, + stats: mockStatsAuditbeatIndex, baseTheme: DARK_THEME, }).map((x) => omit(['append', 'content'], x)) ).toEqual([ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx index ea97d2565922..0f59258949b2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx @@ -15,7 +15,6 @@ import type { WordCloudElementEvent, XYChartElementEvent, } from '@elastic/charts'; -import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; import { EuiBadge } from '@elastic/eui'; import { euiThemeVars } from '@kbn/ui-theme'; import React from 'react'; @@ -39,7 +38,12 @@ import * as i18n from '../index_properties/translations'; import { SameFamilyTab } from './same_family_tab'; import { SummaryTab } from './summary_tab'; import { getFillColor } from './summary_tab/helpers'; -import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../types'; +import type { + EnrichedFieldMetadata, + IlmPhase, + MeteringStatsIndex, + PartitionedFieldMetadata, +} from '../../types'; export const getMissingTimestampComment = (): string => getMarkdownComment({ @@ -105,7 +109,7 @@ export const getTabs = ({ pattern: string; patternDocsCount: number; setSelectedTabId: (tabId: string) => void; - stats: Record | null; + stats: Record | null; theme?: PartialTheme; baseTheme: Theme; }) => [ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.test.ts index bd24ed71d11e..34f4bd244868 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.test.ts @@ -403,11 +403,37 @@ ${MAPPINGS_THAT_CONFLICT_WITH_ECS} isILMAvailable: false, partitionedFieldMetadata: emptyIncompatible, patternDocsCount: 57410, - sizeInBytes: 28413, + sizeInBytes: undefined, }) ).toEqual([ '### auditbeat-custom-index-1\n', - '| Result | Index | Docs | Incompatible fields | Size |\n|--------|-------|------|---------------------|------|\n| ✅ | auditbeat-custom-index-1 | 4 (0.0%) | 0 | 27.7KB |\n\n', + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ✅ | auditbeat-custom-index-1 | 4 (0.0%) | 0 |\n\n', + '### **Incompatible fields** `0` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n', + '\n\n\n', + ]); + }); + + test('it returns the expected comment when `sizeInBytes` is not an integer', () => { + const emptyIncompatible: PartitionedFieldMetadata = { + ...mockPartitionedFieldMetadata, + incompatible: [], // <-- empty + }; + + expect( + getAllIncompatibleMarkdownComments({ + docsCount: 4, + formatBytes, + formatNumber, + ilmPhase: 'unmanaged', + indexName: 'auditbeat-custom-index-1', + isILMAvailable: false, + partitionedFieldMetadata: emptyIncompatible, + patternDocsCount: 57410, + sizeInBytes: undefined, + }) + ).toEqual([ + '### auditbeat-custom-index-1\n', + '| Result | Index | Docs | Incompatible fields |\n|--------|-------|------|---------------------|\n| ✅ | auditbeat-custom-index-1 | 4 (0.0%) | 0 |\n\n', '### **Incompatible fields** `0` **Same family** `0` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n', '\n\n\n', ]); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts index cdb979e417d1..48a83974ba94 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts @@ -62,8 +62,8 @@ import { auditbeatWithAllResults, } from './mock/pattern_rollup/mock_auditbeat_pattern_rollup'; import { mockStats } from './mock/stats/mock_stats'; -import { mockStatsGreenIndex } from './mock/stats/mock_stats_green_index'; -import { mockStatsYellowIndex } from './mock/stats/mock_stats_yellow_index'; +import { mockStatsAuditbeatIndex } from './mock/stats/mock_stats_packetbeat_index'; +import { mockStatsPacketbeatIndex } from './mock/stats/mock_stats_auditbeat_index'; import { COLD_DESCRIPTION, FROZEN_DESCRIPTION, @@ -751,12 +751,12 @@ describe('helpers', () => { describe('getDocsCount', () => { test('it returns the expected docs count when `stats` contains the `indexName`', () => { const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; - const expectedCount = mockStatsYellowIndex[indexName].primaries?.docs?.count; + const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs; expect( getDocsCount({ indexName, - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(expectedCount); }); @@ -767,7 +767,7 @@ describe('helpers', () => { expect( getDocsCount({ indexName, - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(0); }); @@ -789,37 +789,37 @@ describe('helpers', () => { expect( getDocsCount({ indexName, - stats: mockStatsGreenIndex, + stats: mockStatsAuditbeatIndex, }) - ).toEqual(mockStatsGreenIndex[indexName].primaries?.docs?.count); + ).toEqual(mockStatsAuditbeatIndex[indexName].num_docs); }); }); describe('getSizeInBytes', () => { test('it returns the expected size when `stats` contains the `indexName`', () => { const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; - const expectedCount = mockStatsYellowIndex[indexName].primaries?.store?.size_in_bytes; + const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes; expect( getSizeInBytes({ indexName, - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(expectedCount); }); - test('it returns zero when `stats` does NOT contain the `indexName`', () => { + test('it returns undefined when `stats` does NOT contain the `indexName`', () => { const indexName = 'not-gonna-find-it'; expect( getSizeInBytes({ indexName, - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) - ).toEqual(0); + ).toBeUndefined(); }); - test('it returns zero when `stats` is null', () => { + test('it returns undefined when `stats` is null', () => { const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001'; expect( @@ -827,7 +827,7 @@ describe('helpers', () => { indexName, stats: null, }) - ).toEqual(0); + ).toBeUndefined(); }); test('it returns the expected size for a green index, where `primaries.store.size_in_bytes` and `total.store.size_in_bytes` have different values', () => { @@ -836,21 +836,21 @@ describe('helpers', () => { expect( getSizeInBytes({ indexName, - stats: mockStatsGreenIndex, + stats: mockStatsAuditbeatIndex, }) - ).toEqual(mockStatsGreenIndex[indexName].primaries?.store?.size_in_bytes); + ).toEqual(mockStatsAuditbeatIndex[indexName].size_in_bytes); }); }); describe('getTotalDocsCount', () => { test('it returns the expected total given a subset of index names in the stats', () => { const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001'; - const expectedCount = mockStatsYellowIndex[indexName].primaries?.docs?.count; + const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs; expect( getTotalDocsCount({ indexNames: [indexName], - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(expectedCount); }); @@ -864,7 +864,7 @@ describe('helpers', () => { expect( getTotalDocsCount({ indexNames: allIndexNamesInStats, - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(3258632); }); @@ -873,19 +873,19 @@ describe('helpers', () => { expect( getTotalDocsCount({ indexNames: [], // <-- empty - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(0); }); test('it returns the expected total for a green index', () => { const indexName = 'auditbeat-custom-index-1'; - const expectedCount = mockStatsGreenIndex[indexName].primaries?.docs?.count; + const expectedCount = mockStatsAuditbeatIndex[indexName].num_docs; expect( getTotalDocsCount({ indexNames: [indexName], - stats: mockStatsGreenIndex, + stats: mockStatsAuditbeatIndex, }) ).toEqual(expectedCount); }); @@ -894,12 +894,12 @@ describe('helpers', () => { describe('getTotalSizeInBytes', () => { test('it returns the expected total given a subset of index names in the stats', () => { const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001'; - const expectedCount = mockStatsYellowIndex[indexName].primaries?.store?.size_in_bytes; + const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes; expect( getTotalSizeInBytes({ indexNames: [indexName], - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(expectedCount); }); @@ -913,28 +913,58 @@ describe('helpers', () => { expect( getTotalSizeInBytes({ indexNames: allIndexNamesInStats, - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) ).toEqual(1464758182); }); - test('it returns zero given an empty collection of index names', () => { + test('it returns undefined given an empty collection of index names', () => { expect( getTotalSizeInBytes({ indexNames: [], // <-- empty - stats: mockStatsYellowIndex, + stats: mockStatsPacketbeatIndex, }) - ).toEqual(0); + ).toBeUndefined(); }); - test('it returns the expected total for a green index', () => { + test('it returns undefined if sizeInByte in not an integer', () => { const indexName = 'auditbeat-custom-index-1'; - const expectedCount = mockStatsGreenIndex[indexName].primaries?.store?.size_in_bytes; expect( getTotalSizeInBytes({ indexNames: [indexName], - stats: mockStatsGreenIndex, + stats: { [indexName]: { ...mockStatsAuditbeatIndex[indexName], size_in_bytes: null } }, + }) + ).toBeUndefined(); + }); + + test('it returns the expected total for an index', () => { + const indexName = 'auditbeat-custom-index-1'; + const expectedCount = mockStatsAuditbeatIndex[indexName].size_in_bytes; + + expect( + getTotalSizeInBytes({ + indexNames: [indexName], + stats: mockStatsAuditbeatIndex, + }) + ).toEqual(expectedCount); + }); + + test('it returns the expected total for indices', () => { + const expectedCount = Object.values(mockStatsPacketbeatIndex).reduce( + (acc, { size_in_bytes: sizeInBytes }) => { + return acc + (sizeInBytes ?? 0); + }, + 0 + ); + + expect( + getTotalSizeInBytes({ + indexNames: [ + '.ds-packetbeat-8.6.1-2023.02.04-000001', + '.ds-packetbeat-8.5.3-2023.02.04-000001', + ], + stats: mockStatsPacketbeatIndex, }) ).toEqual(expectedCount); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index 2107e7d3949d..6783ab32d21c 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -6,10 +6,7 @@ */ import type { HttpHandler } from '@kbn/core-http-browser'; -import type { - IlmExplainLifecycleLifecycleExplain, - IndicesStatsIndicesStats, -} from '@elastic/elasticsearch/lib/api/types'; +import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types'; import { has, sortBy } from 'lodash/fp'; import { IToasts } from '@kbn/core-notifications-browser'; import { getIlmPhase } from './data_quality_panel/pattern/helpers'; @@ -24,6 +21,7 @@ import type { EnrichedFieldMetadata, ErrorSummary, IlmPhase, + MeteringStatsIndex, PartitionedFieldMetadata, PartitionedFieldMetadataStats, PatternRollup, @@ -42,7 +40,7 @@ export const getIndexNames = ({ ilmExplain: Record | null; ilmPhases: string[]; isILMAvailable: boolean; - stats: Record | null; + stats: Record | null; }): string[] => { if (((isILMAvailable && ilmExplain != null) || !isILMAvailable) && stats != null) { const allIndexNames = Object.keys(stats); @@ -270,15 +268,15 @@ export const getDocsCount = ({ stats, }: { indexName: string; - stats: Record | null; -}): number => (stats && stats[indexName]?.primaries?.docs?.count) ?? 0; + stats: Record | null; +}): number => (stats && stats[indexName]?.num_docs) ?? 0; export const getIndexId = ({ indexName, stats, }: { indexName: string; - stats: Record | null; + stats: Record | null; }): string | null | undefined => stats && stats[indexName]?.uuid; export const getSizeInBytes = ({ @@ -286,15 +284,15 @@ export const getSizeInBytes = ({ stats, }: { indexName: string; - stats: Record | null; -}): number => (stats && stats[indexName]?.primaries?.store?.total_data_set_size_in_bytes) ?? 0; + stats: Record | null; +}): number | undefined => (stats && stats[indexName]?.size_in_bytes) ?? undefined; export const getTotalDocsCount = ({ indexNames, stats, }: { indexNames: string[]; - stats: Record | null; + stats: Record | null; }): number => indexNames.reduce( (acc: number, indexName: string) => acc + getDocsCount({ stats, indexName }), @@ -306,12 +304,22 @@ export const getTotalSizeInBytes = ({ stats, }: { indexNames: string[]; - stats: Record | null; -}): number => - indexNames.reduce( - (acc: number, indexName: string) => acc + getSizeInBytes({ stats, indexName }), - 0 - ); + stats: Record | null; +}): number | undefined => { + let sum; + for (let i = 0; i < indexNames.length; i++) { + const currentSizeInBytes = getSizeInBytes({ stats, indexName: indexNames[i] }); + if (currentSizeInBytes != null) { + if (sum == null) { + sum = 0; + } + sum += currentSizeInBytes; + } else { + return undefined; + } + } + return sum; +}; export const EMPTY_STAT = '--'; @@ -498,7 +506,7 @@ export const formatStorageResult = ({ ilmPhase: result.ilmPhase, markdownComments: result.markdownComments, ecsVersion: report.ecsVersion, - indexId: report.indexId, + indexId: report.indexId ?? '', // ---> we don't have this field when isILMAvailable is false error: result.error, }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts index 513d034f5226..984bf9f117de 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts @@ -36,14 +36,9 @@ export const alertIndexNoResults: PatternRollup = { sizeInBytes: 6423408623, stats: { '.internal.alerts-security.alerts-default-000001': { - health: 'green', - status: 'open', - total: { - docs: { - count: 25914, - deleted: 0, - }, - }, + name: '.internal.alerts-security.alerts-default-000001', + num_docs: 25914, + size_in_bytes: 6423408623, }, }, }; @@ -89,14 +84,9 @@ export const alertIndexWithAllResults: PatternRollup = { sizeInBytes: 29717961631, stats: { '.internal.alerts-security.alerts-default-000001': { - health: 'green', - status: 'open', - total: { - docs: { - count: 26093, - deleted: 0, - }, - }, + name: '.internal.alerts-security.alerts-default-000001', + num_docs: 26093, + size_in_bytes: 0, }, }, }; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts index 15bb39a01714..6f3c7b008a5a 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts @@ -49,57 +49,21 @@ export const auditbeatNoResults: PatternRollup = { stats: { '.ds-auditbeat-8.6.1-2023.02.07-000001': { uuid: 'YpxavlUVTw2x_E_QtADrpg', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 18791790, - total_data_set_size_in_bytes: 18791790, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 19123, - deleted: 0, - }, - }, + size_in_bytes: 18791790, + num_docs: 19123, + name: '.ds-auditbeat-8.6.1-2023.02.07-000001', }, 'auditbeat-custom-empty-index-1': { uuid: 'Iz5FJjsLQla34mD6kBAQBw', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 247, - total_data_set_size_in_bytes: 247, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 0, - deleted: 0, - }, - }, + size_in_bytes: 247, + num_docs: 0, + name: 'auditbeat-custom-empty-index-1', }, 'auditbeat-custom-index-1': { uuid: 'xJvgb2QCQPSjlr7UnW8tFA', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 28409, - total_data_set_size_in_bytes: 28409, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 4, - deleted: 0, - }, - }, + size_in_bytes: 28409, + num_docs: 4, + name: 'auditbeat-custom-index-1', }, }, }; @@ -184,57 +148,21 @@ export const auditbeatWithAllResults: PatternRollup = { stats: { '.ds-auditbeat-8.6.1-2023.02.07-000001': { uuid: 'YpxavlUVTw2x_E_QtADrpg', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 18791790, - total_data_set_size_in_bytes: 18791790, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 19123, - deleted: 0, - }, - }, + size_in_bytes: 18791790, + num_docs: 19123, + name: '.ds-auditbeat-8.6.1-2023.02.07-000001', }, 'auditbeat-custom-empty-index-1': { uuid: 'Iz5FJjsLQla34mD6kBAQBw', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 247, - total_data_set_size_in_bytes: 247, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 0, - deleted: 0, - }, - }, + size_in_bytes: 247, + num_docs: 0, + name: 'auditbeat-custom-empty-index-1', }, 'auditbeat-custom-index-1': { uuid: 'xJvgb2QCQPSjlr7UnW8tFA', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 28409, - total_data_set_size_in_bytes: 28409, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 4, - deleted: 0, - }, - }, + size_in_bytes: 28409, + num_docs: 4, + name: 'auditbeat-custom-index-1', }, }, }; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts index 339f1993e292..ae6199c8ae17 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts @@ -47,39 +47,15 @@ export const packetbeatNoResults: PatternRollup = { stats: { '.ds-packetbeat-8.6.1-2023.02.04-000001': { uuid: 'x5Uuw4j4QM2YidHLNixCwg', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 512194751, - total_data_set_size_in_bytes: 512194751, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 1628343, - deleted: 0, - }, - }, + name: '.ds-packetbeat-8.6.1-2023.02.04-000001', + num_docs: 1628343, + size_in_bytes: 512194751, }, '.ds-packetbeat-8.5.3-2023.02.04-000001': { uuid: 'we0vNWm2Q6iz6uHubyHS6Q', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 584326147, - total_data_set_size_in_bytes: 584326147, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 1630289, - deleted: 0, - }, - }, + size_in_bytes: 584326147, + name: '.ds-packetbeat-8.5.3-2023.02.04-000001', + num_docs: 1630289, }, }, }; @@ -150,37 +126,15 @@ export const packetbeatWithSomeErrors: PatternRollup = { stats: { '.ds-packetbeat-8.6.1-2023.02.04-000001': { uuid: 'x5Uuw4j4QM2YidHLNixCwg', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 512194751, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 1628343, - deleted: 0, - }, - }, + size_in_bytes: 731583142, + name: '.ds-packetbeat-8.6.1-2023.02.04-000001', + num_docs: 1628343, }, '.ds-packetbeat-8.5.3-2023.02.04-000001': { uuid: 'we0vNWm2Q6iz6uHubyHS6Q', - health: 'yellow', - primaries: { - store: { - size_in_bytes: 584326147, - reserved_in_bytes: 0, - }, - }, - status: 'open', - total: { - docs: { - count: 1630289, - deleted: 0, - }, - }, + size_in_bytes: 584326147, + name: '.ds-packetbeat-8.5.3-2023.02.04-000001', + num_docs: 1630289, }, }, }; @@ -244,553 +198,15 @@ export const mockPacketbeatPatternRollup: PatternRollup = { stats: { '.ds-packetbeat-8.6.1-2023.02.04-000001': { uuid: 'x5Uuw4j4QM2YidHLNixCwg', - health: 'yellow', - status: 'open', - primaries: { - docs: { - count: 1628343, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 731583142, - total_data_set_size_in_bytes: 731583142, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 120726, - query_time_in_millis: 234865, - query_current: 0, - fetch_total: 109324, - fetch_time_in_millis: 500584, - fetch_current: 0, - scroll_total: 10432, - scroll_time_in_millis: 3874632, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 1, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 1, - }, - query_cache: { - memory_size_in_bytes: 8316098, - total_count: 34248343, - hit_count: 3138879, - miss_count: 31109464, - cache_size: 4585, - cache_count: 4585, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 12424, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 19, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 304, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 606298841, - }, - request_cache: { - memory_size_in_bytes: 89216, - evictions: 0, - hit_count: 704, - miss_count: 38, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, - total: { - docs: { - count: 1628343, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 731583142, - total_data_set_size_in_bytes: 731583142, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 120726, - query_time_in_millis: 234865, - query_current: 0, - fetch_total: 109324, - fetch_time_in_millis: 500584, - fetch_current: 0, - scroll_total: 10432, - scroll_time_in_millis: 3874632, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 1, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 1, - }, - query_cache: { - memory_size_in_bytes: 8316098, - total_count: 34248343, - hit_count: 3138879, - miss_count: 31109464, - cache_size: 4585, - cache_count: 4585, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 12424, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 19, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 304, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 606298841, - }, - request_cache: { - memory_size_in_bytes: 89216, - evictions: 0, - hit_count: 704, - miss_count: 38, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, + size_in_bytes: 731583142, + name: '.ds-packetbeat-8.6.1-2023.02.04-000001', + num_docs: 1628343, }, '.ds-packetbeat-8.5.3-2023.02.04-000001': { uuid: 'we0vNWm2Q6iz6uHubyHS6Q', - health: 'yellow', - status: 'open', - primaries: { - docs: { - count: 1630289, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 733175040, - total_data_set_size_in_bytes: 733175040, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 120726, - query_time_in_millis: 248138, - query_current: 0, - fetch_total: 109484, - fetch_time_in_millis: 500514, - fetch_current: 0, - scroll_total: 10432, - scroll_time_in_millis: 3871379, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 2, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 1, - }, - query_cache: { - memory_size_in_bytes: 5387543, - total_count: 24212135, - hit_count: 2223357, - miss_count: 21988778, - cache_size: 3275, - cache_count: 3275, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 12336, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 20, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 320, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 606298805, - }, - request_cache: { - memory_size_in_bytes: 89320, - evictions: 0, - hit_count: 704, - miss_count: 38, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, - total: { - docs: { - count: 1630289, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 733175040, - total_data_set_size_in_bytes: 733175040, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 120726, - query_time_in_millis: 248138, - query_current: 0, - fetch_total: 109484, - fetch_time_in_millis: 500514, - fetch_current: 0, - scroll_total: 10432, - scroll_time_in_millis: 3871379, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 2, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 1, - }, - query_cache: { - memory_size_in_bytes: 5387543, - total_count: 24212135, - hit_count: 2223357, - miss_count: 21988778, - cache_size: 3275, - cache_count: 3275, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 12336, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 20, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 320, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 606298805, - }, - request_cache: { - memory_size_in_bytes: 89320, - evictions: 0, - hit_count: 704, - miss_count: 38, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, + size_in_bytes: 733175040, + name: '.ds-packetbeat-8.5.3-2023.02.04-000001', + num_docs: 1630289, }, }, }; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats.tsx index 14465e815ad4..08d7cd759116 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats.tsx @@ -5,832 +5,25 @@ * 2.0. */ -import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; +import { MeteringStatsIndex } from '../../types'; -export const mockStats: Record = { +export const mockStats: Record = { '.ds-packetbeat-8.6.1-2023.02.04-000001': { uuid: 'x5Uuw4j4QM2YidHLNixCwg', - health: 'yellow', - status: 'open', - primaries: { - docs: { - count: 1628343, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 731583142, - total_data_set_size_in_bytes: 731583142, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 32, - query_time_in_millis: 111, - query_current: 0, - fetch_total: 32, - fetch_time_in_millis: 0, - fetch_current: 0, - scroll_total: 0, - scroll_time_in_millis: 0, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 15, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 15, - }, - query_cache: { - memory_size_in_bytes: 0, - total_count: 301, - hit_count: 0, - miss_count: 301, - cache_size: 0, - cache_count: 0, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 1080, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 19, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 304, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 136482466, - }, - request_cache: { - memory_size_in_bytes: 3680, - evictions: 0, - hit_count: 28, - miss_count: 4, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, - total: { - docs: { - count: 1628343, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 731583142, - total_data_set_size_in_bytes: 731583142, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 32, - query_time_in_millis: 111, - query_current: 0, - fetch_total: 32, - fetch_time_in_millis: 0, - fetch_current: 0, - scroll_total: 0, - scroll_time_in_millis: 0, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 15, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 15, - }, - query_cache: { - memory_size_in_bytes: 0, - total_count: 301, - hit_count: 0, - miss_count: 301, - cache_size: 0, - cache_count: 0, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 1080, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 19, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 304, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 136482466, - }, - request_cache: { - memory_size_in_bytes: 3680, - evictions: 0, - hit_count: 28, - miss_count: 4, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, + num_docs: 1628343, + size_in_bytes: 731583142, + name: '.ds-packetbeat-8.6.1-2023.02.04-000001', }, '.ds-packetbeat-8.5.3-2023.02.04-000001': { uuid: 'we0vNWm2Q6iz6uHubyHS6Q', - health: 'yellow', - status: 'open', - primaries: { - docs: { - count: 1630289, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 733175040, - total_data_set_size_in_bytes: 733175040, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 32, - query_time_in_millis: 111, - query_current: 0, - fetch_total: 32, - fetch_time_in_millis: 0, - fetch_current: 0, - scroll_total: 0, - scroll_time_in_millis: 0, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 2, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 2, - }, - query_cache: { - memory_size_in_bytes: 0, - total_count: 203, - hit_count: 0, - miss_count: 203, - cache_size: 0, - cache_count: 0, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 1168, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 20, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 320, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 136482425, - }, - request_cache: { - memory_size_in_bytes: 3688, - evictions: 0, - hit_count: 28, - miss_count: 4, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, - total: { - docs: { - count: 1630289, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 733175040, - total_data_set_size_in_bytes: 733175040, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 32, - query_time_in_millis: 111, - query_current: 0, - fetch_total: 32, - fetch_time_in_millis: 0, - fetch_current: 0, - scroll_total: 0, - scroll_time_in_millis: 0, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 2, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 2, - }, - query_cache: { - memory_size_in_bytes: 0, - total_count: 203, - hit_count: 0, - miss_count: 203, - cache_size: 0, - cache_count: 0, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 1168, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 20, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 320, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 136482425, - }, - request_cache: { - memory_size_in_bytes: 3688, - evictions: 0, - hit_count: 28, - miss_count: 4, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, + num_docs: 1630289, + size_in_bytes: 733175040, + name: '.ds-packetbeat-8.5.3-2023.02.04-000001', }, 'auditbeat-custom-index-1': { uuid: 'uyJDDqGrRQqdBTN0mCF-iw', - health: 'yellow', - status: 'open', - primaries: { - docs: { - count: 4, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 28413, - total_data_set_size_in_bytes: 28413, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 24, - query_time_in_millis: 5, - query_current: 0, - fetch_total: 24, - fetch_time_in_millis: 0, - fetch_current: 0, - scroll_total: 0, - scroll_time_in_millis: 0, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 0, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 0, - }, - query_cache: { - memory_size_in_bytes: 58, - total_count: 0, - hit_count: 0, - miss_count: 0, - cache_size: 0, - cache_count: 0, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 608, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 4, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 0, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 79289897, - }, - request_cache: { - memory_size_in_bytes: 3760, - evictions: 0, - hit_count: 20, - miss_count: 4, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, - total: { - docs: { - count: 4, - deleted: 0, - }, - shard_stats: { - total_count: 1, - }, - store: { - size_in_bytes: 28413, - total_data_set_size_in_bytes: 28413, - reserved_in_bytes: 0, - }, - indexing: { - index_total: 0, - index_time_in_millis: 0, - index_current: 0, - index_failed: 0, - delete_total: 0, - delete_time_in_millis: 0, - delete_current: 0, - noop_update_total: 0, - is_throttled: false, - throttle_time_in_millis: 0, - }, - get: { - total: 0, - time_in_millis: 0, - exists_total: 0, - exists_time_in_millis: 0, - missing_total: 0, - missing_time_in_millis: 0, - current: 0, - }, - search: { - open_contexts: 0, - query_total: 24, - query_time_in_millis: 5, - query_current: 0, - fetch_total: 24, - fetch_time_in_millis: 0, - fetch_current: 0, - scroll_total: 0, - scroll_time_in_millis: 0, - scroll_current: 0, - suggest_total: 0, - suggest_time_in_millis: 0, - suggest_current: 0, - }, - merges: { - current: 0, - current_docs: 0, - current_size_in_bytes: 0, - total: 0, - total_time_in_millis: 0, - total_docs: 0, - total_size_in_bytes: 0, - total_stopped_time_in_millis: 0, - total_throttled_time_in_millis: 0, - total_auto_throttle_in_bytes: 20971520, - }, - refresh: { - total: 2, - total_time_in_millis: 0, - external_total: 2, - external_total_time_in_millis: 0, - listeners: 0, - }, - flush: { - total: 1, - periodic: 1, - total_time_in_millis: 0, - }, - warmer: { - current: 0, - total: 1, - total_time_in_millis: 0, - }, - query_cache: { - memory_size_in_bytes: 58, - total_count: 0, - hit_count: 0, - miss_count: 0, - cache_size: 0, - cache_count: 0, - evictions: 0, - }, - fielddata: { - memory_size_in_bytes: 608, - evictions: 0, - }, - completion: { - size_in_bytes: 0, - }, - segments: { - count: 4, - memory_in_bytes: 0, - terms_memory_in_bytes: 0, - stored_fields_memory_in_bytes: 0, - term_vectors_memory_in_bytes: 0, - norms_memory_in_bytes: 0, - points_memory_in_bytes: 0, - doc_values_memory_in_bytes: 0, - index_writer_memory_in_bytes: 0, - version_map_memory_in_bytes: 0, - fixed_bit_set_memory_in_bytes: 0, - max_unsafe_auto_id_timestamp: -1, - file_sizes: {}, - }, - translog: { - operations: 0, - size_in_bytes: 55, - uncommitted_operations: 0, - uncommitted_size_in_bytes: 55, - earliest_last_modified_age: 79289897, - }, - request_cache: { - memory_size_in_bytes: 3760, - evictions: 0, - hit_count: 20, - miss_count: 4, - }, - recovery: { - current_as_source: 0, - current_as_target: 0, - throttle_time_in_millis: 0, - }, - bulk: { - total_operations: 0, - total_time_in_millis: 0, - total_size_in_bytes: 0, - avg_time_in_millis: 0, - avg_size_in_bytes: 0, - }, - }, + num_docs: 4, + size_in_bytes: 28413, + name: 'auditbeat-custom-index-1', }, }; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_auditbeat_index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_auditbeat_index.tsx new file mode 100644 index 000000000000..4794de530fc4 --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_auditbeat_index.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MeteringStatsIndex } from '../../types'; + +export const mockStatsPacketbeatIndex: Record = { + '.ds-packetbeat-8.6.1-2023.02.04-000001': { + uuid: 'x5Uuw4j4QM2YidHLNixCwg', + num_docs: 1628343, + size_in_bytes: 731583142, + name: '.ds-packetbeat-8.6.1-2023.02.04-000001', + }, + '.ds-packetbeat-8.5.3-2023.02.04-000001': { + uuid: 'we0vNWm2Q6iz6uHubyHS6Q', + num_docs: 1630289, + size_in_bytes: 733175040, + name: '.ds-packetbeat-8.5.3-2023.02.04-000001', + }, +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_packetbeat_index.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_packetbeat_index.ts new file mode 100644 index 000000000000..de202d4e18cc --- /dev/null +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_packetbeat_index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MeteringStatsIndex } from '../../types'; + +export const mockStatsAuditbeatIndex: Record = { + 'auditbeat-custom-index-1': { + uuid: 'jRlr6H_jSAysOLZ6KynoCQ', + size_in_bytes: 28425, + name: 'auditbeat-custom-index-1', + num_docs: 4, + }, +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts index f462777e1fc0..8ebb273a2b4b 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts @@ -8,7 +8,6 @@ import type { IlmExplainLifecycleLifecycleExplain, IndicesGetMappingIndexMappingRecord, - IndicesStatsIndicesStats, } from '@elastic/elasticsearch/lib/api/types'; import type { Direction } from '@elastic/eui'; @@ -119,7 +118,7 @@ export interface PatternRollup { pattern: string; results: Record | undefined; sizeInBytes: number | undefined; - stats: Record | null; + stats: Record | null; } export interface CheckIndexRequest { @@ -175,10 +174,32 @@ export interface SelectedIndex { pattern: string; } +export interface MeteringStatsIndex { + uuid?: string; + name: string; + num_docs: number | null; + size_in_bytes: number | null; + data_stream?: string; +} + +export interface MeteringIndicesStatsResponse { + _shards: { + total: number; + successful: number; + failed: number; + }; + indices: MeteringStatsIndex[]; + datastreams: Array<{ name: string; num_docs: number; size_in_bytes: number }>; + total: { + num_docs: number; + size_in_bytes: number; + }; +} + export type DataQualityIndexCheckedParams = DataQualityCheckAllCompletedParams & { errorCount?: number; ilmPhase?: string; - indexId: string; + indexId?: string; indexName: string; sameFamilyFields?: string[]; unallowedMappingFields?: string[]; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts index 8c85cc115268..6d83a4f6fd0f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts @@ -21,9 +21,8 @@ import { packetbeatNoResults, packetbeatWithSomeErrors, } from '../mock/pattern_rollup/mock_packetbeat_pattern_rollup'; -import { DataQualityCheckResult, PatternRollup } from '../types'; +import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../types'; import { EMPTY_STAT } from '../helpers'; -import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; import { mockPartitionedFieldMetadata } from '../mock/partitioned_field_metadata/mock_partitioned_field_metadata'; import { alertIndexWithAllResults } from '../mock/pattern_rollup/mock_alerts_pattern_rollup'; import { EcsVersion } from '@elastic/ecs'; @@ -168,14 +167,14 @@ describe('helpers', () => { }); describe('updateResultOnCheckCompleted', () => { - const packetbeatStats861: IndicesStatsIndicesStats = + const packetbeatStats861: MeteringStatsIndex = mockPacketbeatPatternRollup.stats != null ? mockPacketbeatPatternRollup.stats['.ds-packetbeat-8.6.1-2023.02.04-000001'] - : {}; - const packetbeatStats853: IndicesStatsIndicesStats = + : ({} as MeteringStatsIndex); + const packetbeatStats853: MeteringStatsIndex = mockPacketbeatPatternRollup.stats != null ? mockPacketbeatPatternRollup.stats['.ds-packetbeat-8.5.3-2023.02.04-000001'] - : {}; + : ({} as MeteringStatsIndex); test('it returns the updated rollups', () => { expect( diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx index fb13413dbbd4..88c1138288a3 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx @@ -9,7 +9,7 @@ import { renderHook } from '@testing-library/react-hooks'; import React from 'react'; import { DataQualityProvider } from '../data_quality_panel/data_quality_context'; -import { mockStatsGreenIndex } from '../mock/stats/mock_stats_green_index'; +import { mockStatsAuditbeatIndex } from '../mock/stats/mock_stats_packetbeat_index'; import { ERROR_LOADING_STATS } from '../translations'; import { useStats, UseStats } from '.'; import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; @@ -65,7 +65,7 @@ describe('useStats', () => { }; beforeEach(async () => { - mockHttpFetch.mockResolvedValue(mockStatsGreenIndex); + mockHttpFetch.mockResolvedValue(mockStatsAuditbeatIndex); const { waitForNextUpdate } = renderHook(() => useStats({ pattern, startDate, endDate }), { wrapper: ContextWrapperILMNotAvailable, @@ -81,7 +81,7 @@ describe('useStats', () => { let statsResult: UseStats | undefined; beforeEach(async () => { - mockHttpFetch.mockResolvedValue(mockStatsGreenIndex); + mockHttpFetch.mockResolvedValue(mockStatsAuditbeatIndex); const { result, waitForNextUpdate } = renderHook(() => useStats(params), { wrapper: ContextWrapper, @@ -91,7 +91,7 @@ describe('useStats', () => { }); test('it returns the expected stats', async () => { - expect(statsResult?.stats).toEqual(mockStatsGreenIndex); + expect(statsResult?.stats).toEqual(mockStatsAuditbeatIndex); }); test('it returns loading: false, because the data has loaded', async () => { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx index fce940de15f7..90af7ded0ba0 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.tsx @@ -5,18 +5,18 @@ * 2.0. */ -import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; import { useEffect, useState } from 'react'; import { HttpFetchQuery } from '@kbn/core/public'; import { useDataQualityContext } from '../data_quality_panel/data_quality_context'; import * as i18n from '../translations'; import { INTERNAL_API_VERSION } from '../helpers'; +import { MeteringStatsIndex } from '../types'; const STATS_ENDPOINT = '/internal/ecs_data_quality_dashboard/stats'; export interface UseStats { - stats: Record | null; + stats: Record | null; error: string | null; loading: boolean; } @@ -31,7 +31,7 @@ export const useStats = ({ startDate?: string | null; }): UseStats => { const { httpFetch, isILMAvailable } = useDataQualityContext(); - const [stats, setStats] = useState | null>(null); + const [stats, setStats] = useState | null>(null); const [error, setError] = useState(null); const [loading, setLoading] = useState(true); @@ -51,7 +51,7 @@ export const useStats = ({ } } - const response = await httpFetch>( + const response = await httpFetch>( `${STATS_ENDPOINT}/${encodedIndexName}`, { version: INTERNAL_API_VERSION, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/common/types.ts b/x-pack/plugins/ecs_data_quality_dashboard/common/types.ts new file mode 100644 index 000000000000..9610c69c7cae --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/common/types.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +export interface MeteringStatsIndex { + uuid?: string; + name: string; + num_docs: number | null; + size_in_bytes: number | null; + data_stream?: string; +} + +export interface MeteringIndicesStatsResponse { + _shards: { + total: number; + successful: number; + failed: number; + }; + indices: MeteringStatsIndex[] | null | undefined; + datastreams: Array<{ name: string; num_docs: number }>; + total: { + num_docs: number; + }; +} diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_metering_stats_index.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_metering_stats_index.ts new file mode 100644 index 000000000000..75ee7381eee5 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_metering_stats_index.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const mockMeteringStatsIndex = { + _shards: { + total: 4, + successful: 2, + failed: 0, + }, + indices: [ + { + name: '.ds-my-datastream-03-02-2024-00001', + num_docs: 3, + size_in_bytes: 15785, + datastream: 'my-datastream', + }, + { + name: 'my-index-000001', + num_docs: 2, + size_in_bytes: 11462, + }, + ], + datastreams: [ + { + name: 'my-datastream', + num_docs: 6, + size_in_bytes: 31752, + }, + ], + total: { + num_docs: 8, + size_in_bytes: 47214, + }, +}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_green_index.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_green_index.ts similarity index 98% rename from x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_green_index.ts rename to x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_green_index.ts index 74b10123e79f..399d4d1b4597 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_green_index.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_green_index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; /** * In a deployment where indices have a `green` health status, because there diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_yellow_index.tsx b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_yellow_index.ts similarity index 98% rename from x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_yellow_index.tsx rename to x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_yellow_index.ts index 49b5b1935218..8f699983b651 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/stats/mock_stats_yellow_index.tsx +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/mock_stats_yellow_index.ts @@ -5,12 +5,11 @@ * 2.0. */ -import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; /** * In a deployment where indices have a `yellow` health status, the - * [`_stats`](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-stats.html) - * API returns, (for an arbitrary index), results where the index's + @@ -14,555 +14,17 @@ import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; * `primaries.docs.count` and `total.docs.count` have the same value, per this * mock `_stats` API output */ diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts index c02d5361bdc8..536fd461c61c 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts @@ -5,8 +5,12 @@ * 2.0. */ -import type { IndicesStatsResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { + IndicesStatsIndicesStats, + IndicesStatsResponse, +} from '@elastic/elasticsearch/lib/api/types'; import type { IScopedClusterClient } from '@kbn/core/server'; +import type { MeteringIndicesStatsResponse, MeteringStatsIndex } from '../../common/types'; export const fetchStats = ( client: IScopedClusterClient, @@ -16,3 +20,54 @@ export const fetchStats = ( expand_wildcards: ['open'], index: indexPattern, }); + +export const parseIndicesStats = ( + statsIndices: Record | undefined +) => + Object.entries(statsIndices ?? {}).reduce>( + (acc, [key, value]) => { + acc[key] = { + uuid: value.uuid, + name: key, + num_docs: value?.primaries?.docs?.count ?? null, + size_in_bytes: value?.primaries?.store?.size_in_bytes ?? null, + }; + return acc; + }, + {} + ); + +export const fetchMeteringStats = ( + client: IScopedClusterClient, + indexPattern: string, + secondaryAuthorization?: string | string[] | undefined +): Promise => + client.asInternalUser.transport.request( + { + method: 'GET', + path: `/_metering/stats/${indexPattern}`, + }, + { headers: { 'es-secondary-authorization': secondaryAuthorization } } + ); + +export const parseMeteringStats = (meteringStatsIndices: MeteringStatsIndex[]) => + meteringStatsIndices.reduce>((acc, curr) => { + acc[curr.name] = curr; + return acc; + }, {}); + +export const pickAvailableMeteringStats = ( + indicesBuckets: Array<{ key: string }>, + meteringStatsIndices: Record +) => + indicesBuckets.reduce((acc: Record, { key }: { key: string }) => { + if (meteringStatsIndices?.[key]) { + acc[key] = { + name: meteringStatsIndices?.[key].name, + num_docs: meteringStatsIndices?.[key].num_docs, + size_in_bytes: null, // We don't have size_in_bytes intentionally when ILM is not available + data_stream: meteringStatsIndices?.[key].data_stream, + }; + } + return acc; + }, {}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts index 729c795e0665..f3ff5ec256ad 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts @@ -6,7 +6,7 @@ */ import { GET_INDEX_STATS } from '../../common/constants'; -import { fetchAvailableIndices, fetchStats } from '../lib'; +import { fetchAvailableIndices, fetchMeteringStats, fetchStats } from '../lib'; import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; @@ -14,11 +14,19 @@ import { requestContextMock } from '../__mocks__/request_context'; import { getIndexStatsRoute } from './get_index_stats'; import type { MockedLogger } from '@kbn/logging-mocks'; import { loggerMock } from '@kbn/logging-mocks'; +import { mockStatsGreenIndex } from '../__mocks__/mock_stats_green_index'; +import { mockStatsYellowIndex } from '../__mocks__/mock_stats_yellow_index'; +import { mockMeteringStatsIndex } from '../__mocks__/mock_metering_stats_index'; -jest.mock('../lib', () => ({ - fetchStats: jest.fn(), - fetchAvailableIndices: jest.fn(), -})); +jest.mock('../lib', () => { + const originalModule = jest.requireActual('../lib'); + return { + ...originalModule, + fetchStats: jest.fn(), + fetchMeteringStats: jest.fn(), + fetchAvailableIndices: jest.fn(), + }; +}); describe('getIndexStatsRoute route', () => { let server: ReturnType; @@ -49,10 +57,41 @@ describe('getIndexStatsRoute route', () => { getIndexStatsRoute(server.router, logger); }); - test('Returns index stats', async () => { - const mockIndices = { 'auditbeat-7.15.1-2022.12.06-000001': {} }; + test('Returns index stats when index health is green', async () => { + const mockIndices = { + 'auditbeat-custom-index-1': { + name: 'auditbeat-custom-index-1', + num_docs: 4, + size_in_bytes: 28425, + uuid: 'jRlr6H_jSAysOLZ6KynoCQ', + }, + }; (fetchStats as jest.Mock).mockResolvedValue({ - indices: mockIndices, + indices: mockStatsGreenIndex, + }); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(200); + expect(response.body).toEqual(mockIndices); + }); + + test('Returns index stats when index health is yellow', async () => { + const mockIndices = { + '.ds-packetbeat-8.6.1-2023.02.04-000001': { + name: '.ds-packetbeat-8.6.1-2023.02.04-000001', + num_docs: 1628343, + size_in_bytes: 731583142, + uuid: 'x5Uuw4j4QM2YidHLNixCwg', + }, + '.ds-packetbeat-8.5.3-2023.02.04-000001': { + name: '.ds-packetbeat-8.5.3-2023.02.04-000001', + num_docs: 1630289, + size_in_bytes: 733175040, + uuid: 'we0vNWm2Q6iz6uHubyHS6Q', + }, + }; + (fetchStats as jest.Mock).mockResolvedValue({ + indices: mockStatsYellowIndex, }); const response = await server.inject(req, requestContextMock.convertContext(context)); @@ -81,9 +120,8 @@ describe('getIndexStatsRoute route', () => { }, }); - const mockIndices = { 'auditbeat-7.15.1-2022.12.06-000001': {} }; (fetchStats as jest.Mock).mockResolvedValue({ - indices: mockIndices, + indices: mockMeteringStatsIndex, }); const response = await server.inject(request, requestContextMock.convertContext(context)); @@ -107,18 +145,19 @@ describe('getIndexStatsRoute route', () => { }); const mockIndices = { - 'auditbeat-7.15.1-2022.12.06-000001': {}, - 'auditbeat-7.15.1-2022.11.06-000001': {}, + 'my-index-000001': { + name: 'my-index-000001', + num_docs: 2, + size_in_bytes: null, + }, }; - (fetchStats as jest.Mock).mockResolvedValue({ - indices: mockIndices, - }); + (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex); (fetchAvailableIndices as jest.Mock).mockResolvedValue({ aggregations: { index: { buckets: [ { - key: 'auditbeat-7.15.1-2022.12.06-000001', + key: 'my-index-000001', }, ], }, @@ -127,7 +166,64 @@ describe('getIndexStatsRoute route', () => { const response = await server.inject(request, requestContextMock.convertContext(context)); expect(response.status).toEqual(200); - expect(response.body).toEqual({ 'auditbeat-7.15.1-2022.12.06-000001': {} }); + expect(response.body).toEqual(mockIndices); + }); + + test('returns an empty object when "meteringStats indices" are not available', async () => { + const request = requestMock.create({ + method: 'get', + path: GET_INDEX_STATS, + params: { + pattern: `auditbeat-*`, + }, + query: { + isILMAvailable: false, + startDate: `now-7d`, + endDate: `now`, + }, + }); + + const mockIndices = {}; + (fetchMeteringStats as jest.Mock).mockResolvedValue({ indices: undefined }); + (fetchAvailableIndices as jest.Mock).mockResolvedValue({ + aggregations: undefined, + }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + expect(response.status).toEqual(200); + expect(response.body).toEqual(mockIndices); + expect(fetchAvailableIndices).not.toHaveBeenCalled(); + expect(logger.warn).toHaveBeenCalledWith( + `No metering stats indices found under pattern: auditbeat-*` + ); + }); + + test('returns an empty object when "availableIndices" indices are not available', async () => { + const request = requestMock.create({ + method: 'get', + path: GET_INDEX_STATS, + params: { + pattern: `auditbeat-*`, + }, + query: { + isILMAvailable: false, + startDate: `now-7d`, + endDate: `now`, + }, + }); + + const mockIndices = {}; + (fetchMeteringStats as jest.Mock).mockResolvedValue(mockMeteringStatsIndex); + (fetchAvailableIndices as jest.Mock).mockResolvedValue({ + aggregations: undefined, + }); + + const response = await server.inject(request, requestContextMock.convertContext(context)); + expect(response.status).toEqual(200); + expect(response.body).toEqual(mockIndices); + expect(logger.warn).toHaveBeenCalledWith( + `No available indices found under pattern: auditbeat-*, in the given date range: now-7d - now` + ); }); }); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index 69d49b861110..665c178c62cd 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -7,8 +7,14 @@ import { i18n } from '@kbn/i18n'; import type { IRouter, Logger } from '@kbn/core/server'; -import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; -import { fetchStats, fetchAvailableIndices } from '../lib'; +import { + fetchStats, + fetchAvailableIndices, + fetchMeteringStats, + parseIndicesStats, + parseMeteringStats, + pickAvailableMeteringStats, +} from '../lib'; import { buildResponse } from '../lib/build_response'; import { GET_INDEX_STATS, INTERNAL_API_VERSION } from '../../common/constants'; import { buildRouteValidation } from '../schemas/common'; @@ -41,12 +47,14 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { const decodedIndexName = decodeURIComponent(request.params.pattern); - const stats = await fetchStats(client, decodedIndexName); const { isILMAvailable, startDate, endDate } = request.query; if (isILMAvailable === true) { + const stats = await fetchStats(client, decodedIndexName); + const parsedIndices = parseIndicesStats(stats.indices); + return response.ok({ - body: stats.indices, + body: parsedIndices, }); } @@ -57,24 +65,43 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { if (startDate && endDate) { const decodedStartDate = decodeURIComponent(startDate); const decodedEndDate = decodeURIComponent(endDate); + const meteringStats = await fetchMeteringStats( + client, + decodedIndexName, + request.headers.authorization + ); - const indices = await fetchAvailableIndices(esClient, { + if (!meteringStats.indices) { + logger.warn(`No metering stats indices found under pattern: ${decodedIndexName}`); + return response.ok({ + body: {}, + }); + } + + const meteringStatsIndices = parseMeteringStats(meteringStats.indices); + + const availableIndices = await fetchAvailableIndices(esClient, { indexPattern: decodedIndexName, startDate: decodedStartDate, endDate: decodedEndDate, }); - const availableIndices = indices?.aggregations?.index?.buckets?.reduce( - (acc: Record, { key }: { key: string }) => { - if (stats.indices?.[key]) { - acc[key] = stats.indices?.[key]; - } - return acc; - }, - {} + + if (!availableIndices.aggregations?.index?.buckets) { + logger.warn( + `No available indices found under pattern: ${decodedIndexName}, in the given date range: ${decodedStartDate} - ${decodedEndDate}` + ); + return response.ok({ + body: {}, + }); + } + + const indices = pickAvailableMeteringStats( + availableIndices.aggregations.index.buckets, + meteringStatsIndices ); return response.ok({ - body: availableIndices, + body: indices, }); } else { return resp.error({ @@ -89,7 +116,6 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { } } catch (err) { logger.error(JSON.stringify(err)); - return resp.error({ body: err.message ?? API_DEFAULT_ERROR_MESSAGE, statusCode: err.statusCode ?? 500, diff --git a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts index 6d3cb9167cb0..1a3a88cbd2f5 100644 --- a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts +++ b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts @@ -25,7 +25,7 @@ export const dataQualityIndexCheckedEvent: DataQualityTelemetryIndexCheckedEvent type: 'keyword', _meta: { description: 'Index uuid', - optional: false, + optional: true, }, }, indexName: { diff --git a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts index 6ae922d45b95..8e24420aa049 100644 --- a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts +++ b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts @@ -11,7 +11,7 @@ import type { TelemetryEventTypes } from '../../constants'; export type ReportDataQualityIndexCheckedParams = ReportDataQualityCheckAllCompletedParams & { errorCount?: number; ilmPhase?: string; - indexId: string; + indexId?: string; indexName: string; sameFamilyFields?: string[]; unallowedMappingFields?: string[];