github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 09:48:58 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][DQD][Tech Debt] Refactor top level helpers (#191233)

Browse source 
addresses https://github.com/elastic/kibana/issues/190964

Third in the series of PRs to address general DQD tech debt

This one builds on previous 2 PRs 

https://github.com/elastic/kibana/pull/190970
https://github.com/elastic/kibana/pull/190978 

Gist of changes:

- split top level helpers into series of utils/* files
- each utils/ file is named after common behavior it export or works
with.
- cleanup dead code
This commit is contained in:
Karen Grigoryan 2024-08-26 16:12:33 +02:00 • committed by GitHub
parent 110ec27668
commit ad360403bc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
73 changed files with 2644 additions and 2562 deletions
Download patch file Download diff file Expand all files Collapse all files

4
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/constants.ts
View file

@ -38,3 +38,7 @@ export const ilmPhaseOptionsStatic: EuiComboBoxOptionOption[] = [
value: 'unmanaged',
},
];
export const EMPTY_STAT = '--';
export const INTERNAL_API_VERSION = '1';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/index.test.tsx
View file

@ -9,7 +9,7 @@ import numeral from '@elastic/numeral';
import { render, screen, waitFor } from '@testing-library/react';
import React from 'react';
import { EMPTY_STAT } from '../../helpers';
import { EMPTY_STAT } from '../../constants';
import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { packetbeatNoResults } from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';

4
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.test.ts
View file

@ -29,8 +29,8 @@ import { mockDataQualityCheckResult } from '../../../mock/data_quality_check_res
import { auditbeatWithAllResults } from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { mockStats } from '../../../mock/stats/mock_stats';
import { DataQualityCheckResult } from '../../../types';
import { getIndexNames, getTotalDocsCount } from '../../../helpers';
import { IndexSummaryTableItem } from './types';
import { getIndexNames, getPatternDocsCount } from './utils/stats';
const hot: IlmExplainLifecycleLifecycleExplainManaged = {
index: '.ds-packetbeat-8.6.1-2023.02.04-000001',
@ -569,7 +569,7 @@ describe('helpers', () => {
ilmPhases: ['hot', 'unmanaged'],
isILMAvailable,
});
const newDocsCount = getTotalDocsCount({ indexNames: newIndexNames, stats: mockStats });
const newDocsCount = getPatternDocsCount({ indexNames: newIndexNames, stats: mockStats });
test('it returns false when the `patternRollup.docsCount` equals newDocsCount', () => {
expect(
shouldCreatePatternRollup({

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/helpers.ts
View file

@ -16,8 +16,8 @@ import type {
SortConfig,
MeteringStatsIndex,
} from '../../../types';
import { getDocsCount, getSizeInBytes } from '../../../helpers';
import { IndexSummaryTableItem } from './types';
import { getDocsCount, getSizeInBytes } from '../../../utils/stats';
export const isManaged = (
ilmExplainRecord: IlmExplainLifecycleLifecycleExplain | undefined

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_ilm_explain/index.tsx
View file

@ -9,7 +9,7 @@ import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch
import { useEffect, useState } from 'react';
import { useDataQualityContext } from '../../../../../data_quality_context';
import { INTERNAL_API_VERSION } from '../../../../../helpers';
import { INTERNAL_API_VERSION } from '../../../../../constants';
import * as i18n from '../../../../../translations';
import { useIsMounted } from '../../../../../hooks/use_is_mounted';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/hooks/use_stats/index.tsx
View file

@ -10,7 +10,7 @@ import { HttpFetchQuery } from '@kbn/core/public';
import { useDataQualityContext } from '../../../../../data_quality_context';
import * as i18n from '../../../../../translations';
import { INTERNAL_API_VERSION } from '../../../../../helpers';
import { INTERNAL_API_VERSION } from '../../../../../constants';
import { MeteringStatsIndex } from '../../../../../types';
import { useIsMounted } from '../../../../../hooks/use_is_mounted';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.test.tsx
View file

@ -9,7 +9,7 @@ import numeral from '@elastic/numeral';
import { render, screen } from '@testing-library/react';
import React, { ComponentProps } from 'react';
import { EMPTY_STAT } from '../../../helpers';
import { EMPTY_STAT } from '../../../constants';
import {
TestDataQualityProviders,
TestExternalProviders,

13
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index.tsx
View file

@ -18,13 +18,8 @@ import {
shouldCreateIndexNames,
shouldCreatePatternRollup,
} from './helpers';
import {
getIndexNames,
getTotalDocsCount,
getTotalPatternIncompatible,
getTotalPatternIndicesChecked,
getTotalSizeInBytes,
} from '../../../helpers';
import { getTotalPatternIncompatible, getTotalPatternIndicesChecked } from '../../../utils/stats';
import { getIndexNames, getPatternDocsCount, getPatternSizeInBytes } from './utils/stats';
import { LoadingEmptyPrompt } from './loading_empty_prompt';
import { PatternSummary } from './pattern_summary';
import { RemoteClustersCallout } from './remote_clusters_callout';
@ -152,7 +147,7 @@ const PatternComponent: React.FC<Props> = ({
useEffect(() => {
const newIndexNames = getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable });
const newDocsCount = getTotalDocsCount({ indexNames: newIndexNames, stats });
const newDocsCount = getPatternDocsCount({ indexNames: newIndexNames, stats });
if (
shouldCreateIndexNames({
@ -188,7 +183,7 @@ const PatternComponent: React.FC<Props> = ({
pattern,
results: undefined,
sizeInBytes: isILMAvailable
? getTotalSizeInBytes({
? getPatternSizeInBytes({
indexNames: getIndexNames({ stats, ilmExplain, ilmPhases, isILMAvailable }),
stats,
}) ?? 0

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index.tsx
View file

@ -21,9 +21,10 @@ import {
} from '@elastic/eui';
import React, { useCallback, useEffect } from 'react';
import moment from 'moment';
import { getDocsCount, getSizeInBytes } from '../../../../utils/stats';
import { useIndicesCheckContext } from '../../../../contexts/indices_check_context';
import { EMPTY_STAT, getDocsCount, getSizeInBytes } from '../../../../helpers';
import { EMPTY_STAT } from '../../../../constants';
import { MeteringStatsIndex, PatternRollup } from '../../../../types';
import { useDataQualityContext } from '../../../../data_quality_context';
import { IndexProperties } from './index_properties';

44
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.test.ts
View file

@ -5,11 +5,7 @@
* 2.0.
*/
import {
getMappingsProperties,
getSortedPartitionedFieldMetadata,
hasAllDataFetchingCompleted,
} from './helpers';
import { getMappingsProperties, getSortedPartitionedFieldMetadata } from './helpers';
import { mockIndicesGetMappingIndexMappingRecords } from '../../../../../mock/indices_get_mapping_index_mapping_record/mock_indices_get_mapping_index_mapping_record';
import { mockMappingsProperties } from '../../../../../mock/mappings_properties/mock_mappings_properties';
import { EcsFlatTyped } from '../../../../../constants';
@ -250,42 +246,4 @@ describe('helpers', () => {
).toBeNull();
});
});
describe('hasAllDataFetchingCompleted', () => {
test('it returns false when both the mappings and unallowed values are loading', () => {
expect(
hasAllDataFetchingCompleted({
loadingMappings: true,
loadingUnallowedValues: true,
})
).toBe(false);
});
test('it returns false when mappings are loading, and unallowed values are NOT loading', () => {
expect(
hasAllDataFetchingCompleted({
loadingMappings: true,
loadingUnallowedValues: false,
})
).toBe(false);
});
test('it returns false when mappings are NOT loading, and unallowed values are loading', () => {
expect(
hasAllDataFetchingCompleted({
loadingMappings: false,
loadingUnallowedValues: true,
})
).toBe(false);
});
test('it returns true when both the mappings and unallowed values have finished loading', () => {
expect(
hasAllDataFetchingCompleted({
loadingMappings: false,
loadingUnallowedValues: false,
})
).toBe(true);
});
});
});

12
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/helpers.ts
View file

@ -12,13 +12,13 @@ import type {
import { sortBy } from 'lodash/fp';
import { EcsFlatTyped } from '../../../../../constants';
import type { PartitionedFieldMetadata, UnallowedValueCount } from '../../../../../types';
import {
getEnrichedFieldMetadata,
getFieldTypes,
getMissingTimestampFieldMetadata,
getPartitionedFieldMetadata,
} from '../../../../../helpers';
import type { PartitionedFieldMetadata, UnallowedValueCount } from '../../../../../types';
} from './utils/metadata';
export const ALL_TAB_ID = 'allTab';
export const ECS_COMPLIANT_TAB_ID = 'ecsCompliantTab';
@ -90,11 +90,3 @@ export const getMappingsProperties = ({
return null;
};
export const hasAllDataFetchingCompleted = ({
loadingMappings,
loadingUnallowedValues,
}: {
loadingMappings: boolean;
loadingUnallowedValues: boolean;
}): boolean => loadingMappings === false && loadingUnallowedValues === false;

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index.test.tsx
View file

@ -9,7 +9,7 @@ import numeral from '@elastic/numeral';
import { render, screen, waitFor } from '@testing-library/react';
import React from 'react';
import { EMPTY_STAT } from '../../../../../helpers';
import { EMPTY_STAT } from '../../../../../constants';
import { auditbeatWithAllResults } from '../../../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import {
TestDataQualityProviders,

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/compare_fields_table/get_common_table_columns/index.tsx
View file

@ -11,7 +11,6 @@ import React from 'react';
import { SameFamily } from '../same_family';
import { EcsAllowedValues } from '../ecs_allowed_values';
import { getIsInSameFamily } from '../../../../../../../../../helpers';
import { IndexInvalidValues } from '../index_invalid_values';
import { CodeDanger, CodeSuccess } from '../../../../../../../../../styles';
import * as i18n from '../translations';
@ -20,6 +19,7 @@ import type {
EnrichedFieldMetadata,
UnallowedValueCount,
} from '../../../../../../../../../types';
import { getIsInSameFamily } from '../../../../utils/get_is_in_same_family';
export const EMPTY_PLACEHOLDER = '--';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/custom_tab/helpers.test.ts
View file

@ -19,7 +19,7 @@ import {
someField,
} from '../../../../../../../../mock/enriched_field_metadata/mock_enriched_field_metadata';
import { mockPartitionedFieldMetadata } from '../../../../../../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
import { EMPTY_STAT } from '../../../../../../../../helpers';
import { EMPTY_STAT } from '../../../../../../../../constants';
const defaultBytesFormat = '0,0.[0]b';
const formatBytes = (value: number | undefined) =>

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/helpers.tsx
View file

@ -9,10 +9,11 @@ import { EuiBadge } from '@elastic/eui';
import React from 'react';
import styled from 'styled-components';
import { getSizeInBytes } from '../../../../../../../utils/stats';
import { getIncompatibleStatBadgeColor } from '../../../../../../../utils/get_incompatible_stat_badge_color';
import { AllTab } from './all_tab';
import { CustomTab } from './custom_tab';
import { EcsCompliantTab } from './ecs_compliant_tab';
import { getIncompatibleStatBadgeColor, getSizeInBytes } from '../../../../../../../helpers';
import { IncompatibleTab } from './incompatible_tab';
import {
ALL_TAB_ID,

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.test.ts
View file

@ -18,7 +18,7 @@ import {
getIncompatibleValuesFields,
showInvalidCallout,
} from './helpers';
import { EMPTY_STAT } from '../../../../../../../../helpers';
import { EMPTY_STAT } from '../../../../../../../../constants';
import {
DETECTION_ENGINE_RULES_MAY_NOT_MATCH,
MAPPINGS_THAT_CONFLICT_WITH_ECS,

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers.ts
View file

@ -34,7 +34,7 @@ import {
DOCUMENT_VALUES_ACTUAL,
ECS_VALUES_EXPECTED,
} from '../compare_fields_table/translations';
import { getIsInSameFamily } from '../../../../../../../../helpers';
import { getIsInSameFamily } from '../../../utils/get_is_in_same_family';
export const getIncompatibleFieldsMarkdownComment = (incompatible: number): string =>
getMarkdownComment({

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/same_family_tab/helpers.test.ts
View file

@ -13,7 +13,7 @@ import {
getSameFamilyMarkdownComment,
getSameFamilyMarkdownTablesComment,
} from './helpers';
import { EMPTY_STAT } from '../../../../../../../../helpers';
import { EMPTY_STAT } from '../../../../../../../../constants';
import { mockPartitionedFieldMetadata } from '../../../../../../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
import { mockPartitionedFieldMetadataWithSameFamily } from '../../../../../../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_stats_panel/index.tsx
View file

@ -13,7 +13,7 @@ import { DOCS } from '../translations';
import { ILM_PHASE } from '../../../../../../translations';
import { SIZE } from '../../../summary_table/translations';
import { Stat } from '../../../../../../stat';
import { getIlmPhaseDescription } from '../../../../../../helpers';
import { getIlmPhaseDescription } from '../../../../../../utils/get_ilm_phase_description';
const StyledFlexItem = styled(EuiFlexItem)`
border-right: 1px solid ${({ theme }) => theme.eui.euiBorderColor};

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.test.ts
View file

@ -45,7 +45,7 @@ import {
getSummaryTableMarkdownRow,
getTabCountsMarkdownComment,
} from './helpers';
import { EMPTY_STAT } from '../../../../../../helpers';
import { EMPTY_STAT } from '../../../../../../constants';
import { mockAllowedValues } from '../../../../../../mock/allowed_values/mock_allowed_values';
import {
eventCategory,

10
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/markdown/helpers.ts
View file

@ -5,6 +5,10 @@
* 2.0.
*/
import {
getTotalPatternIncompatible,
getTotalPatternIndicesChecked,
} from '../../../../../../utils/stats';
import {
ERRORS_MAY_OCCUR,
ERRORS_CALLOUT_SUMMARY,
@ -15,11 +19,7 @@ import {
THE_FOLLOWING_PRIVILEGES_ARE_REQUIRED,
VIEW_INDEX_METADATA,
} from '../../../../../../data_quality_summary/summary_actions/check_status/errors_popover/translations';
import {
EMPTY_STAT,
getTotalPatternIncompatible,
getTotalPatternIndicesChecked,
} from '../../../../../../helpers';
import { EMPTY_STAT } from '../../../../../../constants';
import { SAME_FAMILY } from '../index_check_fields/tabs/compare_fields_table/same_family/translations';
import {
HOT,

40
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.test.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getIsInSameFamily } from './get_is_in_same_family';
describe('getIsInSameFamily', () => {
test('it returns false when ecsExpectedType is undefined', () => {
expect(getIsInSameFamily({ ecsExpectedType: undefined, type: 'keyword' })).toBe(false);
});
const expectedFamilyMembers: {
[key: string]: string[];
} = {
constant_keyword: ['keyword', 'wildcard'], // `keyword` and `wildcard` in the same family as `constant_keyword`
keyword: ['constant_keyword', 'wildcard'],
match_only_text: ['text'],
text: ['match_only_text'],
wildcard: ['keyword', 'constant_keyword'],
};
const ecsExpectedTypes = Object.keys(expectedFamilyMembers);
ecsExpectedTypes.forEach((ecsExpectedType) => {
const otherMembersOfSameFamily = expectedFamilyMembers[ecsExpectedType];
otherMembersOfSameFamily.forEach((type) =>
test(`it returns true for ecsExpectedType '${ecsExpectedType}' when given '${type}', a type in the same family`, () => {
expect(getIsInSameFamily({ ecsExpectedType, type })).toBe(true);
})
);
test(`it returns false for ecsExpectedType '${ecsExpectedType}' when given 'date', a type NOT in the same family`, () => {
expect(getIsInSameFamily({ ecsExpectedType, type: 'date' })).toBe(false);
});
});
});

43
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/get_is_in_same_family.ts Normal file
View file

@ -0,0 +1,43 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
/**
* Per https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html#_core_datatypes
*
* ```
* Field types are grouped by _family_. Types in the same family have exactly
* the same search behavior but may have different space usage or
* performance characteristics.
*
* Currently, there are two type families, `keyword` and `text`. Other type
* families have only a single field type. For example, the `boolean` type
* family consists of one field type: `boolean`.
* ```
*/
export const fieldTypeFamilies: Record<string, Set<string>> = {
keyword: new Set(['keyword', 'constant_keyword', 'wildcard']),
text: new Set(['text', 'match_only_text']),
};
export const getIsInSameFamily = ({
ecsExpectedType,
type,
}: {
ecsExpectedType: string | undefined;
type: string;
}): boolean => {
if (ecsExpectedType != null) {
const allFamilies = Object.values(fieldTypeFamilies);
return allFamilies.reduce<boolean>(
(acc, family) => (acc !== true ? family.has(ecsExpectedType) && family.has(type) : acc),
false
);
} else {
return false;
}
};

402
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.test.ts Normal file
View file

@ -0,0 +1,402 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { omit } from 'lodash/fp';
import {
EnrichedFieldMetadata,
PartitionedFieldMetadata,
UnallowedValueCount,
} from '../../../../../../types';
import { mockMappingsProperties } from '../../../../../../mock/mappings_properties/mock_mappings_properties';
import {
FieldType,
getEnrichedFieldMetadata,
getFieldTypes,
getMissingTimestampFieldMetadata,
getPartitionedFieldMetadata,
isMappingCompatible,
} from './metadata';
import { EcsFlatTyped } from '../../../../../../constants';
import {
hostNameWithTextMapping,
hostNameKeyword,
someField,
someFieldKeyword,
sourceIpWithTextMapping,
sourceIpKeyword,
sourcePort,
timestamp,
eventCategoryWithUnallowedValues,
} from '../../../../../../mock/enriched_field_metadata/mock_enriched_field_metadata';
describe('getFieldTypes', () => {
const expected = [
{
field: '@timestamp',
type: 'date',
},
{
field: 'event.category',
type: 'keyword',
},
{
field: 'host.name',
type: 'text',
},
{
field: 'host.name.keyword',
type: 'keyword',
},
{
field: 'some.field',
type: 'text',
},
{
field: 'some.field.keyword',
type: 'keyword',
},
{
field: 'source.ip',
type: 'text',
},
{
field: 'source.ip.keyword',
type: 'keyword',
},
{
field: 'source.port',
type: 'long',
},
];
test('it flattens the field names and types in the mapping properties', () => {
expect(getFieldTypes(mockMappingsProperties)).toEqual(expected);
});
test('it throws a type error when mappingsProperties is not flatten-able', () => {
// @ts-expect-error
const invalidType: Record<string, unknown> = []; // <-- this is an array, NOT a valid Record<string, unknown>
expect(() => getFieldTypes(invalidType)).toThrowError('Root value is not flatten-able');
});
});
describe('isMappingCompatible', () => {
test('it returns true for an exact match', () => {
expect(isMappingCompatible({ ecsExpectedType: 'keyword', type: 'keyword' })).toBe(true);
});
test("it returns false when both types don't exactly match", () => {
expect(isMappingCompatible({ ecsExpectedType: 'wildcard', type: 'keyword' })).toBe(false);
});
});
describe('getEnrichedFieldMetadata', () => {
/**
* The ECS schema
* https://raw.githubusercontent.com/elastic/ecs/main/generated/ecs/ecs_flat.yml
* defines a handful of fields that have `allowed_values`. For these
* fields, the documents in an index should only have specific values.
*
* This instance of the type `Record<string, UnallowedValueCount[]>`
* represents an index that doesn't have any unallowed values, for the
* specified keys in the map, i.e. `event.category`, `event.kind`, etc.
*
* This will be used to test the happy path. Variants of this
* value will be used to test unhappy paths.
*/
const noUnallowedValues: Record<string, UnallowedValueCount[]> = {
'event.category': [],
'event.kind': [],
'event.outcome': [],
'event.type': [],
};
/**
* Represents an index that has unallowed values, for the
* `event.category` field. The other fields in the collection,
* i.e. `event.kind`, don't have any unallowed values.
*
* This instance will be used to test paths where a field is
* NOT ECS complaint, because the index has unallowed values.
*/
const unallowedValues: Record<string, UnallowedValueCount[]> = {
'event.category': [
{
count: 2,
fieldName: 'an_invalid_category',
},
{
count: 1,
fieldName: 'theory',
},
],
'event.kind': [],
'event.outcome': [],
'event.type': [],
};
/**
* This instance of a `FieldType` has the correct mapping for the
* `event.category` field.
*
* This instance will be used to test paths where the index has
* a valid mapping for the `event.category` field.
*/
const fieldMetadataCorrectMappingType: FieldType = {
field: 'event.category',
type: 'keyword', // <-- this index has the correct mapping type
};
/**
* This `EnrichedFieldMetadata` for the `event.category` field,
* represents a happy path result, where the index being checked:
*
* 1) The `type` of the field in the index, `keyword`, matches the expected
* `type` of the `event.category` field, as defined by the `EcsMetadata`
* 2) The index doesn't have any unallowed values for the `event.category` field
*
* Since this is a happy path result, it has the following values:
* `indexInvalidValues` is an empty array, because the index does not contain any invalid values
* `isEcsCompliant` is true, because the index has the expected mapping type, and no unallowed values
*/
const happyPathResultSample: EnrichedFieldMetadata = {
dashed_name: 'event-category',
description:
'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
example: 'authentication',
flat_name: 'event.category',
ignore_above: 1024,
level: 'core',
name: 'category',
normalize: ['array'],
short: 'Event category. The second categorization field in the hierarchy.',
type: 'keyword',
indexFieldName: 'event.category',
indexFieldType: 'keyword', // a valid mapping, because the `type` property from the `ecsMetadata` is also `keyword`
indexInvalidValues: [], // empty array, because the index does not contain any invalid values
hasEcsMetadata: true,
isEcsCompliant: true, // because the index has the expected mapping type, and no unallowed values
isInSameFamily: false,
};
/**
* Creates expected result matcher based on the happy path result sample. Please, add similar `expect` based assertions to it if anything breaks
* with an ECS upgrade, instead of hardcoding the values.
*/
const expectedResult = (extraFields: Record<string, unknown> = {}) =>
expect.objectContaining({
...happyPathResultSample,
...extraFields,
allowed_values: expect.arrayContaining([
expect.objectContaining({
description: expect.any(String),
name: expect.any(String),
expected_event_types: expect.any(Array),
}),
]),
});
test('it returns the happy path result when the index has no mapping conflicts, and no unallowed values', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index
unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index
})
).toEqual(expectedResult());
});
test('it returns the happy path result when the index has no mapping conflicts, and the unallowedValues map does not contain an entry for the field', () => {
// create an `unallowedValues` that doesn't have an entry for `event.category`:
const noEntryForEventCategory: Record<string, UnallowedValueCount[]> = omit(
'event.category',
unallowedValues
);
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index
unallowedValues: noEntryForEventCategory, // a lookup in this map for the `event.category` field will return undefined
})
).toEqual(expectedResult());
});
test('it returns a result with the expected `indexInvalidValues` and `isEcsCompliant` when the index has no mapping conflict, but it has unallowed values', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index
unallowedValues, // this index has unallowed values for the event.category field
})
).toEqual(
expectedResult({
indexInvalidValues: [
{
count: 2,
fieldName: 'an_invalid_category',
},
{
count: 1,
fieldName: 'theory',
},
],
isEcsCompliant: false, // because there are unallowed values
})
);
});
test('it returns a result with the expected `isEcsCompliant` and `isInSameFamily` when the index type does not match ECS, but NO unallowed values', () => {
const indexFieldType = 'text';
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field: 'event.category', // `event.category` is a `keyword`, per the ECS spec
type: indexFieldType, // this index has a mapping of `text` instead
},
unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index
})
).toEqual(
expectedResult({
indexFieldType,
isEcsCompliant: false, // `keyword` !== `text`
isInSameFamily: false, // `keyword` and `text` are not in the same family
})
);
});
test('it returns a result with the expected `isEcsCompliant` and `isInSameFamily` when the mapping is is in the same family', () => {
const indexFieldType = 'wildcard';
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field: 'event.category', // `event.category` is a `keyword` per the ECS spec
type: indexFieldType, // this index has a mapping of `wildcard` instead
},
unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index
})
).toEqual(
expectedResult({
indexFieldType,
isEcsCompliant: false, // `wildcard` !== `keyword`
isInSameFamily: true, // `wildcard` and `keyword` are in the same family
})
);
});
test('it returns a result with the expected `indexInvalidValues`,`isEcsCompliant`, and `isInSameFamily` when the index has BOTH mapping conflicts, and unallowed values', () => {
const indexFieldType = 'text';
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field: 'event.category', // `event.category` is a `keyword` per the ECS spec
type: indexFieldType, // this index has a mapping of `text` instead
},
unallowedValues, // this index also has unallowed values for the event.category field
})
).toEqual(
expectedResult({
indexFieldType,
indexInvalidValues: [
{
count: 2,
fieldName: 'an_invalid_category',
},
{
count: 1,
fieldName: 'theory',
},
],
isEcsCompliant: false, // because there are BOTH mapping conflicts and unallowed values
isInSameFamily: false, // `text` and `keyword` are not in the same family
})
);
});
test('it returns the expected result for a custom field, i.e. a field that does NOT have an entry in `ecsMetadata`', () => {
const field = 'a_custom_field'; // not defined by ECS
const indexFieldType = 'keyword';
expect(
getEnrichedFieldMetadata({
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field,
type: indexFieldType, // no mapping conflict, because ECS doesn't define this field
},
unallowedValues: noUnallowedValues, // no unallowed values for `a_custom_field` in this index
})
).toEqual({
indexFieldName: field,
indexFieldType,
indexInvalidValues: [],
hasEcsMetadata: false,
isEcsCompliant: false,
isInSameFamily: false, // custom fields are never in the same family
});
});
});
describe('getMissingTimestampFieldMetadata', () => {
test('it returns the expected `EnrichedFieldMetadata`', () => {
expect(getMissingTimestampFieldMetadata()).toEqual({
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-', // the index did NOT define a mapping for @timestamp
indexInvalidValues: [],
isEcsCompliant: false, // an index must define the @timestamp mapping
isInSameFamily: false, // `date` is not a member of any families
});
});
});
describe('getPartitionedFieldMetadata', () => {
test('it places all the `EnrichedFieldMetadata` in the expected categories', () => {
const enrichedFieldMetadata: EnrichedFieldMetadata[] = [
timestamp,
eventCategoryWithUnallowedValues,
hostNameWithTextMapping,
hostNameKeyword,
someField,
someFieldKeyword,
sourceIpWithTextMapping,
sourceIpKeyword,
sourcePort,
];
const expected: PartitionedFieldMetadata = {
all: [
timestamp,
eventCategoryWithUnallowedValues,
hostNameWithTextMapping,
hostNameKeyword,
someField,
someFieldKeyword,
sourceIpWithTextMapping,
sourceIpKeyword,
sourcePort,
],
ecsCompliant: [timestamp, sourcePort],
custom: [hostNameKeyword, someField, someFieldKeyword, sourceIpKeyword],
incompatible: [
eventCategoryWithUnallowedValues,
hostNameWithTextMapping,
sourceIpWithTextMapping,
],
sameFamily: [],
};
expect(getPartitionedFieldMetadata(enrichedFieldMetadata)).toEqual(expected);
});
});

167
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/index_check_flyout/index_properties/utils/metadata.ts Normal file
View file

@ -0,0 +1,167 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { has } from 'lodash/fp';
import { EcsFlatTyped } from '../../../../../../constants';
import {
EcsBasedFieldMetadata,
EnrichedFieldMetadata,
PartitionedFieldMetadata,
UnallowedValueCount,
} from '../../../../../../types';
import { getIsInSameFamily } from './get_is_in_same_family';
export const getPartitionedFieldMetadata = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): PartitionedFieldMetadata =>
enrichedFieldMetadata.reduce<PartitionedFieldMetadata>(
(acc, x) => ({
all: [...acc.all, x],
ecsCompliant: x.isEcsCompliant ? [...acc.ecsCompliant, x] : acc.ecsCompliant,
custom: !x.hasEcsMetadata ? [...acc.custom, x] : acc.custom,
incompatible:
x.hasEcsMetadata && !x.isEcsCompliant && !x.isInSameFamily
? [...acc.incompatible, x]
: acc.incompatible,
sameFamily: x.isInSameFamily ? [...acc.sameFamily, x] : acc.sameFamily,
}),
{
all: [],
ecsCompliant: [],
custom: [],
incompatible: [],
sameFamily: [],
}
);
export interface FieldType {
field: string;
type: string;
}
function shouldReadKeys(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
const getNextPathWithoutProperties = ({
key,
pathWithoutProperties,
value,
}: {
key: string;
pathWithoutProperties: string;
value: unknown;
}): string => {
if (!pathWithoutProperties) {
return key;
}
if (shouldReadKeys(value) && (key === 'properties' || key === 'fields')) {
return `${pathWithoutProperties}`;
} else {
return `${pathWithoutProperties}.${key}`;
}
};
export function getFieldTypes(mappingsProperties: Record<string, unknown>): FieldType[] {
if (!shouldReadKeys(mappingsProperties)) {
throw new TypeError(`Root value is not flatten-able, received ${mappingsProperties}`);
}
const result: FieldType[] = [];
(function flatten(prefix, object, pathWithoutProperties) {
for (const [key, value] of Object.entries(object)) {
const path = prefix ? `${prefix}.${key}` : key;
const nextPathWithoutProperties = getNextPathWithoutProperties({
key,
pathWithoutProperties,
value,
});
if (shouldReadKeys(value)) {
flatten(path, value, nextPathWithoutProperties);
} else {
if (nextPathWithoutProperties.endsWith('.type')) {
const pathWithoutType = nextPathWithoutProperties.slice(
0,
nextPathWithoutProperties.lastIndexOf('.type')
);
result.push({
field: pathWithoutType,
type: `${value}`,
});
}
}
}
})('', mappingsProperties, '');
return result;
}
export const isMappingCompatible = ({
ecsExpectedType,
type,
}: {
ecsExpectedType: string | undefined;
type: string;
}): boolean => type === ecsExpectedType;
export const getEnrichedFieldMetadata = ({
ecsMetadata,
fieldMetadata,
unallowedValues,
}: {
ecsMetadata: EcsFlatTyped;
fieldMetadata: FieldType;
unallowedValues: Record<string, UnallowedValueCount[]>;
}): EnrichedFieldMetadata => {
const { field, type } = fieldMetadata;
const indexInvalidValues = unallowedValues[field] ?? [];
if (has(fieldMetadata.field, ecsMetadata)) {
const ecsExpectedType = ecsMetadata[field].type;
const isEcsCompliant =
isMappingCompatible({ ecsExpectedType, type }) && indexInvalidValues.length === 0;
const isInSameFamily =
!isMappingCompatible({ ecsExpectedType, type }) &&
indexInvalidValues.length === 0 &&
getIsInSameFamily({ ecsExpectedType, type });
return {
...ecsMetadata[field],
indexFieldName: field,
indexFieldType: type,
indexInvalidValues,
hasEcsMetadata: true,
isEcsCompliant,
isInSameFamily,
};
} else {
return {
indexFieldName: field,
indexFieldType: type,
indexInvalidValues: [],
hasEcsMetadata: false,
isEcsCompliant: false,
isInSameFamily: false, // custom fields are never in the same family
};
}
};
export const getMissingTimestampFieldMetadata = (): EcsBasedFieldMetadata => ({
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-',
indexInvalidValues: [],
isEcsCompliant: false,
isInSameFamily: false, // `date` is not a member of any families
});

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/index.tsx
View file

@ -9,8 +9,8 @@ import { EuiBadge, EuiFlexGroup, EuiFlexItem, EuiToolTip } from '@elastic/eui';
import React from 'react';
import styled from 'styled-components';
import { getPatternIlmPhaseDescription } from '../../../../../../helpers';
import type { IlmExplainPhaseCounts, IlmPhase } from '../../../../../../types';
import { getPatternIlmPhaseDescription } from './utils/get_pattern_ilm_phase_description';
const PhaseCountsFlexGroup = styled(EuiFlexGroup)`
display: inline-flex;

53
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/translations.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { i18n } from '@kbn/i18n';
export const UNMANAGED_PATTERN_TOOLTIP = ({
indices,
pattern,
}: {
indices: number;
pattern: string;
}) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.unmanagedPatternTooltip', {
values: { indices, pattern },
defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} unmanaged by Index Lifecycle Management (ILM)`,
});
export const WARM_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.warmPatternTooltip', {
values: { indices, pattern },
defaultMessage:
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} warm. Warm indices are no longer being updated but are still being queried.',
});
export const HOT_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.hotPatternTooltip', {
values: { indices, pattern },
defaultMessage:
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} hot. Hot indices are actively being updated and queried.',
});
export const FROZEN_PATTERN_TOOLTIP = ({
indices,
pattern,
}: {
indices: number;
pattern: string;
}) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.frozenPatternTooltip', {
values: { indices, pattern },
defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.`,
});
export const COLD_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.coldPatternTooltip', {
values: { indices, pattern },
defaultMessage:
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.',
});

106
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.test.ts Normal file
View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getPatternIlmPhaseDescription } from './get_pattern_ilm_phase_description';
describe('getPatternIlmPhaseDescription', () => {
const phases: Array<{
expected: string;
indices: number;
pattern: string;
phase: string;
}> = [
{
expected:
'1 index matching the .alerts-security.alerts-default pattern is hot. Hot indices are actively being updated and queried.',
indices: 1,
pattern: '.alerts-security.alerts-default',
phase: 'hot',
},
{
expected:
'2 indices matching the .alerts-security.alerts-default pattern are hot. Hot indices are actively being updated and queried.',
indices: 2,
pattern: '.alerts-security.alerts-default',
phase: 'hot',
},
{
expected:
'1 index matching the .alerts-security.alerts-default pattern is warm. Warm indices are no longer being updated but are still being queried.',
indices: 1,
pattern: '.alerts-security.alerts-default',
phase: 'warm',
},
{
expected:
'2 indices matching the .alerts-security.alerts-default pattern are warm. Warm indices are no longer being updated but are still being queried.',
indices: 2,
pattern: '.alerts-security.alerts-default',
phase: 'warm',
},
{
expected:
'1 index matching the .alerts-security.alerts-default pattern is cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.',
indices: 1,
pattern: '.alerts-security.alerts-default',
phase: 'cold',
},
{
expected:
'2 indices matching the .alerts-security.alerts-default pattern are cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.',
indices: 2,
pattern: '.alerts-security.alerts-default',
phase: 'cold',
},
{
expected:
"1 index matching the .alerts-security.alerts-default pattern is frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.",
indices: 1,
pattern: '.alerts-security.alerts-default',
phase: 'frozen',
},
{
expected:
"2 indices matching the .alerts-security.alerts-default pattern are frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.",
indices: 2,
pattern: '.alerts-security.alerts-default',
phase: 'frozen',
},
{
expected:
'1 index matching the .alerts-security.alerts-default pattern is unmanaged by Index Lifecycle Management (ILM)',
indices: 1,
pattern: '.alerts-security.alerts-default',
phase: 'unmanaged',
},
{
expected:
'2 indices matching the .alerts-security.alerts-default pattern are unmanaged by Index Lifecycle Management (ILM)',
indices: 2,
pattern: '.alerts-security.alerts-default',
phase: 'unmanaged',
},
{
expected: '',
indices: 1,
pattern: '.alerts-security.alerts-default',
phase: 'some-other-phase',
},
{
expected: '',
indices: 2,
pattern: '.alerts-security.alerts-default',
phase: 'some-other-phase',
},
];
phases.forEach(({ expected, indices, pattern, phase }) => {
test(`it returns the expected description when indices is ${indices}, pattern is ${pattern}, and phase is ${phase}`, () => {
expect(getPatternIlmPhaseDescription({ indices, pattern, phase })).toBe(expected);
});
});
});

33
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/pattern_summary/pattern_label/ilm_phase_counts/utils/get_pattern_ilm_phase_description.ts Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import * as i18n from '../translations';
export const getPatternIlmPhaseDescription = ({
indices,
pattern,
phase,
}: {
indices: number;
pattern: string;
phase: string;
}): string => {
switch (phase) {
case 'hot':
return i18n.HOT_PATTERN_TOOLTIP({ indices, pattern });
case 'warm':
return i18n.WARM_PATTERN_TOOLTIP({ indices, pattern });
case 'cold':
return i18n.COLD_PATTERN_TOOLTIP({ indices, pattern });
case 'frozen':
return i18n.FROZEN_PATTERN_TOOLTIP({ indices, pattern });
case 'unmanaged':
return i18n.UNMANAGED_PATTERN_TOOLTIP({ indices, pattern });
default:
return '';
}
};

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.test.tsx
View file

@ -17,7 +17,7 @@ import { omit } from 'lodash/fp';
import React from 'react';
import { TestExternalProviders } from '../../../../mock/test_providers/test_providers';
import { EMPTY_STAT } from '../../../../helpers';
import { EMPTY_STAT } from '../../../../constants';
import {
getDocsCountPercent,
getIncompatibleStatColor,

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/helpers.tsx
View file

@ -18,7 +18,8 @@ import moment from 'moment';
import styled from 'styled-components';
import { euiThemeVars } from '@kbn/ui-theme';
import { EMPTY_STAT, getIlmPhaseDescription } from '../../../../helpers';
import { EMPTY_STAT } from '../../../../constants';
import { getIlmPhaseDescription } from '../../../../utils/get_ilm_phase_description';
import { INCOMPATIBLE_INDEX_TOOL_TIP } from '../../../../stat_label/translations';
import { INDEX_SIZE_TOOLTIP } from '../../../../translations';
import * as i18n from './translations';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/summary_table/index.test.tsx
View file

@ -9,7 +9,7 @@ import numeral from '@elastic/numeral';
import { render, screen } from '@testing-library/react';
import React from 'react';
import { EMPTY_STAT } from '../../../../helpers';
import { EMPTY_STAT } from '../../../../constants';
import { getSummaryTableColumns } from './helpers';
import { mockIlmExplain } from '../../../../mock/ilm_explain/mock_ilm_explain';
import { auditbeatWithAllResults } from '../../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';

234
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.test.ts Normal file
View file

@ -0,0 +1,234 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { omit } from 'lodash/fp';
import { mockStatsAuditbeatIndex } from '../../../../mock/stats/mock_stats_packetbeat_index';
import { mockStatsPacketbeatIndex } from '../../../../mock/stats/mock_stats_auditbeat_index';
import { mockIlmExplain } from '../../../../mock/ilm_explain/mock_ilm_explain';
import { mockStats } from '../../../../mock/stats/mock_stats';
import { getIndexNames, getPatternDocsCount, getPatternSizeInBytes } from './stats';
import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types';
describe('getIndexNames', () => {
const isILMAvailable = true;
const ilmPhases = ['hot', 'warm', 'unmanaged'];
test('returns the expected index names when they have an ILM phase included in the ilmPhases list', () => {
expect(
getIndexNames({
ilmExplain: mockIlmExplain, // <-- the mock indexes have 'hot' ILM phases
ilmPhases,
isILMAvailable,
stats: mockStats,
})
).toEqual([
'.ds-packetbeat-8.6.1-2023.02.04-000001',
'.ds-packetbeat-8.5.3-2023.02.04-000001',
'auditbeat-custom-index-1',
]);
});
test('returns the expected filtered index names when they do NOT have an ILM phase included in the ilmPhases list', () => {
expect(
getIndexNames({
ilmExplain: mockIlmExplain, // <-- the mock indexes have 'hot' and 'unmanaged' ILM phases...
ilmPhases: ['warm', 'unmanaged'], // <-- ...but we don't ask for 'hot'
isILMAvailable,
stats: mockStats,
})
).toEqual(['auditbeat-custom-index-1']); // <-- the 'unmanaged' index
});
test('returns the expected index names when the `ilmExplain` is missing a record for an index', () => {
// the following `ilmExplain` is missing a record for one of the two packetbeat indexes:
const ilmExplainWithMissingIndex: Record<string, IlmExplainLifecycleLifecycleExplain> = omit(
'.ds-packetbeat-8.6.1-2023.02.04-000001',
mockIlmExplain
);
expect(
getIndexNames({
ilmExplain: ilmExplainWithMissingIndex, // <-- the mock indexes have 'hot' ILM phases...
ilmPhases: ['hot', 'warm', 'unmanaged'],
isILMAvailable,
stats: mockStats,
})
).toEqual(['.ds-packetbeat-8.5.3-2023.02.04-000001', 'auditbeat-custom-index-1']); // <-- only includes two of the three indices, because the other one is missing an ILM explain record
});
test('returns empty index names when `ilmPhases` is empty', () => {
expect(
getIndexNames({
ilmExplain: mockIlmExplain,
ilmPhases: [],
isILMAvailable,
stats: mockStats,
})
).toEqual([]);
});
test('returns empty index names when they have an ILM phase that matches', () => {
expect(
getIndexNames({
ilmExplain: null,
ilmPhases,
isILMAvailable,
stats: mockStats,
})
).toEqual([]);
});
test('returns empty index names when just `stats` is null', () => {
expect(
getIndexNames({
ilmExplain: mockIlmExplain,
ilmPhases,
isILMAvailable,
stats: null,
})
).toEqual([]);
});
test('returns empty index names when both `ilmExplain` and `stats` are null', () => {
expect(
getIndexNames({
ilmExplain: null,
ilmPhases,
isILMAvailable,
stats: null,
})
).toEqual([]);
});
});
describe('getPatternDocsCount', () => {
test('it returns the expected total given a subset of index names in the stats', () => {
const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001';
const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs;
expect(
getPatternDocsCount({
indexNames: [indexName],
stats: mockStatsPacketbeatIndex,
})
).toEqual(expectedCount);
});
test('it returns the expected total given all index names in the stats', () => {
const allIndexNamesInStats = [
'.ds-packetbeat-8.6.1-2023.02.04-000001',
'.ds-packetbeat-8.5.3-2023.02.04-000001',
];
expect(
getPatternDocsCount({
indexNames: allIndexNamesInStats,
stats: mockStatsPacketbeatIndex,
})
).toEqual(3258632);
});
test('it returns zero given an empty collection of index names', () => {
expect(
getPatternDocsCount({
indexNames: [], // <-- empty
stats: mockStatsPacketbeatIndex,
})
).toEqual(0);
});
test('it returns the expected total for a green index', () => {
const indexName = 'auditbeat-custom-index-1';
const expectedCount = mockStatsAuditbeatIndex[indexName].num_docs;
expect(
getPatternDocsCount({
indexNames: [indexName],
stats: mockStatsAuditbeatIndex,
})
).toEqual(expectedCount);
});
});
describe('getPatternSizeInBytes', () => {
test('it returns the expected total given a subset of index names in the stats', () => {
const indexName = '.ds-packetbeat-8.5.3-2023.02.04-000001';
const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes;
expect(
getPatternSizeInBytes({
indexNames: [indexName],
stats: mockStatsPacketbeatIndex,
})
).toEqual(expectedCount);
});
test('it returns the expected total given all index names in the stats', () => {
const allIndexNamesInStats = [
'.ds-packetbeat-8.6.1-2023.02.04-000001',
'.ds-packetbeat-8.5.3-2023.02.04-000001',
];
expect(
getPatternSizeInBytes({
indexNames: allIndexNamesInStats,
stats: mockStatsPacketbeatIndex,
})
).toEqual(1464758182);
});
test('it returns undefined given an empty collection of index names', () => {
expect(
getPatternSizeInBytes({
indexNames: [], // <-- empty
stats: mockStatsPacketbeatIndex,
})
).toBeUndefined();
});
test('it returns undefined if sizeInByte in not an integer', () => {
const indexName = 'auditbeat-custom-index-1';
expect(
getPatternSizeInBytes({
indexNames: [indexName],
stats: { [indexName]: { ...mockStatsAuditbeatIndex[indexName], size_in_bytes: null } },
})
).toBeUndefined();
});
test('it returns the expected total for an index', () => {
const indexName = 'auditbeat-custom-index-1';
const expectedCount = mockStatsAuditbeatIndex[indexName].size_in_bytes;
expect(
getPatternSizeInBytes({
indexNames: [indexName],
stats: mockStatsAuditbeatIndex,
})
).toEqual(expectedCount);
});
test('it returns the expected total for indices', () => {
const expectedCount = Object.values(mockStatsPacketbeatIndex).reduce(
(acc, { size_in_bytes: sizeInBytes }) => {
return acc + (sizeInBytes ?? 0);
},
0
);
expect(
getPatternSizeInBytes({
indexNames: [
'.ds-packetbeat-8.6.1-2023.02.04-000001',
'.ds-packetbeat-8.5.3-2023.02.04-000001',
],
stats: mockStatsPacketbeatIndex,
})
).toEqual(expectedCount);
});
});

72
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/indices_details/pattern/utils/stats.ts Normal file
View file

@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types';
import { getDocsCount, getSizeInBytes } from '../../../../utils/stats';
import { MeteringStatsIndex } from '../../../../types';
import { getIlmPhase } from '../helpers';
export const getPatternDocsCount = ({
indexNames,
stats,
}: {
indexNames: string[];
stats: Record<string, MeteringStatsIndex> | null;
}): number =>
indexNames.reduce(
(acc: number, indexName: string) => acc + getDocsCount({ stats, indexName }),
0
);
export const getPatternSizeInBytes = ({
indexNames,
stats,
}: {
indexNames: string[];
stats: Record<string, MeteringStatsIndex> | null;
}): number | undefined => {
let sum;
for (let i = 0; i < indexNames.length; i++) {
const currentSizeInBytes = getSizeInBytes({ stats, indexName: indexNames[i] });
if (currentSizeInBytes != null) {
if (sum == null) {
sum = 0;
}
sum += currentSizeInBytes;
} else {
return undefined;
}
}
return sum;
};
const EMPTY_INDEX_NAMES: string[] = [];
export const getIndexNames = ({
ilmExplain,
ilmPhases,
isILMAvailable,
stats,
}: {
ilmExplain: Record<string, IlmExplainLifecycleLifecycleExplain> | null;
ilmPhases: string[];
isILMAvailable: boolean;
stats: Record<string, MeteringStatsIndex> | null;
}): string[] => {
if (((isILMAvailable && ilmExplain != null) || !isILMAvailable) && stats != null) {
const allIndexNames = Object.keys(stats);
const filteredByIlmPhase = isILMAvailable
? allIndexNames.filter((indexName) =>
ilmPhases.includes(getIlmPhase(ilmExplain?.[indexName], isILMAvailable) ?? '')
)
: allIndexNames;
return filteredByIlmPhase;
} else {
return EMPTY_INDEX_NAMES;
}
};

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.test.ts
View file

@ -8,7 +8,7 @@
import numeral from '@elastic/numeral';
import { euiThemeVars } from '@kbn/ui-theme';
import { EMPTY_STAT } from '../../helpers';
import { EMPTY_STAT } from '../../constants';
import {
DEFAULT_INDEX_COLOR,
getFillColor,

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/helpers.ts
View file

@ -9,9 +9,9 @@ import type { Datum, Key, ArrayNode } from '@elastic/charts';
import { euiThemeVars } from '@kbn/ui-theme';
import { orderBy } from 'lodash/fp';
import { getDocsCount, getSizeInBytes } from '../../helpers';
import { getIlmPhase } from '../indices_details/pattern/helpers';
import { PatternRollup } from '../../types';
import { getDocsCount, getSizeInBytes } from '../../utils/stats';
export interface LegendItem {
color: string | null;

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/index.test.tsx
View file

@ -9,7 +9,7 @@ import numeral from '@elastic/numeral';
import { render, screen } from '@testing-library/react';
import React from 'react';
import { EMPTY_STAT } from '../../helpers';
import { EMPTY_STAT } from '../../constants';
import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { packetbeatNoResults } from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_details/storage_details/storage_treemap/index.test.tsx
View file

@ -12,7 +12,7 @@ import userEvent from '@testing-library/user-event';
import React from 'react';
import { FlattenedBucket, getFlattenedBuckets, getLegendItems } from '../helpers';
import { EMPTY_STAT } from '../../../helpers';
import { EMPTY_STAT } from '../../../constants';
import { alertIndexWithAllResults } from '../../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { auditbeatWithAllResults } from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { packetbeatNoResults } from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/ilm_phase_filter/index.tsx
View file

@ -15,7 +15,7 @@ import {
import React, { useCallback, useMemo } from 'react';
import { ilmPhaseOptionsStatic } from '../../constants';
import { getIlmPhaseDescription } from '../../helpers';
import { getIlmPhaseDescription } from '../../utils/get_ilm_phase_description';
import {
ILM_PHASE,
INDEX_LIFECYCLE_MANAGEMENT_PHASES,

4
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/index.test.tsx
View file

@ -9,7 +9,7 @@ import numeral from '@elastic/numeral';
import { render, screen } from '@testing-library/react';
import React from 'react';
import { EMPTY_STAT } from '../helpers';
import { EMPTY_STAT } from '../constants';
import { alertIndexWithAllResults } from '../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { auditbeatWithAllResults } from '../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { packetbeatNoResults } from '../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
@ -25,7 +25,7 @@ import {
getTotalIndices,
getTotalIndicesChecked,
getTotalSizeInBytes,
} from '../hooks/use_results_rollup/helpers';
} from '../hooks/use_results_rollup/utils/stats';
const defaultBytesFormat = '0,0.[0]b';
const formatBytes = (value: number | undefined) =>

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/helpers.ts
View file

@ -7,8 +7,8 @@
import { orderBy } from 'lodash/fp';
import { getDocsCount } from '../../../helpers';
import type { IndexToCheck, MeteringStatsIndex, PatternRollup } from '../../../types';
import { getDocsCount } from '../../../utils/stats';
export const getIndexToCheck = ({
indexName,

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/check_all/index.test.tsx
View file

@ -19,7 +19,7 @@ import { mockUnallowedValuesResponse } from '../../../mock/unallowed_values/mock
import { CANCEL, CHECK_ALL } from '../../../translations';
import { OnCheckCompleted, UnallowedValueRequestItem } from '../../../types';
import { CheckAll } from '.';
import { EMPTY_STAT } from '../../../helpers';
import { EMPTY_STAT } from '../../../constants';
const defaultBytesFormat = '0,0.[0]b';
const mockFormatBytes = (value: number | undefined) =>

4
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.test.tsx
View file

@ -10,7 +10,7 @@ import { render, screen } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import React from 'react';
import { EMPTY_STAT } from '../../helpers';
import { EMPTY_STAT } from '../../constants';
import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { packetbeatNoResults } from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
@ -26,7 +26,7 @@ import {
getTotalIndices,
getTotalIndicesChecked,
getTotalSizeInBytes,
} from '../../hooks/use_results_rollup/helpers';
} from '../../hooks/use_results_rollup/utils/stats';
const mockCopyToClipboard = jest.fn((value) => true);
jest.mock('@elastic/eui', () => {

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/index.tsx
View file

@ -27,10 +27,11 @@ import {
getSummaryTableItems,
} from '../../data_quality_details/indices_details/pattern/helpers';
import type { DataQualityCheckResult, IndexToCheck, PatternRollup } from '../../types';
import { getErrorSummaries, getSizeInBytes } from '../../helpers';
import { useDataQualityContext } from '../../data_quality_context';
import { useResultsRollupContext } from '../../contexts/results_rollup_context';
import { Actions } from '../../actions';
import { getErrorSummaries } from './utils/get_error_summaries';
import { getSizeInBytes } from '../../utils/stats';
const StyledActionsContainerFlexItem = styled(EuiFlexItem)`
margin-top: auto;

188
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.test.ts Normal file
View file

@ -0,0 +1,188 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { alertIndexNoResults } from '../../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import {
auditbeatNoResults,
auditbeatWithAllResults,
} from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import {
packetbeatNoResults,
packetbeatWithSomeErrors,
} from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
import { DataQualityCheckResult, PatternRollup } from '../../../types';
import {
getErrorSummaries,
getErrorSummariesForRollup,
getErrorSummary,
} from './get_error_summaries';
describe('getErrorSummary', () => {
test('it returns the expected error summary', () => {
const resultWithError: DataQualityCheckResult = {
docsCount: 1630289,
error:
'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001',
ilmPhase: 'hot',
incompatible: undefined,
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'packetbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
};
expect(getErrorSummary(resultWithError)).toEqual({
error:
'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001',
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
pattern: 'packetbeat-*',
});
});
});
describe('getErrorSummariesForRollup', () => {
test('it returns the expected array of `ErrorSummary` when the `PatternRollup` contains errors', () => {
expect(getErrorSummariesForRollup(packetbeatWithSomeErrors)).toEqual([
{
error:
'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001',
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
pattern: 'packetbeat-*',
},
]);
});
test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` contains all results, with NO errors', () => {
expect(getErrorSummariesForRollup(auditbeatWithAllResults)).toEqual([]);
});
test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` has NO results', () => {
expect(getErrorSummariesForRollup(auditbeatNoResults)).toEqual([]);
});
test('it returns the an empty array of `ErrorSummary` when the `PatternRollup` is undefined', () => {
expect(getErrorSummariesForRollup(undefined)).toEqual([]);
});
test('it returns BOTH the expected (root) pattern-level error, and an index-level error when `PatternRollup` has both', () => {
const withPatternLevelError: PatternRollup = {
...packetbeatWithSomeErrors,
error: 'This is a pattern-level error',
};
expect(getErrorSummariesForRollup(withPatternLevelError)).toEqual([
{
error: 'This is a pattern-level error',
indexName: null,
pattern: 'packetbeat-*',
},
{
error:
'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001',
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
pattern: 'packetbeat-*',
},
]);
});
test('it returns the expected (root) pattern-level error when there are no index-level results', () => {
const withPatternLevelError: PatternRollup = {
...auditbeatNoResults,
error: 'This is a pattern-level error',
};
expect(getErrorSummariesForRollup(withPatternLevelError)).toEqual([
{
error: 'This is a pattern-level error',
indexName: null,
pattern: 'auditbeat-*',
},
]);
});
});
describe('getErrorSummaries', () => {
test('it returns an empty array when patternRollups is empty', () => {
expect(getErrorSummaries({})).toEqual([]);
});
test('it returns an empty array when none of the patternRollups have errors', () => {
expect(
getErrorSummaries({
'.alerts-security.alerts-default': alertIndexNoResults,
'auditbeat-*': auditbeatWithAllResults,
'packetbeat-*': packetbeatNoResults,
})
).toEqual([]);
});
test('it returns the expected array of `ErrorSummary` when some of the `PatternRollup` contain errors', () => {
expect(
getErrorSummaries({
'.alerts-security.alerts-default': alertIndexNoResults,
'auditbeat-*': auditbeatWithAllResults,
'packetbeat-*': packetbeatWithSomeErrors, // <-- has errors
})
).toEqual([
{
error:
'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001',
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
pattern: 'packetbeat-*',
},
]);
});
test('it returns the expected array of `ErrorSummary` when there are both pattern-level and index-level errors', () => {
const withPatternLevelError: PatternRollup = {
...auditbeatNoResults,
error: 'This is a pattern-level error',
};
expect(
getErrorSummaries({
'.alerts-security.alerts-default': alertIndexNoResults,
'auditbeat-*': withPatternLevelError, // <-- has pattern-level errors
'packetbeat-*': packetbeatWithSomeErrors, // <-- has index-level errors
})
).toEqual([
{
error: 'This is a pattern-level error',
indexName: null,
pattern: 'auditbeat-*',
},
{
error:
'Error loading mappings for .ds-packetbeat-8.5.3-2023.02.04-000001: Error: simulated error fetching index .ds-packetbeat-8.5.3-2023.02.04-000001',
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
pattern: 'packetbeat-*',
},
]);
});
test('it returns the expected array of `ErrorSummary` when there are just pattern-level errors', () => {
const withPatternLevelError: PatternRollup = {
...auditbeatNoResults,
error: 'This is a pattern-level error',
};
expect(
getErrorSummaries({
'.alerts-security.alerts-default': alertIndexNoResults,
'auditbeat-*': withPatternLevelError, // <-- has pattern-level errors
'packetbeat-*': packetbeatNoResults,
})
).toEqual([
{
error: 'This is a pattern-level error',
indexName: null,
pattern: 'auditbeat-*',
},
]);
});
});

56
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/data_quality_summary/summary_actions/utils/get_error_summaries.ts Normal file
View file

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import sortBy from 'lodash/fp/sortBy';
import { DataQualityCheckResult, ErrorSummary, PatternRollup } from '../../../types';
export const getErrorSummary = ({
error,
indexName,
pattern,
}: DataQualityCheckResult): ErrorSummary => ({
error: String(error),
indexName,
pattern,
});
export const getErrorSummariesForRollup = (
patternRollup: PatternRollup | undefined
): ErrorSummary[] => {
const maybePatternErrorSummary: ErrorSummary[] =
patternRollup != null && patternRollup.error != null
? [{ pattern: patternRollup.pattern, indexName: null, error: patternRollup.error }]
: [];
if (patternRollup != null && patternRollup.results != null) {
const unsortedResults: DataQualityCheckResult[] = Object.values(patternRollup.results);
const sortedResults = sortBy('indexName', unsortedResults);
return sortedResults.reduce<ErrorSummary[]>(
(acc, result) => [...acc, ...(result.error != null ? [getErrorSummary(result)] : [])],
maybePatternErrorSummary
);
} else {
return maybePatternErrorSummary;
}
};
export const getErrorSummaries = (
patternRollups: Record<string, PatternRollup>
): ErrorSummary[] => {
const allPatterns: string[] = Object.keys(patternRollups);
// sort the patterns A-Z:
const sortedPatterns = [...allPatterns].sort((a, b) => {
return a.localeCompare(b);
});
return sortedPatterns.reduce<ErrorSummary[]>(
(acc, pattern) => [...acc, ...getErrorSummariesForRollup(patternRollups[pattern])],
[]
);
};

1537
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.test.ts
View file

File diff suppressed because it is too large Load diff

615
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/helpers.ts
View file

@ -1,615 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { HttpHandler } from '@kbn/core-http-browser';
import type { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types';
import { has, sortBy } from 'lodash/fp';
import { IToasts } from '@kbn/core-notifications-browser';
import { getIlmPhase } from './data_quality_details/indices_details/pattern/helpers';
import * as i18n from './translations';
import type {
DataQualityCheckResult,
DataQualityIndexCheckedParams,
EcsBasedFieldMetadata,
EnrichedFieldMetadata,
ErrorSummary,
IlmPhase,
IncompatibleFieldMappingItem,
IncompatibleFieldValueItem,
MeteringStatsIndex,
PartitionedFieldMetadata,
PartitionedFieldMetadataStats,
PatternRollup,
SameFamilyFieldItem,
UnallowedValueCount,
} from './types';
import { EcsFlatTyped } from './constants';
const EMPTY_INDEX_NAMES: string[] = [];
export const INTERNAL_API_VERSION = '1';
export const getIndexNames = ({
ilmExplain,
ilmPhases,
isILMAvailable,
stats,
}: {
ilmExplain: Record<string, IlmExplainLifecycleLifecycleExplain> | null;
ilmPhases: string[];
isILMAvailable: boolean;
stats: Record<string, MeteringStatsIndex> | null;
}): string[] => {
if (((isILMAvailable && ilmExplain != null) || !isILMAvailable) && stats != null) {
const allIndexNames = Object.keys(stats);
const filteredByIlmPhase = isILMAvailable
? allIndexNames.filter((indexName) =>
ilmPhases.includes(getIlmPhase(ilmExplain?.[indexName], isILMAvailable) ?? '')
)
: allIndexNames;
return filteredByIlmPhase;
} else {
return EMPTY_INDEX_NAMES;
}
};
export interface FieldType {
field: string;
type: string;
}
function shouldReadKeys(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
const getNextPathWithoutProperties = ({
key,
pathWithoutProperties,
value,
}: {
key: string;
pathWithoutProperties: string;
value: unknown;
}): string => {
if (!pathWithoutProperties) {
return key;
}
if (shouldReadKeys(value) && (key === 'properties' || key === 'fields')) {
return `${pathWithoutProperties}`;
} else {
return `${pathWithoutProperties}.${key}`;
}
};
export function getFieldTypes(mappingsProperties: Record<string, unknown>): FieldType[] {
if (!shouldReadKeys(mappingsProperties)) {
throw new TypeError(`Root value is not flatten-able, received ${mappingsProperties}`);
}
const result: FieldType[] = [];
(function flatten(prefix, object, pathWithoutProperties) {
for (const [key, value] of Object.entries(object)) {
const path = prefix ? `${prefix}.${key}` : key;
const nextPathWithoutProperties = getNextPathWithoutProperties({
key,
pathWithoutProperties,
value,
});
if (shouldReadKeys(value)) {
flatten(path, value, nextPathWithoutProperties);
} else {
if (nextPathWithoutProperties.endsWith('.type')) {
const pathWithoutType = nextPathWithoutProperties.slice(
0,
nextPathWithoutProperties.lastIndexOf('.type')
);
result.push({
field: pathWithoutType,
type: `${value}`,
});
}
}
}
})('', mappingsProperties, '');
return result;
}
/**
* Per https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html#_core_datatypes
*
* ```
* Field types are grouped by _family_. Types in the same family have exactly
* the same search behavior but may have different space usage or
* performance characteristics.
*
* Currently, there are two type families, `keyword` and `text`. Other type
* families have only a single field type. For example, the `boolean` type
* family consists of one field type: `boolean`.
* ```
*/
export const fieldTypeFamilies: Record<string, Set<string>> = {
keyword: new Set(['keyword', 'constant_keyword', 'wildcard']),
text: new Set(['text', 'match_only_text']),
};
export const getIsInSameFamily = ({
ecsExpectedType,
type,
}: {
ecsExpectedType: string | undefined;
type: string;
}): boolean => {
if (ecsExpectedType != null) {
const allFamilies = Object.values(fieldTypeFamilies);
return allFamilies.reduce<boolean>(
(acc, family) => (acc !== true ? family.has(ecsExpectedType) && family.has(type) : acc),
false
);
} else {
return false;
}
};
export const isMappingCompatible = ({
ecsExpectedType,
type,
}: {
ecsExpectedType: string | undefined;
type: string;
}): boolean => type === ecsExpectedType;
export const getEnrichedFieldMetadata = ({
ecsMetadata,
fieldMetadata,
unallowedValues,
}: {
ecsMetadata: EcsFlatTyped;
fieldMetadata: FieldType;
unallowedValues: Record<string, UnallowedValueCount[]>;
}): EnrichedFieldMetadata => {
const { field, type } = fieldMetadata;
const indexInvalidValues = unallowedValues[field] ?? [];
if (has(fieldMetadata.field, ecsMetadata)) {
const ecsExpectedType = ecsMetadata[field].type;
const isEcsCompliant =
isMappingCompatible({ ecsExpectedType, type }) && indexInvalidValues.length === 0;
const isInSameFamily =
!isMappingCompatible({ ecsExpectedType, type }) &&
indexInvalidValues.length === 0 &&
getIsInSameFamily({ ecsExpectedType, type });
return {
...ecsMetadata[field],
indexFieldName: field,
indexFieldType: type,
indexInvalidValues,
hasEcsMetadata: true,
isEcsCompliant,
isInSameFamily,
};
} else {
return {
indexFieldName: field,
indexFieldType: type,
indexInvalidValues: [],
hasEcsMetadata: false,
isEcsCompliant: false,
isInSameFamily: false, // custom fields are never in the same family
};
}
};
export const getMissingTimestampFieldMetadata = (): EcsBasedFieldMetadata => ({
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-',
indexInvalidValues: [],
isEcsCompliant: false,
isInSameFamily: false, // `date` is not a member of any families
});
export const getPartitionedFieldMetadata = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): PartitionedFieldMetadata =>
enrichedFieldMetadata.reduce<PartitionedFieldMetadata>(
(acc, x) => ({
all: [...acc.all, x],
ecsCompliant: x.isEcsCompliant ? [...acc.ecsCompliant, x] : acc.ecsCompliant,
custom: !x.hasEcsMetadata ? [...acc.custom, x] : acc.custom,
incompatible:
x.hasEcsMetadata && !x.isEcsCompliant && !x.isInSameFamily
? [...acc.incompatible, x]
: acc.incompatible,
sameFamily: x.isInSameFamily ? [...acc.sameFamily, x] : acc.sameFamily,
}),
{
all: [],
ecsCompliant: [],
custom: [],
incompatible: [],
sameFamily: [],
}
);
export const getPartitionedFieldMetadataStats = (
partitionedFieldMetadata: PartitionedFieldMetadata
): PartitionedFieldMetadataStats => {
const { all, ecsCompliant, custom, incompatible, sameFamily } = partitionedFieldMetadata;
return {
all: all.length,
ecsCompliant: ecsCompliant.length,
custom: custom.length,
incompatible: incompatible.length,
sameFamily: sameFamily.length,
};
};
export const getDocsCount = ({
indexName,
stats,
}: {
indexName: string;
stats: Record<string, MeteringStatsIndex> | null;
}): number => (stats && stats[indexName]?.num_docs) ?? 0;
export const getIndexId = ({
indexName,
stats,
}: {
indexName: string;
stats: Record<string, MeteringStatsIndex> | null;
}): string | null | undefined => stats && stats[indexName]?.uuid;
export const getSizeInBytes = ({
indexName,
stats,
}: {
indexName: string;
stats: Record<string, MeteringStatsIndex> | null;
}): number | undefined => (stats && stats[indexName]?.size_in_bytes) ?? undefined;
export const getTotalDocsCount = ({
indexNames,
stats,
}: {
indexNames: string[];
stats: Record<string, MeteringStatsIndex> | null;
}): number =>
indexNames.reduce(
(acc: number, indexName: string) => acc + getDocsCount({ stats, indexName }),
0
);
export const getTotalSizeInBytes = ({
indexNames,
stats,
}: {
indexNames: string[];
stats: Record<string, MeteringStatsIndex> | null;
}): number | undefined => {
let sum;
for (let i = 0; i < indexNames.length; i++) {
const currentSizeInBytes = getSizeInBytes({ stats, indexName: indexNames[i] });
if (currentSizeInBytes != null) {
if (sum == null) {
sum = 0;
}
sum += currentSizeInBytes;
} else {
return undefined;
}
}
return sum;
};
export const EMPTY_STAT = '--';
/**
* Returns an i18n description of an an ILM phase
*/
export const getIlmPhaseDescription = (phase: string): string => {
switch (phase) {
case 'hot':
return i18n.HOT_DESCRIPTION;
case 'warm':
return i18n.WARM_DESCRIPTION;
case 'cold':
return i18n.COLD_DESCRIPTION;
case 'frozen':
return i18n.FROZEN_DESCRIPTION;
case 'unmanaged':
return i18n.UNMANAGED_DESCRIPTION;
default:
return ' ';
}
};
export const getPatternIlmPhaseDescription = ({
indices,
pattern,
phase,
}: {
indices: number;
pattern: string;
phase: string;
}): string => {
switch (phase) {
case 'hot':
return i18n.HOT_PATTERN_TOOLTIP({ indices, pattern });
case 'warm':
return i18n.WARM_PATTERN_TOOLTIP({ indices, pattern });
case 'cold':
return i18n.COLD_PATTERN_TOOLTIP({ indices, pattern });
case 'frozen':
return i18n.FROZEN_PATTERN_TOOLTIP({ indices, pattern });
case 'unmanaged':
return i18n.UNMANAGED_PATTERN_TOOLTIP({ indices, pattern });
default:
return '';
}
};
export const getTotalPatternIncompatible = (
results: Record<string, DataQualityCheckResult> | undefined
): number | undefined => {
if (results == null) {
return undefined;
}
const allResults = Object.values(results);
return allResults.reduce<number>((acc, { incompatible }) => acc + (incompatible ?? 0), 0);
};
export const getTotalPatternIndicesChecked = (patternRollup: PatternRollup | undefined): number => {
if (patternRollup != null && patternRollup.results != null) {
const allResults = Object.values(patternRollup.results);
const nonErrorResults = allResults.filter(({ error }) => error == null);
return nonErrorResults.length;
} else {
return 0;
}
};
export const getTotalPatternSameFamily = (
results: Record<string, DataQualityCheckResult> | undefined
): number | undefined => {
if (results == null) {
return undefined;
}
const allResults = Object.values(results);
return allResults.reduce<number>((acc, { sameFamily }) => acc + (sameFamily ?? 0), 0);
};
export const getIncompatibleStatBadgeColor = (incompatible: number | undefined): string =>
incompatible != null && incompatible > 0 ? 'danger' : 'hollow';
export const getErrorSummary = ({
error,
indexName,
pattern,
}: DataQualityCheckResult): ErrorSummary => ({
error: String(error),
indexName,
pattern,
});
export const getErrorSummariesForRollup = (
patternRollup: PatternRollup | undefined
): ErrorSummary[] => {
const maybePatternErrorSummary: ErrorSummary[] =
patternRollup != null && patternRollup.error != null
? [{ pattern: patternRollup.pattern, indexName: null, error: patternRollup.error }]
: [];
if (patternRollup != null && patternRollup.results != null) {
const unsortedResults: DataQualityCheckResult[] = Object.values(patternRollup.results);
const sortedResults = sortBy('indexName', unsortedResults);
return sortedResults.reduce<ErrorSummary[]>(
(acc, result) => [...acc, ...(result.error != null ? [getErrorSummary(result)] : [])],
maybePatternErrorSummary
);
} else {
return maybePatternErrorSummary;
}
};
export const getErrorSummaries = (
patternRollups: Record<string, PatternRollup>
): ErrorSummary[] => {
const allPatterns: string[] = Object.keys(patternRollups);
// sort the patterns A-Z:
const sortedPatterns = [...allPatterns].sort((a, b) => {
return a.localeCompare(b);
});
return sortedPatterns.reduce<ErrorSummary[]>(
(acc, pattern) => [...acc, ...getErrorSummariesForRollup(patternRollups[pattern])],
[]
);
};
export const POST_INDEX_RESULTS = '/internal/ecs_data_quality_dashboard/results';
export const GET_INDEX_RESULTS_LATEST =
'/internal/ecs_data_quality_dashboard/results_latest/{pattern}';
export interface StorageResult {
batchId: string;
indexName: string;
indexPattern: string;
isCheckAll: boolean;
checkedAt: number;
docsCount: number;
totalFieldCount: number;
ecsFieldCount: number;
customFieldCount: number;
incompatibleFieldCount: number;
incompatibleFieldMappingItems: IncompatibleFieldMappingItem[];
incompatibleFieldValueItems: IncompatibleFieldValueItem[];
sameFamilyFieldCount: number;
sameFamilyFields: string[];
sameFamilyFieldItems: SameFamilyFieldItem[];
unallowedMappingFields: string[];
unallowedValueFields: string[];
sizeInBytes: number;
ilmPhase?: IlmPhase;
markdownComments: string[];
ecsVersion: string;
indexId: string;
error: string | null;
}
export const formatStorageResult = ({
result,
report,
partitionedFieldMetadata,
}: {
result: DataQualityCheckResult;
report: DataQualityIndexCheckedParams;
partitionedFieldMetadata: PartitionedFieldMetadata;
}): StorageResult => {
const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = [];
const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = [];
const sameFamilyFieldItems: SameFamilyFieldItem[] = [];
partitionedFieldMetadata.incompatible.forEach((field) => {
if (field.type !== field.indexFieldType) {
incompatibleFieldMappingItems.push({
fieldName: field.indexFieldName,
expectedValue: field.type,
actualValue: field.indexFieldType,
description: field.description,
});
}
if (field.indexInvalidValues.length > 0) {
incompatibleFieldValueItems.push({
fieldName: field.indexFieldName,
expectedValues: field.allowed_values?.map((x) => x.name) ?? [],
actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })),
description: field.description,
});
}
});
partitionedFieldMetadata.sameFamily.forEach((field) => {
sameFamilyFieldItems.push({
fieldName: field.indexFieldName,
expectedValue: field.type,
actualValue: field.indexFieldType,
description: field.description,
});
});
return {
batchId: report.batchId,
indexName: result.indexName,
indexPattern: result.pattern,
isCheckAll: report.isCheckAll,
checkedAt: result.checkedAt ?? Date.now(),
docsCount: result.docsCount ?? 0,
totalFieldCount: partitionedFieldMetadata.all.length,
ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
customFieldCount: partitionedFieldMetadata.custom.length,
incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
incompatibleFieldMappingItems,
incompatibleFieldValueItems,
sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
sameFamilyFields: report.sameFamilyFields ?? [],
sameFamilyFieldItems,
unallowedMappingFields: report.unallowedMappingFields ?? [],
unallowedValueFields: report.unallowedValueFields ?? [],
sizeInBytes: report.sizeInBytes ?? 0,
ilmPhase: result.ilmPhase,
markdownComments: result.markdownComments,
ecsVersion: report.ecsVersion,
indexId: report.indexId ?? '',
error: result.error,
};
};
export const formatResultFromStorage = ({
storageResult,
pattern,
}: {
storageResult: StorageResult;
pattern: string;
}): DataQualityCheckResult => ({
docsCount: storageResult.docsCount,
error: storageResult.error,
ilmPhase: storageResult.ilmPhase,
incompatible: storageResult.incompatibleFieldCount,
indexName: storageResult.indexName,
markdownComments: storageResult.markdownComments,
sameFamily: storageResult.sameFamilyFieldCount,
checkedAt: storageResult.checkedAt,
pattern,
});
export async function postStorageResult({
storageResult,
httpFetch,
toasts,
abortController = new AbortController(),
}: {
storageResult: StorageResult;
httpFetch: HttpHandler;
toasts: IToasts;
abortController?: AbortController;
}): Promise<void> {
try {
await httpFetch<void>(POST_INDEX_RESULTS, {
method: 'POST',
signal: abortController.signal,
version: INTERNAL_API_VERSION,
body: JSON.stringify(storageResult),
});
} catch (err) {
toasts.addError(err, { title: i18n.POST_RESULT_ERROR_TITLE });
}
}
export async function getStorageResults({
pattern,
httpFetch,
toasts,
abortController,
}: {
pattern: string;
httpFetch: HttpHandler;
toasts: IToasts;
abortController: AbortController;
}): Promise<StorageResult[]> {
try {
const route = GET_INDEX_RESULTS_LATEST.replace('{pattern}', pattern);
const results = await httpFetch<StorageResult[]>(route, {
method: 'GET',
signal: abortController.signal,
version: INTERNAL_API_VERSION,
});
return results;
} catch (err) {
toasts.addError(err, { title: i18n.GET_RESULTS_ERROR_TITLE });
return [];
}
}

10
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/constants.ts Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const POST_INDEX_RESULTS = '/internal/ecs_data_quality_dashboard/results';
export const GET_INDEX_RESULTS_LATEST =
'/internal/ecs_data_quality_dashboard/results_latest/{pattern}';

28
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/index.tsx
View file

@ -7,10 +7,10 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import { EcsVersion } from '@elastic/ecs';
import { isEmpty } from 'lodash/fp';
import { IToasts } from '@kbn/core-notifications-browser';
import { HttpHandler } from '@kbn/core-http-browser';
import {
getTotalDocsCount,
getTotalIncompatible,
@ -18,25 +18,22 @@ import {
getTotalIndicesChecked,
getTotalSameFamily,
getTotalSizeInBytes,
updateResultOnCheckCompleted,
} from './helpers';
getTotalPatternSameFamily,
getIndexId,
} from './utils/stats';
import {
getStorageResults,
postStorageResult,
formatStorageResult,
formatResultFromStorage,
} from './utils/storage';
import { getPatternRollupsWithLatestCheckResult } from './utils/get_pattern_rollups_with_latest_check_result';
import type {
DataQualityCheckResult,
OnCheckCompleted,
PatternRollup,
TelemetryEvents,
} from '../../types';
import {
getDocsCount,
getIndexId,
getStorageResults,
getSizeInBytes,
getTotalPatternSameFamily,
postStorageResult,
formatStorageResult,
formatResultFromStorage,
} from '../../helpers';
import {
getIlmPhase,
getIndexIncompatible,
@ -48,6 +45,7 @@ import {
} from '../../data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers';
import { UseResultsRollupReturnValue } from './types';
import { useIsMounted } from '../use_is_mounted';
import { getDocsCount, getSizeInBytes } from '../../utils/stats';
interface Props {
ilmPhases: string[];
@ -172,7 +170,7 @@ export const useResultsRollup = ({
isCheckAll,
}) => {
setPatternRollups((currentPatternRollups) => {
const updatedRollups = updateResultOnCheckCompleted({
const updatedRollups = getPatternRollupsWithLatestCheckResult({
error,
formatBytes,
formatNumber,

35
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/types.ts
View file

@ -5,7 +5,14 @@
* 2.0.
*/
import { OnCheckCompleted, PatternRollup } from '../../types';
import {
IlmPhase,
IncompatibleFieldMappingItem,
IncompatibleFieldValueItem,
OnCheckCompleted,
PatternRollup,
SameFamilyFieldItem,
} from '../../types';
export interface UseResultsRollupReturnValue {
onCheckCompleted: OnCheckCompleted;
@ -26,3 +33,29 @@ export interface UseResultsRollupReturnValue {
}) => void;
updatePatternRollup: (patternRollup: PatternRollup) => void;
}
export interface StorageResult {
batchId: string;
indexName: string;
indexPattern: string;
isCheckAll: boolean;
checkedAt: number;
docsCount: number;
totalFieldCount: number;
ecsFieldCount: number;
customFieldCount: number;
incompatibleFieldCount: number;
incompatibleFieldMappingItems: IncompatibleFieldMappingItem[];
incompatibleFieldValueItems: IncompatibleFieldValueItem[];
sameFamilyFieldCount: number;
sameFamilyFields: string[];
sameFamilyFieldItems: SameFamilyFieldItem[];
unallowedMappingFields: string[];
unallowedValueFields: string[];
sizeInBytes: number;
ilmPhase?: IlmPhase;
markdownComments: string[];
ecsVersion: string;
indexId: string;
error: string | null;
}

153
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.test.ts → x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.test.ts
View file

@ -7,24 +7,11 @@
import numeral from '@elastic/numeral';
import {
getTotalDocsCount,
getTotalIncompatible,
getTotalIndices,
getTotalIndicesChecked,
getTotalSameFamily,
updateResultOnCheckCompleted,
} from './helpers';
import { auditbeatWithAllResults } from '../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import {
mockPacketbeatPatternRollup,
packetbeatNoResults,
packetbeatWithSomeErrors,
} from '../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../../types';
import { EMPTY_STAT } from '../../helpers';
import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
import { alertIndexWithAllResults } from '../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { getPatternRollupsWithLatestCheckResult } from './get_pattern_rollups_with_latest_check_result';
import { mockPacketbeatPatternRollup } from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
import { MeteringStatsIndex, PatternRollup } from '../../../types';
import { EMPTY_STAT } from '../../../constants';
import { mockPartitionedFieldMetadata } from '../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
import { EcsVersion } from '@elastic/ecs';
const defaultBytesFormat = '0,0.[0]b';
@ -35,11 +22,6 @@ const defaultNumberFormat = '0,0.[000]';
const formatNumber = (value: number | undefined) =>
value != null ? numeral(value).format(defaultNumberFormat) : EMPTY_STAT;
const patternRollups: Record<string, PatternRollup> = {
'auditbeat-*': auditbeatWithAllResults, // indices: 3
'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2
};
describe('helpers', () => {
let originalFetch: (typeof global)['fetch'];
@ -51,121 +33,6 @@ describe('helpers', () => {
global.fetch = originalFetch;
});
describe('getTotalSameFamily', () => {
const defaultDataQualityCheckResult: DataQualityCheckResult = {
docsCount: 26093,
error: null,
ilmPhase: 'hot',
incompatible: 0,
indexName: '.internal.alerts-security.alerts-default-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: '.alerts-security.alerts-default',
sameFamily: 7,
checkedAt: 1706526408000,
};
const alertIndexWithSameFamily: PatternRollup = {
...alertIndexWithAllResults,
results: {
'.internal.alerts-security.alerts-default-000001': {
...defaultDataQualityCheckResult,
},
},
};
const withSameFamily: Record<string, PatternRollup> = {
'.internal.alerts-security.alerts-default-000001': alertIndexWithSameFamily,
};
test('it returns the expected count when patternRollups has sameFamily', () => {
expect(getTotalSameFamily(withSameFamily)).toEqual(7);
});
test('it returns undefined when patternRollups is empty', () => {
expect(getTotalSameFamily({})).toBeUndefined();
});
test('it returns zero when none of the rollups have same family', () => {
expect(getTotalSameFamily(patternRollups)).toEqual(0);
});
});
describe('getTotalIndices', () => {
test('it returns the expected total when ALL `PatternRollup`s have an `indices`', () => {
expect(getTotalIndices(patternRollups)).toEqual(5);
});
test('it returns undefined when only SOME of the `PatternRollup`s have an `indices`', () => {
const someIndicesAreUndefined: Record<string, PatternRollup> = {
'auditbeat-*': {
...auditbeatWithAllResults,
indices: undefined, // <--
},
'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2
};
expect(getTotalIndices(someIndicesAreUndefined)).toBeUndefined();
});
});
describe('getTotalDocsCount', () => {
test('it returns the expected total when ALL `PatternRollup`s have a `docsCount`', () => {
expect(getTotalDocsCount(patternRollups)).toEqual(
Number(auditbeatWithAllResults.docsCount) + Number(mockPacketbeatPatternRollup.docsCount)
);
});
test('it returns undefined when only SOME of the `PatternRollup`s have a `docsCount`', () => {
const someIndicesAreUndefined: Record<string, PatternRollup> = {
'auditbeat-*': {
...auditbeatWithAllResults,
docsCount: undefined, // <--
},
'packetbeat-*': mockPacketbeatPatternRollup,
};
expect(getTotalDocsCount(someIndicesAreUndefined)).toBeUndefined();
});
});
describe('getTotalIncompatible', () => {
test('it returns the expected total when ALL `PatternRollup`s have `results`', () => {
expect(getTotalIncompatible(patternRollups)).toEqual(4);
});
test('it returns the expected total when only SOME of the `PatternRollup`s have `results`', () => {
const someResultsAreUndefined: Record<string, PatternRollup> = {
'auditbeat-*': auditbeatWithAllResults,
'packetbeat-*': packetbeatNoResults, // <-- results is undefined
};
expect(getTotalIncompatible(someResultsAreUndefined)).toEqual(4);
});
test('it returns undefined when NONE of the `PatternRollup`s have `results`', () => {
const someResultsAreUndefined: Record<string, PatternRollup> = {
'packetbeat-*': packetbeatNoResults, // <-- results is undefined
};
expect(getTotalIncompatible(someResultsAreUndefined)).toBeUndefined();
});
});
describe('getTotalIndicesChecked', () => {
test('it returns the expected total', () => {
expect(getTotalIndicesChecked(patternRollups)).toEqual(3);
});
test('it returns the expected total when errors have occurred', () => {
const someErrors: Record<string, PatternRollup> = {
'auditbeat-*': auditbeatWithAllResults, // indices: 3
'packetbeat-*': packetbeatWithSomeErrors, // <-- indices: 2, but one has errors
};
expect(getTotalIndicesChecked(someErrors)).toEqual(4);
});
});
describe('updateResultOnCheckCompleted', () => {
const packetbeatStats861: MeteringStatsIndex =
mockPacketbeatPatternRollup.stats != null
@ -178,7 +45,7 @@ describe('helpers', () => {
test('it returns the updated rollups', () => {
expect(
updateResultOnCheckCompleted({
getPatternRollupsWithLatestCheckResult({
error: null,
formatBytes,
formatNumber,
@ -280,7 +147,7 @@ describe('helpers', () => {
};
expect(
updateResultOnCheckCompleted({
getPatternRollupsWithLatestCheckResult({
error: null,
formatBytes,
formatNumber,
@ -377,7 +244,7 @@ describe('helpers', () => {
test('it returns the expected results when `partitionedFieldMetadata` is null', () => {
expect(
updateResultOnCheckCompleted({
getPatternRollupsWithLatestCheckResult({
error: null,
formatBytes,
formatNumber,
@ -472,7 +339,7 @@ describe('helpers', () => {
};
expect(
updateResultOnCheckCompleted({
getPatternRollupsWithLatestCheckResult({
error: null,
formatBytes,
formatNumber,
@ -532,7 +399,7 @@ describe('helpers', () => {
};
expect(
updateResultOnCheckCompleted({
getPatternRollupsWithLatestCheckResult({
error: null,
formatBytes,
formatNumber,

92
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/get_pattern_rollups_with_latest_check_result.ts Normal file
View file

@ -0,0 +1,92 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getIndexDocsCountFromRollup } from '../../../data_quality_summary/summary_actions/check_all/helpers';
import { getIlmPhase } from '../../../data_quality_details/indices_details/pattern/helpers';
import { getAllIncompatibleMarkdownComments } from '../../../data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers';
import { getSizeInBytes } from '../../../utils/stats';
import type { IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../../types';
export const getPatternRollupsWithLatestCheckResult = ({
error,
formatBytes,
formatNumber,
indexName,
isILMAvailable,
partitionedFieldMetadata,
pattern,
patternRollups,
}: {
error: string | null;
formatBytes: (value: number | undefined) => string;
formatNumber: (value: number | undefined) => string;
indexName: string;
isILMAvailable: boolean;
partitionedFieldMetadata: PartitionedFieldMetadata | null;
pattern: string;
patternRollups: Record<string, PatternRollup>;
}): Record<string, PatternRollup> => {
const patternRollup: PatternRollup | undefined = patternRollups[pattern];
if (patternRollup != null) {
const ilmExplain = patternRollup.ilmExplain;
const ilmPhase: IlmPhase | undefined =
ilmExplain != null ? getIlmPhase(ilmExplain[indexName], isILMAvailable) : undefined;
const docsCount = getIndexDocsCountFromRollup({
indexName,
patternRollup,
});
const patternDocsCount = patternRollup.docsCount ?? 0;
const sizeInBytes = getSizeInBytes({ indexName, stats: patternRollup.stats });
const markdownComments =
partitionedFieldMetadata != null
? getAllIncompatibleMarkdownComments({
docsCount,
formatBytes,
formatNumber,
ilmPhase,
indexName,
isILMAvailable,
partitionedFieldMetadata,
patternDocsCount,
sizeInBytes,
})
: [];
const incompatible = partitionedFieldMetadata?.incompatible.length;
const sameFamily = partitionedFieldMetadata?.sameFamily.length;
const checkedAt = partitionedFieldMetadata ? Date.now() : undefined;
return {
...patternRollups,
[pattern]: {
...patternRollup,
results: {
...(patternRollup.results ?? {}),
[indexName]: {
docsCount,
error,
ilmPhase,
incompatible,
indexName,
markdownComments,
pattern,
sameFamily,
checkedAt,
},
},
},
};
} else {
return patternRollups;
}
};

231
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.test.ts Normal file
View file

@ -0,0 +1,231 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { alertIndexWithAllResults } from '../../../mock/pattern_rollup/mock_alerts_pattern_rollup';
import { auditbeatWithAllResults } from '../../../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import {
mockPacketbeatPatternRollup,
packetbeatNoResults,
packetbeatWithSomeErrors,
} from '../../../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
import { mockStats } from '../../../mock/stats/mock_stats';
import { DataQualityCheckResult, PatternRollup } from '../../../types';
import {
getIndexId,
getTotalDocsCount,
getTotalIncompatible,
getTotalIndices,
getTotalIndicesChecked,
getTotalPatternSameFamily,
getTotalSameFamily,
} from './stats';
const patternRollups: Record<string, PatternRollup> = {
'auditbeat-*': auditbeatWithAllResults, // indices: 3
'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2
};
describe('getTotalPatternSameFamily', () => {
const baseResult: DataQualityCheckResult = {
docsCount: 4,
error: null,
ilmPhase: 'unmanaged',
incompatible: 3,
indexName: 'auditbeat-custom-index-1',
markdownComments: [
'### auditbeat-custom-index-1\n',
'| Result | Index | Docs | Incompatible fields | ILM Phase |\n|--------|-------|------|---------------------|-----------|\n| ❌ | auditbeat-custom-index-1 | 4 (0.0%) | 3 | `unmanaged` |\n\n',
'### **Incompatible fields** `3` **Custom fields** `4` **ECS compliant fields** `2` **All fields** `9`\n',
"#### 3 incompatible fields, 0 fields with mappings in the same family\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\nIncompatible fields with mappings in the same family have exactly the same search behavior but may have different space usage or performance characteristics.\n\nWhen an incompatible field is not in the same family:\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n",
'\n#### Incompatible field mappings - auditbeat-custom-index-1\n\n\n| Field | ECS mapping type (expected) | Index mapping type (actual) | \n|-------|-----------------------------|-----------------------------|\n| host.name | `keyword` | `text` |\n| source.ip | `ip` | `text` |\n\n#### Incompatible field values - auditbeat-custom-index-1\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `an_invalid_category` (2),\n`theory` (1) |\n\n',
],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
};
it('returns undefined when results is undefined', () => {
expect(getTotalPatternSameFamily(undefined)).toBeUndefined();
});
it('returns 0 when results is an empty object', () => {
expect(getTotalPatternSameFamily({})).toBe(0);
});
it('should sum sameFamily values and return the total', () => {
const results: Record<string, DataQualityCheckResult> = {
a: {
...baseResult,
indexName: 'a',
markdownComments: [],
pattern: 'pattern',
sameFamily: 2,
},
b: {
...baseResult,
indexName: 'b',
markdownComments: [],
pattern: 'pattern',
sameFamily: 3,
},
c: { ...baseResult, indexName: 'c', markdownComments: [], pattern: 'pattern' },
};
expect(getTotalPatternSameFamily(results)).toBe(5);
});
it('handles a mix of defined and undefined sameFamily values', () => {
const results: Record<string, DataQualityCheckResult> = {
a: {
...baseResult,
indexName: 'a',
markdownComments: [],
pattern: 'pattern',
sameFamily: 1,
},
b: {
...baseResult,
indexName: 'b',
markdownComments: [],
pattern: 'pattern',
sameFamily: undefined,
},
c: {
...baseResult,
indexName: 'c',
markdownComments: [],
pattern: 'pattern',
sameFamily: 2,
},
};
expect(getTotalPatternSameFamily(results)).toBe(3);
});
});
describe('getTotalSameFamily', () => {
const defaultDataQualityCheckResult: DataQualityCheckResult = {
docsCount: 26093,
error: null,
ilmPhase: 'hot',
incompatible: 0,
indexName: '.internal.alerts-security.alerts-default-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: '.alerts-security.alerts-default',
sameFamily: 7,
checkedAt: 1706526408000,
};
const alertIndexWithSameFamily: PatternRollup = {
...alertIndexWithAllResults,
results: {
'.internal.alerts-security.alerts-default-000001': {
...defaultDataQualityCheckResult,
},
},
};
const withSameFamily: Record<string, PatternRollup> = {
'.internal.alerts-security.alerts-default-000001': alertIndexWithSameFamily,
};
test('it returns the expected count when patternRollups has sameFamily', () => {
expect(getTotalSameFamily(withSameFamily)).toEqual(7);
});
test('it returns undefined when patternRollups is empty', () => {
expect(getTotalSameFamily({})).toBeUndefined();
});
test('it returns zero when none of the rollups have same family', () => {
expect(getTotalSameFamily(patternRollups)).toEqual(0);
});
});
describe('getTotalIndices', () => {
test('it returns the expected total when ALL `PatternRollup`s have an `indices`', () => {
expect(getTotalIndices(patternRollups)).toEqual(5);
});
test('it returns undefined when only SOME of the `PatternRollup`s have an `indices`', () => {
const someIndicesAreUndefined: Record<string, PatternRollup> = {
'auditbeat-*': {
...auditbeatWithAllResults,
indices: undefined, // <--
},
'packetbeat-*': mockPacketbeatPatternRollup, // indices: 2
};
expect(getTotalIndices(someIndicesAreUndefined)).toBeUndefined();
});
});
describe('getTotalDocsCount', () => {
test('it returns the expected total when ALL `PatternRollup`s have a `docsCount`', () => {
expect(getTotalDocsCount(patternRollups)).toEqual(
Number(auditbeatWithAllResults.docsCount) + Number(mockPacketbeatPatternRollup.docsCount)
);
});
test('it returns undefined when only SOME of the `PatternRollup`s have a `docsCount`', () => {
const someIndicesAreUndefined: Record<string, PatternRollup> = {
'auditbeat-*': {
...auditbeatWithAllResults,
docsCount: undefined, // <--
},
'packetbeat-*': mockPacketbeatPatternRollup,
};
expect(getTotalDocsCount(someIndicesAreUndefined)).toBeUndefined();
});
});
describe('getTotalIncompatible', () => {
test('it returns the expected total when ALL `PatternRollup`s have `results`', () => {
expect(getTotalIncompatible(patternRollups)).toEqual(4);
});
test('it returns the expected total when only SOME of the `PatternRollup`s have `results`', () => {
const someResultsAreUndefined: Record<string, PatternRollup> = {
'auditbeat-*': auditbeatWithAllResults,
'packetbeat-*': packetbeatNoResults, // <-- results is undefined
};
expect(getTotalIncompatible(someResultsAreUndefined)).toEqual(4);
});
test('it returns undefined when NONE of the `PatternRollup`s have `results`', () => {
const someResultsAreUndefined: Record<string, PatternRollup> = {
'packetbeat-*': packetbeatNoResults, // <-- results is undefined
};
expect(getTotalIncompatible(someResultsAreUndefined)).toBeUndefined();
});
});
describe('getTotalIndicesChecked', () => {
test('it returns the expected total', () => {
expect(getTotalIndicesChecked(patternRollups)).toEqual(3);
});
test('it returns the expected total when errors have occurred', () => {
const someErrors: Record<string, PatternRollup> = {
'auditbeat-*': auditbeatWithAllResults, // indices: 3
'packetbeat-*': packetbeatWithSomeErrors, // <-- indices: 2, but one has errors
};
expect(getTotalIndicesChecked(someErrors)).toEqual(4);
});
});
describe('getIndexId', () => {
it('returns the expected index ID', () => {
expect(getIndexId({ indexName: 'auditbeat-custom-index-1', stats: mockStats })).toEqual(
'uyJDDqGrRQqdBTN0mCF-iw'
);
});
});

104
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/helpers.ts → x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/stats.ts
View file

@ -5,16 +5,20 @@
* 2.0.
*/
import { getIndexDocsCountFromRollup } from '../../data_quality_summary/summary_actions/check_all/helpers';
import { getIlmPhase } from '../../data_quality_details/indices_details/pattern/helpers';
import { getAllIncompatibleMarkdownComments } from '../../data_quality_details/indices_details/pattern/index_check_flyout/index_properties/index_check_fields/tabs/incompatible_tab/helpers';
import {
getSizeInBytes,
getTotalPatternIncompatible,
getTotalPatternIndicesChecked,
getTotalPatternSameFamily,
} from '../../helpers';
import type { IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../types';
import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../../../types';
import { getTotalPatternIncompatible, getTotalPatternIndicesChecked } from '../../../utils/stats';
export const getTotalPatternSameFamily = (
results: Record<string, DataQualityCheckResult> | undefined
): number | undefined => {
if (results == null) {
return undefined;
}
const allResults = Object.values(results);
return allResults.reduce<number>((acc, { sameFamily }) => acc + (sameFamily ?? 0), 0);
};
export const getTotalIndices = (
patternRollups: Record<string, PatternRollup>
@ -87,82 +91,10 @@ export const getTotalIndicesChecked = (patternRollups: Record<string, PatternRol
);
};
export const updateResultOnCheckCompleted = ({
error,
formatBytes,
formatNumber,
export const getIndexId = ({
indexName,
isILMAvailable,
partitionedFieldMetadata,
pattern,
patternRollups,
stats,
}: {
error: string | null;
formatBytes: (value: number | undefined) => string;
formatNumber: (value: number | undefined) => string;
indexName: string;
isILMAvailable: boolean;
partitionedFieldMetadata: PartitionedFieldMetadata | null;
pattern: string;
patternRollups: Record<string, PatternRollup>;
}): Record<string, PatternRollup> => {
const patternRollup: PatternRollup | undefined = patternRollups[pattern];
if (patternRollup != null) {
const ilmExplain = patternRollup.ilmExplain;
const ilmPhase: IlmPhase | undefined =
ilmExplain != null ? getIlmPhase(ilmExplain[indexName], isILMAvailable) : undefined;
const docsCount = getIndexDocsCountFromRollup({
indexName,
patternRollup,
});
const patternDocsCount = patternRollup.docsCount ?? 0;
const sizeInBytes = getSizeInBytes({ indexName, stats: patternRollup.stats });
const markdownComments =
partitionedFieldMetadata != null
? getAllIncompatibleMarkdownComments({
docsCount,
formatBytes,
formatNumber,
ilmPhase,
indexName,
isILMAvailable,
partitionedFieldMetadata,
patternDocsCount,
sizeInBytes,
})
: [];
const incompatible = partitionedFieldMetadata?.incompatible.length;
const sameFamily = partitionedFieldMetadata?.sameFamily.length;
const checkedAt = partitionedFieldMetadata ? Date.now() : undefined;
return {
...patternRollups,
[pattern]: {
...patternRollup,
results: {
...(patternRollup.results ?? {}),
[indexName]: {
docsCount,
error,
ilmPhase,
incompatible,
indexName,
markdownComments,
pattern,
sameFamily,
checkedAt,
},
},
},
};
} else {
return patternRollups;
}
};
stats: Record<string, MeteringStatsIndex> | null;
}): string | null | undefined => stats && stats[indexName]?.uuid;

202
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.test.ts Normal file
View file

@ -0,0 +1,202 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { httpServiceMock } from '@kbn/core-http-browser-mocks';
import { mockPartitionedFieldMetadataWithSameFamily } from '../../../mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family';
import { StorageResult } from '../types';
import { formatStorageResult, getStorageResults, postStorageResult } from './storage';
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
describe('formatStorageResult', () => {
it('should correctly format the input data into a StorageResult object', () => {
const inputData: Parameters<typeof formatStorageResult>[number] = {
result: {
indexName: 'testIndex',
pattern: 'testPattern',
checkedAt: 1627545600000,
docsCount: 100,
incompatible: 3,
sameFamily: 1,
ilmPhase: 'hot',
markdownComments: ['test comments'],
error: null,
},
report: {
batchId: 'testBatch',
isCheckAll: true,
sameFamilyFields: ['agent.type'],
unallowedMappingFields: ['event.category', 'host.name', 'source.ip'],
unallowedValueFields: ['event.category'],
sizeInBytes: 5000,
ecsVersion: '1.0.0',
indexName: 'testIndex',
indexId: 'testIndexId',
},
partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily,
};
const expectedResult: StorageResult = {
batchId: 'testBatch',
indexName: 'testIndex',
indexPattern: 'testPattern',
isCheckAll: true,
checkedAt: 1627545600000,
docsCount: 100,
totalFieldCount: 10,
ecsFieldCount: 2,
customFieldCount: 4,
incompatibleFieldCount: 3,
incompatibleFieldMappingItems: [
{
fieldName: 'event.category',
expectedValue: 'keyword',
actualValue: 'constant_keyword',
description:
'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
},
{
fieldName: 'host.name',
expectedValue: 'keyword',
actualValue: 'text',
description:
'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
},
{
fieldName: 'source.ip',
expectedValue: 'ip',
actualValue: 'text',
description: 'IP address of the source (IPv4 or IPv6).',
},
],
incompatibleFieldValueItems: [
{
fieldName: 'event.category',
expectedValues: [
'authentication',
'configuration',
'database',
'driver',
'email',
'file',
'host',
'iam',
'intrusion_detection',
'malware',
'network',
'package',
'process',
'registry',
'session',
'threat',
'vulnerability',
'web',
],
actualValues: [{ name: 'an_invalid_category', count: 2 }],
description:
'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
},
],
sameFamilyFieldCount: 1,
sameFamilyFields: ['agent.type'],
sameFamilyFieldItems: [
{
fieldName: 'agent.type',
expectedValue: 'keyword',
actualValue: 'constant_keyword',
description:
'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
},
],
unallowedMappingFields: ['event.category', 'host.name', 'source.ip'],
unallowedValueFields: ['event.category'],
sizeInBytes: 5000,
ilmPhase: 'hot',
markdownComments: ['test comments'],
ecsVersion: '1.0.0',
indexId: 'testIndexId',
error: null,
};
expect(formatStorageResult(inputData)).toEqual(expectedResult);
});
});
describe('postStorageResult', () => {
const { fetch } = httpServiceMock.createStartContract();
const { toasts } = notificationServiceMock.createStartContract();
beforeEach(() => {
fetch.mockClear();
});
test('it posts the result', async () => {
const storageResult = { indexName: 'test' } as unknown as StorageResult;
await postStorageResult({
storageResult,
httpFetch: fetch,
abortController: new AbortController(),
toasts,
});
expect(fetch).toHaveBeenCalledWith(
'/internal/ecs_data_quality_dashboard/results',
expect.objectContaining({
method: 'POST',
body: JSON.stringify(storageResult),
})
);
});
test('it throws error', async () => {
const storageResult = { indexName: 'test' } as unknown as StorageResult;
fetch.mockRejectedValueOnce('test-error');
await postStorageResult({
httpFetch: fetch,
storageResult,
abortController: new AbortController(),
toasts,
});
expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) });
});
});
describe('getStorageResults', () => {
const { fetch } = httpServiceMock.createStartContract();
const { toasts } = notificationServiceMock.createStartContract();
beforeEach(() => {
fetch.mockClear();
});
test('it gets the results', async () => {
await getStorageResults({
httpFetch: fetch,
abortController: new AbortController(),
pattern: 'auditbeat-*',
toasts,
});
expect(fetch).toHaveBeenCalledWith(
'/internal/ecs_data_quality_dashboard/results_latest/auditbeat-*',
expect.objectContaining({
method: 'GET',
})
);
});
it('should catch error', async () => {
fetch.mockRejectedValueOnce('test-error');
const results = await getStorageResults({
httpFetch: fetch,
abortController: new AbortController(),
pattern: 'auditbeat-*',
toasts,
});
expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) });
expect(results).toEqual([]);
});
});

157
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/hooks/use_results_rollup/utils/storage.ts Normal file
View file

@ -0,0 +1,157 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { HttpHandler } from '@kbn/core-http-browser';
import { IToasts } from '@kbn/core-notifications-browser';
import {
DataQualityCheckResult,
DataQualityIndexCheckedParams,
IncompatibleFieldMappingItem,
IncompatibleFieldValueItem,
PartitionedFieldMetadata,
SameFamilyFieldItem,
} from '../../../types';
import { StorageResult } from '../types';
import { GET_INDEX_RESULTS_LATEST, POST_INDEX_RESULTS } from '../constants';
import { INTERNAL_API_VERSION } from '../../../constants';
import { GET_RESULTS_ERROR_TITLE, POST_RESULT_ERROR_TITLE } from '../../../translations';
export const formatStorageResult = ({
result,
report,
partitionedFieldMetadata,
}: {
result: DataQualityCheckResult;
report: DataQualityIndexCheckedParams;
partitionedFieldMetadata: PartitionedFieldMetadata;
}): StorageResult => {
const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = [];
const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = [];
const sameFamilyFieldItems: SameFamilyFieldItem[] = [];
partitionedFieldMetadata.incompatible.forEach((field) => {
if (field.type !== field.indexFieldType) {
incompatibleFieldMappingItems.push({
fieldName: field.indexFieldName,
expectedValue: field.type,
actualValue: field.indexFieldType,
description: field.description,
});
}
if (field.indexInvalidValues.length > 0) {
incompatibleFieldValueItems.push({
fieldName: field.indexFieldName,
expectedValues: field.allowed_values?.map((x) => x.name) ?? [],
actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })),
description: field.description,
});
}
});
partitionedFieldMetadata.sameFamily.forEach((field) => {
sameFamilyFieldItems.push({
fieldName: field.indexFieldName,
expectedValue: field.type,
actualValue: field.indexFieldType,
description: field.description,
});
});
return {
batchId: report.batchId,
indexName: result.indexName,
indexPattern: result.pattern,
isCheckAll: report.isCheckAll,
checkedAt: result.checkedAt ?? Date.now(),
docsCount: result.docsCount ?? 0,
totalFieldCount: partitionedFieldMetadata.all.length,
ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
customFieldCount: partitionedFieldMetadata.custom.length,
incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
incompatibleFieldMappingItems,
incompatibleFieldValueItems,
sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
sameFamilyFields: report.sameFamilyFields ?? [],
sameFamilyFieldItems,
unallowedMappingFields: report.unallowedMappingFields ?? [],
unallowedValueFields: report.unallowedValueFields ?? [],
sizeInBytes: report.sizeInBytes ?? 0,
ilmPhase: result.ilmPhase,
markdownComments: result.markdownComments,
ecsVersion: report.ecsVersion,
indexId: report.indexId ?? '',
error: result.error,
};
};
export const formatResultFromStorage = ({
storageResult,
pattern,
}: {
storageResult: StorageResult;
pattern: string;
}): DataQualityCheckResult => ({
docsCount: storageResult.docsCount,
error: storageResult.error,
ilmPhase: storageResult.ilmPhase,
incompatible: storageResult.incompatibleFieldCount,
indexName: storageResult.indexName,
markdownComments: storageResult.markdownComments,
sameFamily: storageResult.sameFamilyFieldCount,
checkedAt: storageResult.checkedAt,
pattern,
});
export async function postStorageResult({
storageResult,
httpFetch,
toasts,
abortController = new AbortController(),
}: {
storageResult: StorageResult;
httpFetch: HttpHandler;
toasts: IToasts;
abortController?: AbortController;
}): Promise<void> {
try {
await httpFetch<void>(POST_INDEX_RESULTS, {
method: 'POST',
signal: abortController.signal,
version: INTERNAL_API_VERSION,
body: JSON.stringify(storageResult),
});
} catch (err) {
toasts.addError(err, { title: POST_RESULT_ERROR_TITLE });
}
}
export async function getStorageResults({
pattern,
httpFetch,
toasts,
abortController,
}: {
pattern: string;
httpFetch: HttpHandler;
toasts: IToasts;
abortController: AbortController;
}): Promise<StorageResult[]> {
try {
const route = GET_INDEX_RESULTS_LATEST.replace('{pattern}', pattern);
const results = await httpFetch<StorageResult[]>(route, {
method: 'GET',
signal: abortController.signal,
version: INTERNAL_API_VERSION,
});
return results;
} catch (err) {
toasts.addError(err, { title: GET_RESULTS_ERROR_TITLE });
return [];
}
}

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/index.tsx
View file

@ -13,13 +13,12 @@ import React, { useCallback, useMemo, useState } from 'react';
import type { IToasts } from '@kbn/core-notifications-browser';
import { EuiComboBoxOptionOption, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui';
import { DataQualityProvider } from './data_quality_context';
import { EMPTY_STAT } from './helpers';
import { ReportDataQualityCheckAllCompleted, ReportDataQualityIndexChecked } from './types';
import { ResultsRollupContext } from './contexts/results_rollup_context';
import { IndicesCheckContext } from './contexts/indices_check_context';
import { useIndicesCheck } from './hooks/use_indices_check';
import { useResultsRollup } from './hooks/use_results_rollup';
import { ilmPhaseOptionsStatic } from './constants';
import { ilmPhaseOptionsStatic, EMPTY_STAT } from './constants';
import { DataQualitySummary } from './data_quality_summary';
import { DataQualityDetails } from './data_quality_details';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/mock/test_providers/utils/get_merged_data_quality_context_props.ts
View file

@ -8,7 +8,7 @@
import numeral from '@elastic/numeral';
import { DataQualityProviderProps } from '../../../data_quality_context';
import { EMPTY_STAT } from '../../../helpers';
import { EMPTY_STAT } from '../../../constants';
export const getMergedDataQualityContextProps = (
dataQualityContextProps?: Partial<DataQualityProviderProps>

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/stats_rollup/index.tsx
View file

@ -9,10 +9,11 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import React from 'react';
import styled from 'styled-components';
import { EMPTY_STAT, getIncompatibleStatBadgeColor } from '../helpers';
import { EMPTY_STAT } from '../constants';
import { useDataQualityContext } from '../data_quality_context';
import * as i18n from '../stat_label/translations';
import { Stat } from '../stat';
import { getIncompatibleStatBadgeColor } from '../utils/get_incompatible_stat_badge_color';
const StyledStatWrapperFlexItem = styled(EuiFlexItem)`
padding: 0 ${({ theme }) => theme.eui.euiSize};

45
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/translations.ts
View file

@ -42,13 +42,6 @@ export const COLD_DESCRIPTION = i18n.translate(
}
);
export const COLD_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.coldPatternTooltip', {
values: { indices, pattern },
defaultMessage:
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} cold. Cold indices are no longer being updated and are queried infrequently. The information still needs to be searchable, but it’s okay if those queries are slower.',
});
export const COPIED_RESULTS_TOAST_TITLE = i18n.translate(
'securitySolutionPackages.ecsDataQualityDashboard.toasts.copiedResultsToastTitle',
{
@ -166,18 +159,6 @@ export const FROZEN_DESCRIPTION = i18n.translate(
}
);
export const FROZEN_PATTERN_TOOLTIP = ({
indices,
pattern,
}: {
indices: number;
pattern: string;
}) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.frozenPatternTooltip', {
values: { indices, pattern },
defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} frozen. Frozen indices are no longer being updated and are queried rarely. The information still needs to be searchable, but it's okay if those queries are extremely slow.`,
});
export const HOT_DESCRIPTION = i18n.translate(
'securitySolutionPackages.ecsDataQualityDashboard.hotDescription',
{
@ -185,13 +166,6 @@ export const HOT_DESCRIPTION = i18n.translate(
}
);
export const HOT_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.hotPatternTooltip', {
values: { indices, pattern },
defaultMessage:
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} hot. Hot indices are actively being updated and queried.',
});
/** The tooltip for the `ILM phase` combo box on the Data Quality Dashboard */
export const INDEX_LIFECYCLE_MANAGEMENT_PHASES: string = i18n.translate(
'securitySolutionPackages.ecsDataQualityDashboard.indexLifecycleManagementPhasesTooltip',
@ -267,18 +241,6 @@ export const UNMANAGED_DESCRIPTION = i18n.translate(
}
);
export const UNMANAGED_PATTERN_TOOLTIP = ({
indices,
pattern,
}: {
indices: number;
pattern: string;
}) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.unmanagedPatternTooltip', {
values: { indices, pattern },
defaultMessage: `{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} unmanaged by Index Lifecycle Management (ILM)`,
});
export const WARM_DESCRIPTION = i18n.translate(
'securitySolutionPackages.ecsDataQualityDashboard.warmDescription',
{
@ -286,13 +248,6 @@ export const WARM_DESCRIPTION = i18n.translate(
}
);
export const WARM_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pattern: string }) =>
i18n.translate('securitySolutionPackages.ecsDataQualityDashboard.warmPatternTooltip', {
values: { indices, pattern },
defaultMessage:
'{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} warm. Warm indices are no longer being updated but are still being queried.',
});
export const POST_RESULT_ERROR_TITLE = i18n.translate(
'securitySolutionPackages.ecsDataQualityDashboard.postResultErrorTitle',
{ defaultMessage: 'Error writing saved data quality check results' }

8
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/types.ts
View file

@ -85,14 +85,6 @@ export interface PartitionedFieldMetadata {
sameFamily: EcsBasedFieldMetadata[];
}
export interface PartitionedFieldMetadataStats {
all: number;
custom: number;
ecsCompliant: number;
incompatible: number;
sameFamily: number;
}
export interface UnallowedValueRequestItem {
allowedValues: string[];
indexFieldName: string;

3
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/check_index.test.ts
View file

@ -6,7 +6,6 @@
*/
import { checkIndex, EMPTY_PARTITIONED_FIELD_METADATA } from './check_index';
import { EMPTY_STAT } from '../helpers';
import { mockMappingsResponse } from '../mock/mappings_response/mock_mappings_response';
import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values';
import { UnallowedValueRequestItem, UnallowedValueSearchResult } from '../types';
@ -17,7 +16,7 @@ import {
import { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types';
import { getUnallowedValues } from './fetch_unallowed_values';
import { getUnallowedValueRequestItems } from './get_unallowed_value_request_items';
import { EcsFlatTyped } from '../constants';
import { EcsFlatTyped, EMPTY_STAT } from '../constants';
let mockFetchMappings = jest.fn(
(_: { abortController: AbortController; patternOrIndexName: string }) =>

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_mappings.ts
View file

@ -9,7 +9,7 @@ import type { HttpHandler } from '@kbn/core-http-browser';
import type { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types';
import * as i18n from '../translations';
import { INTERNAL_API_VERSION } from '../helpers';
import { INTERNAL_API_VERSION } from '../constants';
export const MAPPINGS_API_ROUTE = '/internal/ecs_data_quality_dashboard/mappings';

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.test.ts
View file

@ -15,7 +15,7 @@ import {
} from './fetch_unallowed_values';
import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values';
import { UnallowedValueRequestItem, UnallowedValueSearchResult } from '../types';
import { INTERNAL_API_VERSION } from '../helpers';
import { INTERNAL_API_VERSION } from '../constants';
describe('helpers', () => {
let originalFetch: (typeof global)['fetch'];

6
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/fetch_unallowed_values.ts
View file

@ -6,7 +6,8 @@
*/
import type { HttpHandler } from '@kbn/core-http-browser';
import { INTERNAL_API_VERSION } from '../helpers';
import { INTERNAL_API_VERSION } from '../constants';
import * as i18n from '../translations';
import type {
Bucket,
@ -15,8 +16,6 @@ import type {
UnallowedValueSearchResult,
} from '../types';
const UNALLOWED_VALUES_API_ROUTE = '/internal/ecs_data_quality_dashboard/unallowed_field_values';
export const isBucket = (maybeBucket: unknown): maybeBucket is Bucket =>
maybeBucket != null &&
typeof (maybeBucket as Bucket).key === 'string' &&
@ -65,6 +64,7 @@ export const getUnallowedValues = ({
}, {});
};
const UNALLOWED_VALUES_API_ROUTE = '/internal/ecs_data_quality_dashboard/unallowed_field_values';
export async function fetchUnallowedValues({
abortController,
httpFetch,

55
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.test.ts Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getIlmPhaseDescription } from './get_ilm_phase_description';
import {
COLD_DESCRIPTION,
FROZEN_DESCRIPTION,
HOT_DESCRIPTION,
UNMANAGED_DESCRIPTION,
WARM_DESCRIPTION,
} from '../translations';
describe('helpers', () => {
describe('getIlmPhaseDescription', () => {
const phases: Array<{
phase: string;
expected: string;
}> = [
{
phase: 'hot',
expected: HOT_DESCRIPTION,
},
{
phase: 'warm',
expected: WARM_DESCRIPTION,
},
{
phase: 'cold',
expected: COLD_DESCRIPTION,
},
{
phase: 'frozen',
expected: FROZEN_DESCRIPTION,
},
{
phase: 'unmanaged',
expected: UNMANAGED_DESCRIPTION,
},
{
phase: 'something-else',
expected: ' ',
},
];
phases.forEach(({ phase, expected }) => {
test(`it returns ${expected} when phase is ${phase}`, () => {
expect(getIlmPhaseDescription(phase)).toBe(expected);
});
});
});
});

28
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_ilm_phase_description.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import * as i18n from '../translations';
/**
* Returns an i18n description of an an ILM phase
*/
export const getIlmPhaseDescription = (phase: string): string => {
switch (phase) {
case 'hot':
return i18n.HOT_DESCRIPTION;
case 'warm':
return i18n.WARM_DESCRIPTION;
case 'cold':
return i18n.COLD_DESCRIPTION;
case 'frozen':
return i18n.FROZEN_DESCRIPTION;
case 'unmanaged':
return i18n.UNMANAGED_DESCRIPTION;
default:
return ' ';
}
};

28
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.test.ts Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getIncompatibleStatBadgeColor } from './get_incompatible_stat_badge_color';
describe('getIncompatibleStatBadgeColor', () => {
describe('when incompatible is greater than 0', () => {
it('returns danger', () => {
expect(getIncompatibleStatBadgeColor(1)).toBe('danger');
});
});
describe('when incompatible is 0', () => {
it('returns hollow', () => {
expect(getIncompatibleStatBadgeColor(0)).toBe('hollow');
});
});
describe('when incompatible is undefined', () => {
it('returns hollow', () => {
expect(getIncompatibleStatBadgeColor(undefined)).toBe('hollow');
});
});
});

9
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/get_incompatible_stat_badge_color.ts Normal file
View file

@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export const getIncompatibleStatBadgeColor = (incompatible: number | undefined): string =>
incompatible != null && incompatible > 0 ? 'danger' : 'hollow';

252
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.test.ts Normal file
View file

@ -0,0 +1,252 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
auditbeatNoResults,
auditbeatWithAllResults,
} from '../mock/pattern_rollup/mock_auditbeat_pattern_rollup';
import { packetbeatWithSomeErrors } from '../mock/pattern_rollup/mock_packetbeat_pattern_rollup';
import { mockStatsPacketbeatIndex } from '../mock/stats/mock_stats_auditbeat_index';
import { mockStatsAuditbeatIndex } from '../mock/stats/mock_stats_packetbeat_index';
import { DataQualityCheckResult } from '../types';
import {
getDocsCount,
getSizeInBytes,
getTotalPatternIncompatible,
getTotalPatternIndicesChecked,
} from './stats';
describe('getTotalPatternIndicesChecked', () => {
test('it returns zero when `patternRollup` is undefined', () => {
expect(getTotalPatternIndicesChecked(undefined)).toEqual(0);
});
test('it returns zero when `patternRollup` does NOT have any results', () => {
expect(getTotalPatternIndicesChecked(auditbeatNoResults)).toEqual(0);
});
test('it returns the expected total when all indices in `patternRollup` have results', () => {
expect(getTotalPatternIndicesChecked(auditbeatWithAllResults)).toEqual(3);
});
test('it returns the expected total when some indices in `patternRollup` have errors', () => {
expect(getTotalPatternIndicesChecked(packetbeatWithSomeErrors)).toEqual(1);
});
});
describe('getDocsCount', () => {
test('it returns the expected docs count when `stats` contains the `indexName`', () => {
const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
const expectedCount = mockStatsPacketbeatIndex[indexName].num_docs;
expect(
getDocsCount({
indexName,
stats: mockStatsPacketbeatIndex,
})
).toEqual(expectedCount);
});
test('it returns zero when `stats` does NOT contain the `indexName`', () => {
const indexName = 'not-gonna-find-it';
expect(
getDocsCount({
indexName,
stats: mockStatsPacketbeatIndex,
})
).toEqual(0);
});
test('it returns zero when `stats` is null', () => {
const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
expect(
getDocsCount({
indexName,
stats: null,
})
).toEqual(0);
});
test('it returns the expected total for a green index, where `primaries.docs.count` and `total.docs.count` have different values', () => {
const indexName = 'auditbeat-custom-index-1';
expect(
getDocsCount({
indexName,
stats: mockStatsAuditbeatIndex,
})
).toEqual(mockStatsAuditbeatIndex[indexName].num_docs);
});
});
describe('getSizeInBytes', () => {
test('it returns the expected size when `stats` contains the `indexName`', () => {
const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
const expectedCount = mockStatsPacketbeatIndex[indexName].size_in_bytes;
expect(
getSizeInBytes({
indexName,
stats: mockStatsPacketbeatIndex,
})
).toEqual(expectedCount);
});
test('it returns undefined when `stats` does NOT contain the `indexName`', () => {
const indexName = 'not-gonna-find-it';
expect(
getSizeInBytes({
indexName,
stats: mockStatsPacketbeatIndex,
})
).toBeUndefined();
});
test('it returns undefined when `stats` is null', () => {
const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
expect(
getSizeInBytes({
indexName,
stats: null,
})
).toBeUndefined();
});
test('it returns the expected size for a green index, where `primaries.store.size_in_bytes` and `total.store.size_in_bytes` have different values', () => {
const indexName = 'auditbeat-custom-index-1';
expect(
getSizeInBytes({
indexName,
stats: mockStatsAuditbeatIndex,
})
).toEqual(mockStatsAuditbeatIndex[indexName].size_in_bytes);
});
});
describe('getTotalPatternIncompatible', () => {
test('it returns zero when multiple indices in the results results have a count of zero', () => {
const results: Record<string, DataQualityCheckResult> = {
'.ds-packetbeat-8.5.3-2023.02.04-000001': {
docsCount: 1630289,
error: null,
ilmPhase: 'hot',
incompatible: 0,
indexName: '.ds-packetbeat-8.5.3-2023.02.04-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'packetbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
'.ds-packetbeat-8.6.1-2023.02.04-000001': {
docsCount: 1628343,
error: null,
ilmPhase: 'hot',
incompatible: 0,
indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'packetbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
};
expect(getTotalPatternIncompatible(results)).toEqual(0);
});
test("it returns the expected total when some indices have incompatible fields, but others don't", () => {
const results: Record<string, DataQualityCheckResult> = {
'.ds-auditbeat-8.6.1-2023.02.07-000001': {
docsCount: 18086,
error: null,
ilmPhase: 'hot',
incompatible: 0,
indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
'auditbeat-custom-index-1': {
docsCount: 4,
error: null,
ilmPhase: 'unmanaged',
incompatible: 3,
indexName: 'auditbeat-custom-index-1',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
'auditbeat-custom-empty-index-1': {
docsCount: 0,
error: null,
ilmPhase: 'unmanaged',
incompatible: 1,
indexName: 'auditbeat-custom-empty-index-1',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
};
expect(getTotalPatternIncompatible(results)).toEqual(4);
});
test('it returns the expected total when some indices have undefined incompatible counts', () => {
const results: Record<string, DataQualityCheckResult> = {
'.ds-auditbeat-8.6.1-2023.02.07-000001': {
docsCount: 18086,
error: null,
ilmPhase: 'hot',
incompatible: undefined, // <-- this index has an undefined `incompatible`
indexName: '.ds-auditbeat-8.6.1-2023.02.07-000001',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
'auditbeat-custom-index-1': {
docsCount: 4,
error: null,
ilmPhase: 'unmanaged',
incompatible: 3,
indexName: 'auditbeat-custom-index-1',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
'auditbeat-custom-empty-index-1': {
docsCount: 0,
error: null,
ilmPhase: 'unmanaged',
incompatible: 1,
indexName: 'auditbeat-custom-empty-index-1',
markdownComments: ['foo', 'bar', 'baz'],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: Date.now(),
},
};
expect(getTotalPatternIncompatible(results)).toEqual(4);
});
test('it returns zero when `results` is empty', () => {
expect(getTotalPatternIncompatible({})).toEqual(0);
});
test('it returns undefined when `results` is undefined', () => {
expect(getTotalPatternIncompatible(undefined)).toBeUndefined();
});
});

47
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality_panel/utils/stats.ts Normal file
View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { DataQualityCheckResult, MeteringStatsIndex, PatternRollup } from '../types';
export const getSizeInBytes = ({
indexName,
stats,
}: {
indexName: string;
stats: Record<string, MeteringStatsIndex> | null;
}): number | undefined => (stats && stats[indexName]?.size_in_bytes) ?? undefined;
export const getDocsCount = ({
indexName,
stats,
}: {
indexName: string;
stats: Record<string, MeteringStatsIndex> | null;
}): number => (stats && stats[indexName]?.num_docs) ?? 0;
export const getTotalPatternIndicesChecked = (patternRollup: PatternRollup | undefined): number => {
if (patternRollup != null && patternRollup.results != null) {
const allResults = Object.values(patternRollup.results);
const nonErrorResults = allResults.filter(({ error }) => error == null);
return nonErrorResults.length;
} else {
return 0;
}
};
export const getTotalPatternIncompatible = (
results: Record<string, DataQualityCheckResult> | undefined
): number | undefined => {
if (results == null) {
return undefined;
}
const allResults = Object.values(results);
return allResults.reduce<number>((acc, { incompatible }) => acc + (incompatible ?? 0), 0);
};

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/index.ts
View file

@ -7,7 +7,7 @@
export { DataQualityPanel } from './impl/data_quality_panel';
export { getIlmPhaseDescription } from './impl/data_quality_panel/helpers';
export { getIlmPhaseDescription } from './impl/data_quality_panel/utils/get_ilm_phase_description';
export {
DATA_QUALITY_PROMPT_CONTEXT_PILL,