github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Security Solution][DQD] Persist new fields in results storage (#185025)

Browse source 
Addresses #184751

## Summary

This PR addresses couple of issues:

### Main:
Persist revamped `resultsFieldMap` schema fields, namely
`incompatibleFieldMappingItems`, `incompatibleFieldValueItems` and
`sameFamilyFieldItems` in the `StorageResult` after index check, so that
after release user can start accumulating data in these fields, while we
prepare main UI changes.

### Additional:
Improve and narrow down existing in-house `EcsFlat` override type that
originally comes from `@elastic/ecs` npm package, because currently it
is too generic and too loose, resulting in an unnecessary conditional
checks and leads to perception of impossible states most of which are
refactored, cleaned and fixed in this PR.

### Screenshots

![image](1cd13459-cf15-4026-84e8-3dea05eedf4d)

![image](92593502-598a-439c-8c8e-fe3174ba963e)

![image](67472930-5aee-4689-b748-44235bf4d9c0)

### How to test

1. Prepare index with invalid mapping and value fields + 1 same family
field
```graphql
DELETE test-field-items

PUT test-field-items
{
  "mappings": {
    "properties": {
      "event.category": { "type": "keyword"},
      "agent.type": {"type": "constant_keyword" },
      "source.ip": {"type": "text"}
    }
  }
}

PUT test-field-items/_doc/1
{
  "@timestamp": "2016-05-23T08:05:34.853Z",
  "event.category": "behavior"
}

PUT test-field-items/_doc/2
{
  "@timestamp": "2016-05-23T08:05:34.853Z",
  "event.category": "shmehavior"
}
```  
2. Open DQD dashboard in kibana
3. Create `test-*` data-view with `test-*` index pattern
4. Select it in the sourcerer
5. Click expand button near test-field-items index 
6. Verify that you have 1 mapping + 1 value incompatible field + 1 same
family field
7. Open kibana devtools 
8. Run
```graphql
GET .kibana-data-quality-dashboard-results-default/_search
{
  "size": 0,
  "query": { 
    "term": {
      "indexName": {
        "value": "test-field-items"
      }
    } 
  },
  "aggs": {
    "latest": {
      "terms": { "field": "indexName", "size": 10000 },
      "aggs": { 
        "latest_doc": { 
          "top_hits": { 
            "size": 1, 
            "sort": [{ "@timestamp": { "order": "desc" } }] 
          } 
        } 
      }
    }
  }
}
```
9. Verify that latest result contains `incompatibleFieldItems` and
`sameFamilyFieldItems` of expected shape:
```json5
//...
                     "incompatibleFieldValueItems": [
                      {
                        "fieldName": "event.category",
                        "expectedValues": [
                          "api",
                          "authentication",
                          "configuration",
                          "database",
                          "driver",
                          "email",
                          "file",
                          "host",
                          "iam",
                          "intrusion_detection",
                          "library",
                          "malware",
                          "network",
                          "package",
                          "process",
                          "registry",
                          "session",
                          "threat",
                          "vulnerability",
                          "web"
                        ],
                        "actualValues": [
                          { "name": "behavior",  count: 2 },
                          { "name": "shmehavior", count: 1}
                        ],
                        "description": """This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.
`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.
This field is an array. This will allow proper categorization of some events that fall in multiple categories."""
                      }
                     ],
                     "incompatibleFieldMappingItems": [
                      {
                        "fieldName": "source.ip",
                        "expectedValue": "ip",
                        "actualValue": "text",
                        "description": "IP address of the source (IPv4 or IPv6)."
                      }
                    ]
//...
"sameFamilyFieldItems": [
                      {
                        "fieldName": "agent.type",
                        "expectedValue": "keyword",
                        "actualValue": "constant_keyword",
                        "description": """Type of the agent.
The agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine."""
                      }
                    ]
```
This commit is contained in:
Karen Grigoryan 2024-06-13 11:14:48 +02:00 • committed by GitHub
parent ed70d4c6ff
commit 4bc122703c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
45 changed files with 568 additions and 624 deletions
Download patch file Download diff file Expand all files Collapse all files

9
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.test.tsx
View file

@ -12,6 +12,7 @@ import React from 'react';
import { SAME_FAMILY } from '../../data_quality_panel/same_family/translations';
import {
eventCategory,
someField,
eventCategoryWithUnallowedValues,
} from '../../mock/enriched_field_metadata/mock_enriched_field_metadata';
import { TestProviders } from '../../mock/test_providers/test_providers';
@ -261,15 +262,9 @@ describe('getCommonTableColumns', () => {
const columns = getCommonTableColumns();
const descriptionolumnRender = columns[5].render;
const withDescription: EnrichedFieldMetadata = {
...eventCategory,
description: undefined,
};
render(
<TestProviders>
{descriptionolumnRender != null &&
descriptionolumnRender(withDescription.description, withDescription)}
{descriptionolumnRender != null && descriptionolumnRender(undefined, someField)}
</TestProviders>
);

24
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_common_table_columns/index.tsx
View file

@ -44,19 +44,25 @@ export const getCommonTableColumns = (): Array<
{
field: 'indexFieldType',
name: i18n.INDEX_MAPPING_TYPE_ACTUAL,
render: (_, x) =>
x.type != null && x.indexFieldType !== x.type ? (
getIsInSameFamily({ ecsExpectedType: x.type, type: x.indexFieldType }) ? (
render: (_, x) => {
// if custom field or ecs based field with mapping match
if (!x.hasEcsMetadata || x.indexFieldType === x.type) {
return <CodeSuccess data-test-subj="codeSuccess">{x.indexFieldType}</CodeSuccess>;
}
// mapping mismatch due to same family
if (getIsInSameFamily({ ecsExpectedType: x.type, type: x.indexFieldType })) {
return (
<div>
<CodeSuccess data-test-subj="codeSuccess">{x.indexFieldType}</CodeSuccess>
<SameFamily />
</div>
) : (
<CodeDanger data-test-subj="codeDanger">{x.indexFieldType}</CodeDanger>
)
) : (
<CodeSuccess data-test-subj="codeSuccess">{x.indexFieldType}</CodeSuccess>
),
);
}
// mapping mismatch
return <CodeDanger data-test-subj="codeDanger">{x.indexFieldType}</CodeDanger>;
},
sortable: true,
truncateText: false,
width: '15%',

21
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.test.tsx
View file

@ -12,8 +12,8 @@ import React from 'react';
import { SAME_FAMILY } from '../../data_quality_panel/same_family/translations';
import { TestProviders } from '../../mock/test_providers/test_providers';
import { eventCategory } from '../../mock/enriched_field_metadata/mock_enriched_field_metadata';
import { EnrichedFieldMetadata } from '../../types';
import { EMPTY_PLACEHOLDER, getIncompatibleMappingsTableColumns } from '.';
import { EcsBasedFieldMetadata } from '../../types';
import { getIncompatibleMappingsTableColumns } from '.';
describe('getIncompatibleMappingsTableColumns', () => {
test('it returns the expected column configuration', () => {
@ -65,19 +65,6 @@ describe('getIncompatibleMappingsTableColumns', () => {
expect(screen.getByTestId('codeSuccess')).toHaveTextContent(expected);
});
test('it renders an empty placeholder when type is undefined', () => {
const columns = getIncompatibleMappingsTableColumns();
const typeColumnRender = columns[1].render;
render(
<TestProviders>
{typeColumnRender != null && typeColumnRender(undefined, eventCategory)}
</TestProviders>
);
expect(screen.getByTestId('codeSuccess')).toHaveTextContent(EMPTY_PLACEHOLDER);
});
});
describe('indexFieldType column render()', () => {
@ -88,7 +75,7 @@ describe('getIncompatibleMappingsTableColumns', () => {
const columns = getIncompatibleMappingsTableColumns();
const indexFieldTypeColumnRender = columns[2].render;
const withTypeMismatchSameFamily: EnrichedFieldMetadata = {
const withTypeMismatchSameFamily: EcsBasedFieldMetadata = {
...eventCategory, // `event.category` is a `keyword` per the ECS spec
indexFieldType, // this index has a mapping of `wildcard` instead of `keyword`
isInSameFamily: true, // `wildcard` and `keyword` are in the same family
@ -121,7 +108,7 @@ describe('getIncompatibleMappingsTableColumns', () => {
const columns = getIncompatibleMappingsTableColumns();
const indexFieldTypeColumnRender = columns[2].render;
const withTypeMismatchDifferentFamily: EnrichedFieldMetadata = {
const withTypeMismatchDifferentFamily: EcsBasedFieldMetadata = {
...eventCategory, // `event.category` is a `keyword` per the ECS spec
indexFieldType, // this index has a mapping of `text` instead of `keyword`
isInSameFamily: false, // `text` and `wildcard` are not in the same family

10
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/get_incompatible_mappings_table_columns/index.tsx
View file

@ -11,12 +11,12 @@ import React from 'react';
import { SameFamily } from '../../data_quality_panel/same_family';
import { CodeDanger, CodeSuccess } from '../../styles';
import * as i18n from '../translations';
import type { EnrichedFieldMetadata } from '../../types';
import type { EcsBasedFieldMetadata } from '../../types';
export const EMPTY_PLACEHOLDER = '--';
export const getIncompatibleMappingsTableColumns = (): Array<
EuiTableFieldDataColumnType<EnrichedFieldMetadata>
EuiTableFieldDataColumnType<EcsBasedFieldMetadata>
> => [
{
field: 'indexFieldName',
@ -28,11 +28,7 @@ export const getIncompatibleMappingsTableColumns = (): Array<
{
field: 'type',
name: i18n.ECS_MAPPING_TYPE_EXPECTED,
render: (type: string) => (
<CodeSuccess data-test-subj="codeSuccess">
{type != null ? type : EMPTY_PLACEHOLDER}
</CodeSuccess>
),
render: (type: string) => <CodeSuccess data-test-subj="codeSuccess">{type}</CodeSuccess>,
sortable: true,
truncateText: false,
width: '25%',

55
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.test.tsx
View file

@ -10,7 +10,6 @@ import { omit } from 'lodash/fp';
import React from 'react';
import {
EMPTY_PLACEHOLDER,
getCustomTableColumns,
getEcsCompliantTableColumns,
getIncompatibleValuesTableColumns,
@ -117,31 +116,6 @@ describe('helpers', () => {
expect(screen.queryByTestId('typePlaceholder')).not.toBeInTheDocument();
});
});
describe('when `type` is undefined', () => {
beforeEach(() => {
const withUndefinedType = {
...eventCategory,
type: undefined, // <--
};
const columns = getEcsCompliantTableColumns();
const typeRender = columns[1].render;
render(
<TestProviders>
<>{typeRender != null && typeRender(withUndefinedType.type, withUndefinedType)}</>
</TestProviders>
);
});
test('it does NOT render the `type`', () => {
expect(screen.queryByTestId('type')).not.toBeInTheDocument();
});
test('it renders the placeholder', () => {
expect(screen.getByTestId('typePlaceholder')).toHaveTextContent(EMPTY_PLACEHOLDER);
});
});
});
describe('allowed values render()', () => {
@ -230,35 +204,6 @@ describe('helpers', () => {
expect(screen.queryByTestId('emptyPlaceholder')).not.toBeInTheDocument();
});
});
describe('when `description` is undefined', () => {
const withUndefinedDescription = {
...eventCategory,
description: undefined, // <--
};
beforeEach(() => {
const columns = getEcsCompliantTableColumns();
const descriptionRender = columns[3].render;
render(
<TestProviders>
<>
{descriptionRender != null &&
descriptionRender(withUndefinedDescription.description, withUndefinedDescription)}
</>
</TestProviders>
);
});
test('it does NOT render the `description`', () => {
expect(screen.queryByTestId('description')).not.toBeInTheDocument();
});
test('it renders the placeholder', () => {
expect(screen.getByTestId('emptyPlaceholder')).toBeInTheDocument();
});
});
});
});

27
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/helpers.tsx
View file

@ -13,12 +13,17 @@ import { EcsAllowedValues } from './ecs_allowed_values';
import { IndexInvalidValues } from './index_invalid_values';
import { CodeSuccess } from '../styles';
import * as i18n from './translations';
import type { AllowedValue, EnrichedFieldMetadata, UnallowedValueCount } from '../types';
import type {
AllowedValue,
CustomFieldMetadata,
EcsBasedFieldMetadata,
UnallowedValueCount,
} from '../types';
export const EMPTY_PLACEHOLDER = '--';
export const getCustomTableColumns = (): Array<
EuiTableFieldDataColumnType<EnrichedFieldMetadata>
EuiTableFieldDataColumnType<CustomFieldMetadata>
> => [
{
field: 'indexFieldName',
@ -40,7 +45,7 @@ export const getCustomTableColumns = (): Array<
];
export const getEcsCompliantTableColumns = (): Array<
EuiTableFieldDataColumnType<EnrichedFieldMetadata>
EuiTableFieldDataColumnType<EcsBasedFieldMetadata>
> => [
{
field: 'indexFieldName',
@ -52,12 +57,7 @@ export const getEcsCompliantTableColumns = (): Array<
{
field: 'type',
name: i18n.ECS_MAPPING_TYPE,
render: (type: string | undefined) =>
type != null ? (
<CodeSuccess data-test-subj="type">{type}</CodeSuccess>
) : (
<EuiCode data-test-subj="typePlaceholder">{EMPTY_PLACEHOLDER}</EuiCode>
),
render: (type: string) => <CodeSuccess data-test-subj="type">{type}</CodeSuccess>,
sortable: true,
truncateText: false,
width: '25%',
@ -75,12 +75,7 @@ export const getEcsCompliantTableColumns = (): Array<
{
field: 'description',
name: i18n.ECS_DESCRIPTION,
render: (description: string | undefined) =>
description != null ? (
<span data-test-subj="description">{description}</span>
) : (
<EuiCode data-test-subj="emptyPlaceholder">{EMPTY_PLACEHOLDER}</EuiCode>
),
render: (description: string) => <span data-test-subj="description">{description}</span>,
sortable: false,
truncateText: false,
width: '25%',
@ -88,7 +83,7 @@ export const getEcsCompliantTableColumns = (): Array<
];
export const getIncompatibleValuesTableColumns = (): Array<
EuiTableFieldDataColumnType<EnrichedFieldMetadata>
EuiTableFieldDataColumnType<EcsBasedFieldMetadata>
> => [
{
field: 'indexFieldName',

16
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/compare_fields_table/index.tsx
View file

@ -20,17 +20,17 @@ const search: Search = {
},
};
interface Props {
enrichedFieldMetadata: EnrichedFieldMetadata[];
getTableColumns: () => Array<EuiTableFieldDataColumnType<EnrichedFieldMetadata>>;
interface Props<T extends EnrichedFieldMetadata> {
enrichedFieldMetadata: T[];
getTableColumns: () => Array<EuiTableFieldDataColumnType<T>>;
title: string;
}
const CompareFieldsTableComponent: React.FC<Props> = ({
const CompareFieldsTableComponent = <T extends EnrichedFieldMetadata>({
enrichedFieldMetadata,
getTableColumns,
title,
}) => {
}: Props<T>): React.ReactElement => {
const columns = useMemo(() => getTableColumns(), [getTableColumns]);
return (
@ -53,4 +53,8 @@ const CompareFieldsTableComponent: React.FC<Props> = ({
CompareFieldsTableComponent.displayName = 'CompareFieldsTableComponent';
export const CompareFieldsTable = React.memo(CompareFieldsTableComponent);
export const CompareFieldsTable = React.memo(
CompareFieldsTableComponent
// React.memo doesn't pass generics through so
// this is a cheap fix without sacrificing type safety
) as typeof CompareFieldsTableComponent;

12
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/constants.ts Normal file
View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EcsFlat } from '@elastic/ecs';
import { EcsFieldMetadata } from './types';
export const EcsFlatTyped = EcsFlat as unknown as Record<string, EcsFieldMetadata>;
export type EcsFlatTyped = typeof EcsFlatTyped;

78
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.test.tsx
View file

@ -5,20 +5,15 @@
* 2.0.
*/
import { EcsFlat } from '@elastic/ecs';
import { omit } from 'lodash/fp';
import { EcsFlatTyped } from '../../constants';
import { getUnallowedValueRequestItems, getValidValues, hasAllowedValues } from './helpers';
import { AllowedValue, EcsMetadata } from '../../types';
const ecsMetadata: Record<string, EcsMetadata> = EcsFlat as unknown as Record<string, EcsMetadata>;
describe('helpers', () => {
describe('hasAllowedValues', () => {
test('it returns true for a field that has `allowed_values`', () => {
expect(
hasAllowedValues({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldName: 'event.category',
})
).toBe(true);
@ -27,7 +22,7 @@ describe('helpers', () => {
test('it returns false for a field that does NOT have `allowed_values`', () => {
expect(
hasAllowedValues({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldName: 'host.name',
})
).toBe(false);
@ -36,25 +31,16 @@ describe('helpers', () => {
test('it returns false for a field that does NOT exist in `ecsMetadata`', () => {
expect(
hasAllowedValues({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldName: 'does.NOT.exist',
})
).toBe(false);
});
test('it returns false when `ecsMetadata` is null', () => {
expect(
hasAllowedValues({
ecsMetadata: null, // <--
fieldName: 'event.category',
})
).toBe(false);
});
});
describe('getValidValues', () => {
test('it returns the expected valid values', () => {
expect(getValidValues(ecsMetadata['event.category'])).toEqual(
expect(getValidValues(EcsFlatTyped['event.category'])).toEqual(
expect.arrayContaining([
'authentication',
'configuration',
@ -79,60 +65,19 @@ describe('helpers', () => {
});
test('it returns an empty array when the `field` does NOT have `allowed_values`', () => {
expect(getValidValues(ecsMetadata['host.name'])).toEqual([]);
expect(getValidValues(EcsFlatTyped['host.name'])).toEqual([]);
});
test('it returns an empty array when `field` is undefined', () => {
expect(getValidValues(undefined)).toEqual([]);
});
test('it skips `allowed_values` where `name` is undefined', () => {
// omit the `name` property from the `database` `AllowedValue`:
const missingDatabase =
ecsMetadata['event.category'].allowed_values?.map((x) =>
x.name === 'database' ? omit<AllowedValue>('name', x) : x
) ?? [];
const field = {
...ecsMetadata['event.category'],
allowed_values: missingDatabase,
};
expect(getValidValues(field)).toEqual(
expect.arrayContaining([
'authentication',
'configuration',
'driver',
'email',
'file',
'host',
'iam',
'intrusion_detection',
'malware',
'network',
'package',
'process',
'registry',
'session',
'threat',
'vulnerability',
'web',
])
);
expect(getValidValues(field)).not.toEqual(
expect.arrayContaining([
// there should be no entry for 'database'
'database',
])
);
});
});
describe('getUnallowedValueRequestItems', () => {
test('it returns the expected request items', () => {
expect(
getUnallowedValueRequestItems({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
indexName: 'auditbeat-*',
})
).toEqual([
@ -203,14 +148,5 @@ describe('helpers', () => {
},
]);
});
test('it returns an empty array when `ecsMetadata` is null', () => {
expect(
getUnallowedValueRequestItems({
ecsMetadata: null, // <--
indexName: 'auditbeat-*',
})
).toEqual([]);
});
});
});

44
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/allowed_values/helpers.tsx
View file

@ -5,40 +5,38 @@
* 2.0.
*/
import type { EcsMetadata, UnallowedValueRequestItem } from '../../types';
import type { EcsFlatTyped } from '../../constants';
import type { EcsFieldMetadata, UnallowedValueRequestItem } from '../../types';
export const hasAllowedValues = ({
ecsMetadata,
fieldName,
}: {
ecsMetadata: Record<string, EcsMetadata> | null;
ecsMetadata: EcsFlatTyped;
fieldName: string;
}): boolean =>
ecsMetadata != null ? (ecsMetadata[fieldName]?.allowed_values?.length ?? 0) > 0 : false;
}): boolean => (ecsMetadata[fieldName]?.allowed_values?.length ?? 0) > 0;
export const getValidValues = (field: EcsMetadata | undefined): string[] =>
field?.allowed_values?.flatMap(({ name }) => (name != null ? name : [])) ?? [];
export const getValidValues = (field?: EcsFieldMetadata): string[] =>
field?.allowed_values?.flatMap(({ name }) => name) ?? [];
export const getUnallowedValueRequestItems = ({
ecsMetadata,
indexName,
}: {
ecsMetadata: Record<string, EcsMetadata> | null;
ecsMetadata: EcsFlatTyped;
indexName: string;
}): UnallowedValueRequestItem[] =>
ecsMetadata != null
? Object.keys(ecsMetadata).reduce<UnallowedValueRequestItem[]>(
(acc, fieldName) =>
hasAllowedValues({ ecsMetadata, fieldName })
? [
...acc,
{
indexName,
indexFieldName: fieldName,
allowedValues: getValidValues(ecsMetadata[fieldName]),
},
]
: acc,
[]
)
: [];
Object.keys(ecsMetadata).reduce<UnallowedValueRequestItem[]>(
(acc, fieldName) =>
hasAllowedValues({ ecsMetadata, fieldName })
? [
...acc,
{
indexName,
indexFieldName: fieldName,
allowedValues: getValidValues(ecsMetadata[fieldName]),
},
]
: acc,
[]
);

62
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.test.ts
View file

@ -5,15 +5,14 @@
* 2.0.
*/
import { EcsFlat, EcsVersion } from '@elastic/ecs';
import { EcsVersion } from '@elastic/ecs';
import { checkIndex, EMPTY_PARTITIONED_FIELD_METADATA } from './check_index';
import { EMPTY_STAT } from '../../../../helpers';
import { mockMappingsResponse } from '../../../../mock/mappings_response/mock_mappings_response';
import { mockUnallowedValuesResponse } from '../../../../mock/unallowed_values/mock_unallowed_values';
import { EcsMetadata, UnallowedValueRequestItem } from '../../../../types';
const ecsMetadata = EcsFlat as unknown as Record<string, EcsMetadata>;
import { UnallowedValueRequestItem } from '../../../../types';
import { EcsFlatTyped } from '../../../../constants';
let mockFetchMappings = jest.fn(
({
@ -99,7 +98,7 @@ describe('checkIndex', () => {
abortController: new AbortController(),
batchId: 'batch-id',
checkAllStartTime: Date.now(),
ecsMetadata,
ecsMetadata: EcsFlatTyped,
formatBytes,
formatNumber,
httpFetch,
@ -149,7 +148,7 @@ describe('checkIndex', () => {
abortController,
batchId: 'batch-id',
checkAllStartTime: Date.now(),
ecsMetadata,
ecsMetadata: EcsFlatTyped,
formatBytes,
formatNumber,
httpFetch,
@ -164,51 +163,6 @@ describe('checkIndex', () => {
});
});
describe('when `ecsMetadata` is null', () => {
const onCheckCompleted = jest.fn();
beforeEach(async () => {
jest.clearAllMocks();
await checkIndex({
abortController: new AbortController(),
batchId: 'batch-id',
checkAllStartTime: Date.now(),
ecsMetadata: null, // <--
formatBytes,
formatNumber,
httpFetch,
indexName,
isLastCheck: false,
onCheckCompleted,
pattern,
version: EcsVersion,
});
});
test('it invokes onCheckCompleted with a null `error`', () => {
expect(onCheckCompleted.mock.calls[0][0].error).toBeNull();
});
test('it invokes onCheckCompleted with the expected `indexName`', () => {
expect(onCheckCompleted.mock.calls[0][0].indexName).toEqual(indexName);
});
test('it invokes onCheckCompleted with the default `partitionedFieldMetadata`', () => {
expect(onCheckCompleted.mock.calls[0][0].partitionedFieldMetadata).toEqual(
EMPTY_PARTITIONED_FIELD_METADATA
);
});
test('it invokes onCheckCompleted with the expected `pattern`', () => {
expect(onCheckCompleted.mock.calls[0][0].pattern).toEqual(pattern);
});
test('it invokes onCheckCompleted with the expected `version`', () => {
expect(onCheckCompleted.mock.calls[0][0].version).toEqual(EcsVersion);
});
});
describe('when an error occurs', () => {
const onCheckCompleted = jest.fn();
const error = 'simulated fetch mappings error';
@ -230,7 +184,7 @@ describe('checkIndex', () => {
abortController: new AbortController(),
batchId: 'batch-id',
checkAllStartTime: Date.now(),
ecsMetadata,
ecsMetadata: EcsFlatTyped,
formatBytes,
formatNumber,
httpFetch,
@ -284,7 +238,7 @@ describe('checkIndex', () => {
abortController: new AbortController(),
batchId: 'batch-id',
checkAllStartTime: Date.now(),
ecsMetadata,
ecsMetadata: EcsFlatTyped,
formatBytes,
formatNumber,
httpFetch,
@ -346,7 +300,7 @@ describe('checkIndex', () => {
abortController,
batchId: 'batch-id',
checkAllStartTime: Date.now(),
ecsMetadata,
ecsMetadata: EcsFlatTyped,
formatBytes,
formatNumber,
httpFetch,

5
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/check_index.ts
View file

@ -12,9 +12,10 @@ import {
getSortedPartitionedFieldMetadata,
} from '../../../index_properties/helpers';
import * as i18n from './translations';
import type { EcsMetadata, OnCheckCompleted, PartitionedFieldMetadata } from '../../../../types';
import type { OnCheckCompleted, PartitionedFieldMetadata } from '../../../../types';
import { fetchMappings } from '../../../../use_mappings/helpers';
import { fetchUnallowedValues, getUnallowedValues } from '../../../../use_unallowed_values/helpers';
import { EcsFlatTyped } from '../../../../constants';
export const EMPTY_PARTITIONED_FIELD_METADATA: PartitionedFieldMetadata = {
all: [],
@ -41,7 +42,7 @@ export async function checkIndex({
abortController: AbortController;
batchId: string;
checkAllStartTime: number;
ecsMetadata: Record<string, EcsMetadata> | null;
ecsMetadata: EcsFlatTyped;
formatBytes: (value: number | undefined) => string;
formatNumber: (value: number | undefined) => string;
httpFetch: HttpHandler;

7
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_summary/summary_actions/check_all/index.tsx
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { EcsFlat, EcsVersion } from '@elastic/ecs';
import { EcsVersion } from '@elastic/ecs';
import { EuiButton } from '@elastic/eui';
import React, { useCallback, useEffect, useRef, useState } from 'react';
@ -16,7 +16,8 @@ import { checkIndex } from './check_index';
import { useDataQualityContext } from '../../../data_quality_context';
import { getAllIndicesToCheck } from './helpers';
import * as i18n from '../../../../translations';
import type { EcsMetadata, IndexToCheck, OnCheckCompleted } from '../../../../types';
import type { IndexToCheck, OnCheckCompleted } from '../../../../types';
import { EcsFlatTyped } from '../../../../constants';
const CheckAllButton = styled(EuiButton)`
width: 112px;
@ -97,7 +98,7 @@ const CheckAllComponent: React.FC<Props> = ({
abortController: abortController.current,
batchId,
checkAllStartTime: startTime,
ecsMetadata: EcsFlat as unknown as Record<string, EcsMetadata>,
ecsMetadata: EcsFlatTyped,
formatBytes,
formatNumber,
httpFetch,

52
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.test.ts
View file

@ -5,8 +5,6 @@
* 2.0.
*/
import { EcsFlat } from '@elastic/ecs';
import {
getMappingsProperties,
getSortedPartitionedFieldMetadata,
@ -14,16 +12,14 @@ import {
} from './helpers';
import { mockIndicesGetMappingIndexMappingRecords } from '../../mock/indices_get_mapping_index_mapping_record/mock_indices_get_mapping_index_mapping_record';
import { mockMappingsProperties } from '../../mock/mappings_properties/mock_mappings_properties';
import { EcsMetadata } from '../../types';
const ecsMetadata: Record<string, EcsMetadata> = EcsFlat as unknown as Record<string, EcsMetadata>;
import { EcsFlatTyped } from '../../constants';
describe('helpers', () => {
describe('getSortedPartitionedFieldMetadata', () => {
test('it returns null when mappings are loading', () => {
expect(
getSortedPartitionedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
loadingMappings: true, // <--
mappingsProperties: mockMappingsProperties,
unallowedValues: {},
@ -31,21 +27,10 @@ describe('helpers', () => {
).toBeNull();
});
test('it returns null when `ecsMetadata` is null', () => {
expect(
getSortedPartitionedFieldMetadata({
ecsMetadata: null, // <--
loadingMappings: false,
mappingsProperties: mockMappingsProperties,
unallowedValues: {},
})
).toBeNull();
});
test('it returns null when `unallowedValues` is null', () => {
expect(
getSortedPartitionedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
loadingMappings: false,
mappingsProperties: mockMappingsProperties,
unallowedValues: null, // <--
@ -54,30 +39,27 @@ describe('helpers', () => {
});
describe('when `mappingsProperties` is unknown', () => {
const incompatibleFieldMetadata = {
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-',
indexInvalidValues: [],
isEcsCompliant: false,
isInSameFamily: false,
};
const expected = {
all: [],
all: [incompatibleFieldMetadata],
custom: [],
ecsCompliant: [],
incompatible: [
{
description:
'Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events.',
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-',
indexInvalidValues: [],
isEcsCompliant: false,
isInSameFamily: false,
type: 'date',
},
],
incompatible: [incompatibleFieldMetadata],
sameFamily: [],
};
test('it returns a `PartitionedFieldMetadata` with an `incompatible` `@timestamp` when `mappingsProperties` is undefined', () => {
expect(
getSortedPartitionedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
loadingMappings: false,
mappingsProperties: undefined, // <--
unallowedValues: {},
@ -88,7 +70,7 @@ describe('helpers', () => {
test('it returns a `PartitionedFieldMetadata` with an `incompatible` `@timestamp` when `mappingsProperties` is null', () => {
expect(
getSortedPartitionedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
loadingMappings: false,
mappingsProperties: null, // <--
unallowedValues: {},
@ -116,7 +98,7 @@ describe('helpers', () => {
expect(
getSortedPartitionedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
loadingMappings: false,
mappingsProperties: mockMappingsProperties,
unallowedValues,

16
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/helpers.ts
View file

@ -10,6 +10,7 @@ import type {
MappingProperty,
} from '@elastic/elasticsearch/lib/api/types';
import { sortBy } from 'lodash/fp';
import { EcsFlatTyped } from '../../constants';
import {
getEnrichedFieldMetadata,
@ -17,7 +18,7 @@ import {
getMissingTimestampFieldMetadata,
getPartitionedFieldMetadata,
} from '../../helpers';
import type { EcsMetadata, PartitionedFieldMetadata, UnallowedValueCount } from '../../types';
import type { PartitionedFieldMetadata, UnallowedValueCount } from '../../types';
export const ALL_TAB_ID = 'allTab';
export const ECS_COMPLIANT_TAB_ID = 'ecsCompliantTab';
@ -40,19 +41,26 @@ export const getSortedPartitionedFieldMetadata = ({
mappingsProperties,
unallowedValues,
}: {
ecsMetadata: Record<string, EcsMetadata> | null;
ecsMetadata: EcsFlatTyped;
loadingMappings: boolean;
mappingsProperties: Record<string, MappingProperty> | null | undefined;
unallowedValues: Record<string, UnallowedValueCount[]> | null;
}): PartitionedFieldMetadata | null => {
if (loadingMappings || ecsMetadata == null || unallowedValues == null) {
if (loadingMappings || unallowedValues == null) {
return null;
}
// this covers scenario when we try to check an empty index
// or index without required @timestamp field in the mapping
//
// we create an artifical incompatible timestamp field metadata
// so that we can signal to user that the incompatibility is due to missing timestamp
if (mappingsProperties == null) {
const missingTimestampFieldMetadata = getMissingTimestampFieldMetadata();
return {
...EMPTY_METADATA,
incompatible: [getMissingTimestampFieldMetadata()],
all: [missingTimestampFieldMetadata],
incompatible: [missingTimestampFieldMetadata],
};
}

9
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx
View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { EcsFlat, EcsVersion } from '@elastic/ecs';
import { EcsVersion } from '@elastic/ecs';
import type {
FlameElementEvent,
HeatmapElementEvent,
@ -39,12 +39,13 @@ import {
getSameFamilyFields,
} from '../tabs/incompatible_tab/helpers';
import * as i18n from './translations';
import type { EcsMetadata, IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../types';
import type { IlmPhase, PartitionedFieldMetadata, PatternRollup } from '../../types';
import { useAddToNewCase } from '../../use_add_to_new_case';
import { useMappings } from '../../use_mappings';
import { useUnallowedValues } from '../../use_unallowed_values';
import { useDataQualityContext } from '../data_quality_context';
import { formatStorageResult, postStorageResult, getSizeInBytes } from '../../helpers';
import { EcsFlatTyped } from '../../constants';
const EMPTY_MARKDOWN_COMMENTS: string[] = [];
@ -109,7 +110,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
const requestItems = useMemo(
() =>
getUnallowedValueRequestItems({
ecsMetadata: EcsFlat as unknown as Record<string, EcsMetadata>,
ecsMetadata: EcsFlatTyped,
indexName,
}),
[indexName]
@ -134,7 +135,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
const partitionedFieldMetadata: PartitionedFieldMetadata | null = useMemo(
() =>
getSortedPartitionedFieldMetadata({
ecsMetadata: EcsFlat as unknown as Record<string, EcsMetadata>,
ecsMetadata: EcsFlatTyped,
loadingMappings,
mappingsProperties,
unallowedValues,

17
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.test.ts
View file

@ -57,7 +57,7 @@ import {
import { SAME_FAMILY } from '../../same_family/translations';
import { INCOMPATIBLE_FIELD_MAPPINGS_TABLE_TITLE } from '../../tabs/incompatible_tab/translations';
import {
EnrichedFieldMetadata,
EcsBasedFieldMetadata,
ErrorSummary,
PatternRollup,
UnallowedValueCount,
@ -230,17 +230,6 @@ describe('helpers', () => {
'| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |'
);
});
test('it returns the expected table rows when some have allowed values', () => {
const withAllowedValues = [
...mockCustomFields,
eventCategory, // note: this is not a real-world use case, because custom fields don't have allowed values
];
expect(getCustomMarkdownTableRows(withAllowedValues)).toEqual(
'| host.name.keyword | `keyword` | `--` |\n| some.field | `text` | `--` |\n| some.field.keyword | `keyword` | `--` |\n| source.ip.keyword | `keyword` | `--` |\n| event.category | `keyword` | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` |'
);
});
});
describe('getSameFamilyBadge', () => {
@ -265,7 +254,7 @@ describe('helpers', () => {
describe('getIncompatibleMappingsMarkdownTableRows', () => {
test('it returns the expected table rows when the field is in the same family', () => {
const eventCategoryWithWildcard: EnrichedFieldMetadata = {
const eventCategoryWithWildcard: EcsBasedFieldMetadata = {
...eventCategory, // `event.category` is a `keyword` per the ECS spec
indexFieldType: 'wildcard', // this index has a mapping of `wildcard` instead of `keyword`
isInSameFamily: true, // `wildcard` and `keyword` are in the same family
@ -282,7 +271,7 @@ describe('helpers', () => {
});
test('it returns the expected table rows when the field is NOT in the same family', () => {
const eventCategoryWithText: EnrichedFieldMetadata = {
const eventCategoryWithText: EcsBasedFieldMetadata = {
...eventCategory, // `event.category` is a `keyword` per the ECS spec
indexFieldType: 'text', // this index has a mapping of `text` instead of `keyword`
isInSameFamily: false, // `text` and `keyword` are NOT in the same family

24
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/markdown/helpers.ts
View file

@ -25,6 +25,8 @@ import { HOT, WARM, COLD, FROZEN, UNMANAGED } from '../../../ilm_phases_empty_pr
import * as i18n from '../translations';
import type {
AllowedValue,
CustomFieldMetadata,
EcsBasedFieldMetadata,
EnrichedFieldMetadata,
ErrorSummary,
IlmExplainPhaseCounts,
@ -85,23 +87,21 @@ export const getIndexInvalidValues = (indexInvalidValues: UnallowedValueCount[])
.map(({ fieldName, count }) => `${getCodeFormattedValue(escape(fieldName))} (${count})`)
.join(', '); // newlines are instead joined with spaces
export const getCustomMarkdownTableRows = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): string =>
enrichedFieldMetadata
export const getCustomMarkdownTableRows = (customFieldMetadata: CustomFieldMetadata[]): string =>
customFieldMetadata
.map(
(x) =>
`| ${escape(x.indexFieldName)} | ${getCodeFormattedValue(
x.indexFieldType
)} | ${getAllowedValues(x.allowed_values)} |`
)} | ${getAllowedValues(undefined)} |`
)
.join('\n');
export const getSameFamilyBadge = (enrichedFieldMetadata: EnrichedFieldMetadata): string =>
enrichedFieldMetadata.isInSameFamily ? getCodeFormattedValue(SAME_FAMILY) : '';
export const getSameFamilyBadge = (ecsBasedFieldMetadata: EcsBasedFieldMetadata): string =>
ecsBasedFieldMetadata.isInSameFamily ? getCodeFormattedValue(SAME_FAMILY) : '';
export const getIncompatibleMappingsMarkdownTableRows = (
incompatibleMappings: EnrichedFieldMetadata[]
incompatibleMappings: EcsBasedFieldMetadata[]
): string =>
incompatibleMappings
.map(
@ -113,7 +113,7 @@ export const getIncompatibleMappingsMarkdownTableRows = (
.join('\n');
export const getIncompatibleValuesMarkdownTableRows = (
incompatibleValues: EnrichedFieldMetadata[]
incompatibleValues: EcsBasedFieldMetadata[]
): string =>
incompatibleValues
.map(
@ -173,14 +173,14 @@ ${getMarkdownTableRows(errorSummary)}
`
: '';
export const getMarkdownTable = ({
export const getMarkdownTable = <T extends EnrichedFieldMetadata[]>({
enrichedFieldMetadata,
getMarkdownTableRows,
headerNames,
title,
}: {
enrichedFieldMetadata: EnrichedFieldMetadata[];
getMarkdownTableRows: (enrichedFieldMetadata: EnrichedFieldMetadata[]) => string;
enrichedFieldMetadata: T;
getMarkdownTableRows: (enrichedFieldMetadata: T) => string;
headerNames: string[];
title: string;
}): string =>

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.test.tsx
View file

@ -23,7 +23,7 @@ describe('CustomCallout', () => {
beforeEach(() => {
render(
<TestProviders>
<CustomCallout enrichedFieldMetadata={[hostNameKeyword, someField]}>
<CustomCallout customFieldMetadata={[hostNameKeyword, someField]}>
<div data-test-subj="children">{content}</div>
</CustomCallout>
</TestProviders>

12
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/custom_callout/index.tsx
View file

@ -9,27 +9,27 @@ import { EcsVersion } from '@elastic/ecs';
import { EuiCallOut, EuiSpacer } from '@elastic/eui';
import React, { useMemo } from 'react';
import type { EnrichedFieldMetadata } from '../../../../types';
import type { CustomFieldMetadata } from '../../../../types';
import * as i18n from '../../../index_properties/translations';
interface Props {
children?: React.ReactNode;
enrichedFieldMetadata: EnrichedFieldMetadata[];
customFieldMetadata: CustomFieldMetadata[];
}
const CustomCalloutComponent: React.FC<Props> = ({ children, enrichedFieldMetadata }) => {
const CustomCalloutComponent: React.FC<Props> = ({ children, customFieldMetadata }) => {
const title = useMemo(
() => (
<span data-test-subj="title">{i18n.CUSTOM_CALLOUT_TITLE(enrichedFieldMetadata.length)}</span>
<span data-test-subj="title">{i18n.CUSTOM_CALLOUT_TITLE(customFieldMetadata.length)}</span>
),
[enrichedFieldMetadata.length]
[customFieldMetadata.length]
);
return (
<EuiCallOut color="primary" size="s" title={title}>
<div data-test-subj="fieldsNotDefinedByEcs">
{i18n.CUSTOM_CALLOUT({ fieldCount: enrichedFieldMetadata.length, version: EcsVersion })}
{i18n.CUSTOM_CALLOUT({ fieldCount: customFieldMetadata.length, version: EcsVersion })}
</div>
<EuiSpacer size="s" />
<div data-test-subj="ecsIsPermissive">{i18n.ECS_IS_A_PERMISSIVE_SCHEMA}</div>

82
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.test.ts
View file

@ -1,82 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { getIncompatiableFieldsInSameFamilyCount } from './helpers';
import {
eventCategory,
eventCategoryWithUnallowedValues,
hostNameWithTextMapping,
someField,
sourceIpWithTextMapping,
} from '../../../../mock/enriched_field_metadata/mock_enriched_field_metadata';
import { EnrichedFieldMetadata } from '../../../../types';
const sameFamily: EnrichedFieldMetadata = {
...eventCategory, // `event.category` is a `keyword` per the ECS spec
indexFieldType: 'wildcard', // this index has a mapping of `wildcard` instead of `keyword`
isInSameFamily: true, // `wildcard` and `keyword` are in the same family
isEcsCompliant: false, // wildcard !== keyword
};
describe('helpers', () => {
describe('getFieldsInSameFamilyCount', () => {
test('it filters out fields that are ECS compliant', () => {
expect(
getIncompatiableFieldsInSameFamilyCount([
eventCategory, // isEcsCompliant: true, indexInvalidValues.length: 0, isInSameFamily: true, `keyword` and `keyword` are in the same family
])
).toEqual(0);
});
test('it filters out fields with unallowed values', () => {
expect(
getIncompatiableFieldsInSameFamilyCount([
eventCategoryWithUnallowedValues, // isEcsCompliant: false, indexInvalidValues.length: 2, isInSameFamily: true, `keyword` and `keyword` are in the same family
])
).toEqual(0);
});
test('it filters out fields that are not in the same family', () => {
expect(
getIncompatiableFieldsInSameFamilyCount([
hostNameWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, `keyword` and `text` are not in the family
])
).toEqual(0);
});
test('it returns 1 for an incompatible field in the same family', () => {
expect(
getIncompatiableFieldsInSameFamilyCount([
sameFamily, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: true, `wildcard` and `keyword` are in the same family
])
).toEqual(1);
});
test('it returns the expected count when some of the input should be counted', () => {
expect(
getIncompatiableFieldsInSameFamilyCount([
sameFamily,
eventCategoryWithUnallowedValues, // isEcsCompliant: false, indexInvalidValues.length: 2, isInSameFamily: true, `keyword` and `keyword` are in the same family
hostNameWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length, isInSameFamily: false, `text` and `keyword` not in the same family
someField, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, custom fields are never in the same family
sourceIpWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, `ip` is not a member of any families
])
).toEqual(1);
});
test('it returns zero when none of the input should be counted', () => {
expect(
getIncompatiableFieldsInSameFamilyCount([
eventCategoryWithUnallowedValues, // isEcsCompliant: false, indexInvalidValues.length: 2, isInSameFamily: true, `keyword` and `keyword` are in the same family
hostNameWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length, isInSameFamily: false, `text` and `keyword` not in the same family
someField, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, custom fields are never in the same family
sourceIpWithTextMapping, // isEcsCompliant: false, indexInvalidValues.length: 0, isInSameFamily: false, `ip` is not a member of any families
])
).toEqual(0);
});
});
});

15
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/helpers.ts
View file

@ -1,15 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EnrichedFieldMetadata } from '../../../../types';
export const getIncompatiableFieldsInSameFamilyCount = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): number =>
enrichedFieldMetadata.filter(
(x) => !x.isEcsCompliant && x.indexInvalidValues.length === 0 && x.isInSameFamily
).length;

6
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/index.test.tsx
View file

@ -21,12 +21,12 @@ import {
sourceIpWithTextMapping,
} from '../../../../mock/enriched_field_metadata/mock_enriched_field_metadata';
import { TestProviders } from '../../../../mock/test_providers/test_providers';
import { EnrichedFieldMetadata } from '../../../../types';
import { EcsBasedFieldMetadata } from '../../../../types';
import { IncompatibleCallout } from '.';
const content = 'Is your name Michael?';
const eventCategoryWithWildcard: EnrichedFieldMetadata = {
const eventCategoryWithWildcard: EcsBasedFieldMetadata = {
...eventCategory, // `event.category` is a `keyword` per the ECS spec
indexFieldType: 'wildcard', // this index has a mapping of `wildcard` instead of `keyword`
isInSameFamily: true, // `wildcard` and `keyword` are in the same family
@ -38,7 +38,7 @@ describe('IncompatibleCallout', () => {
render(
<TestProviders>
<IncompatibleCallout
enrichedFieldMetadata={[
ecsBasedFieldMetadata={[
eventCategoryWithWildcard, // `wildcard` and `keyword`
eventCategoryWithUnallowedValues, // `keyword` and `keyword`
hostNameWithTextMapping, // `keyword` and `text`

8
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/incompatible_callout/index.tsx
View file

@ -12,15 +12,15 @@ import React, { useMemo } from 'react';
import * as i18n from '../../../index_properties/translations';
import { CalloutItem } from '../../styles';
import type { EnrichedFieldMetadata } from '../../../../types';
import type { EcsBasedFieldMetadata } from '../../../../types';
interface Props {
children?: React.ReactNode;
enrichedFieldMetadata: EnrichedFieldMetadata[];
ecsBasedFieldMetadata: EcsBasedFieldMetadata[];
}
const IncompatibleCalloutComponent: React.FC<Props> = ({ children, enrichedFieldMetadata }) => {
const fieldCount = enrichedFieldMetadata.length;
const IncompatibleCalloutComponent: React.FC<Props> = ({ children, ecsBasedFieldMetadata }) => {
const fieldCount = ecsBasedFieldMetadata.length;
const title = useMemo(
() => <span data-test-subj="title">{i18n.INCOMPATIBLE_CALLOUT_TITLE(fieldCount)}</span>,
[fieldCount]

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.test.tsx
View file

@ -21,7 +21,7 @@ describe('SameFamilyCallout', () => {
render(
<TestProviders>
<SameFamilyCallout
enrichedFieldMetadata={mockPartitionedFieldMetadataWithSameFamily.sameFamily}
ecsBasedFieldMetadata={mockPartitionedFieldMetadataWithSameFamily.sameFamily}
>
<div data-test-subj="children">{content}</div>
</SameFamilyCallout>

12
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/callouts/same_family_callout/index.tsx
View file

@ -10,28 +10,28 @@ import { EuiCallOut, EuiSpacer, EuiText } from '@elastic/eui';
import React, { useMemo } from 'react';
import * as i18n from '../../../index_properties/translations';
import type { EnrichedFieldMetadata } from '../../../../types';
import type { EcsBasedFieldMetadata } from '../../../../types';
interface Props {
children?: React.ReactNode;
enrichedFieldMetadata: EnrichedFieldMetadata[];
ecsBasedFieldMetadata: EcsBasedFieldMetadata[];
}
const SameFamilyCalloutComponent: React.FC<Props> = ({ children, enrichedFieldMetadata }) => {
const SameFamilyCalloutComponent: React.FC<Props> = ({ children, ecsBasedFieldMetadata }) => {
const title = useMemo(
() => (
<span data-test-subj="title">
{i18n.SAME_FAMILY_CALLOUT_TITLE(enrichedFieldMetadata.length)}
{i18n.SAME_FAMILY_CALLOUT_TITLE(ecsBasedFieldMetadata.length)}
</span>
),
[enrichedFieldMetadata.length]
[ecsBasedFieldMetadata.length]
);
return (
<EuiCallOut color="primary" size="s" title={title}>
<div data-test-subj="fieldsDefinedByEcs">
{i18n.SAME_FAMILY_CALLOUT({
fieldCount: enrichedFieldMetadata.length,
fieldCount: ecsBasedFieldMetadata.length,
version: EcsVersion,
})}
</div>

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.test.ts
View file

@ -35,7 +35,7 @@ const formatNumber = (value: number | undefined) =>
describe('helpers', () => {
describe('getCustomMarkdownComment', () => {
test('it returns a comment for custom fields with the expected field counts and ECS version', () => {
expect(getCustomMarkdownComment({ enrichedFieldMetadata: [hostNameKeyword, someField] }))
expect(getCustomMarkdownComment({ customFieldMetadata: [hostNameKeyword, someField] }))
.toEqual(`#### 2 Custom field mappings
These fields are not defined by the Elastic Common Schema (ECS), version ${EcsVersion}.

16
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/helpers.ts
View file

@ -19,26 +19,26 @@ import {
} from '../../index_properties/markdown/helpers';
import * as i18n from '../../index_properties/translations';
import { getFillColor } from '../summary_tab/helpers';
import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types';
import type { CustomFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types';
export const getCustomMarkdownComment = ({
enrichedFieldMetadata,
customFieldMetadata,
}: {
enrichedFieldMetadata: EnrichedFieldMetadata[];
customFieldMetadata: CustomFieldMetadata[];
}): string =>
getMarkdownComment({
suggestedAction: `${i18n.CUSTOM_CALLOUT({
fieldCount: enrichedFieldMetadata.length,
fieldCount: customFieldMetadata.length,
version: EcsVersion,
})}
${i18n.ECS_IS_A_PERMISSIVE_SCHEMA}
`,
title: i18n.CUSTOM_CALLOUT_TITLE(enrichedFieldMetadata.length),
title: i18n.CUSTOM_CALLOUT_TITLE(customFieldMetadata.length),
});
export const showCustomCallout = (enrichedFieldMetadata: EnrichedFieldMetadata[]): boolean =>
enrichedFieldMetadata.length > 0;
export const showCustomCallout = (customFieldMetadata: CustomFieldMetadata[]): boolean =>
customFieldMetadata.length > 0;
export const getCustomColor = (partitionedFieldMetadata: PartitionedFieldMetadata): string =>
showCustomCallout(partitionedFieldMetadata.custom)
@ -80,7 +80,7 @@ export const getAllCustomMarkdownComments = ({
}),
getTabCountsMarkdownComment(partitionedFieldMetadata),
getCustomMarkdownComment({
enrichedFieldMetadata: partitionedFieldMetadata.custom,
customFieldMetadata: partitionedFieldMetadata.custom,
}),
getMarkdownTable({
enrichedFieldMetadata: partitionedFieldMetadata.custom,

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/custom_tab/index.tsx
View file

@ -91,7 +91,7 @@ const CustomTabComponent: React.FC<Props> = ({
<>
{showCustomCallout(partitionedFieldMetadata.custom) ? (
<>
<CustomCallout enrichedFieldMetadata={partitionedFieldMetadata.custom}>
<CustomCallout customFieldMetadata={partitionedFieldMetadata.custom}>
<EuiFlexGroup alignItems="center" gutterSize="none">
<EuiFlexItem grow={false}>
<EuiButtonEmpty aria-label={i18n.COPY_TO_CLIPBOARD} flush="both" onClick={onCopy}>

5
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.test.tsx
View file

@ -11,7 +11,6 @@ import { omit } from 'lodash/fp';
import {
eventCategory,
someField,
timestamp,
} from '../../mock/enriched_field_metadata/mock_enriched_field_metadata';
import { mockPartitionedFieldMetadata } from '../../mock/partitioned_field_metadata/mock_partitioned_field_metadata';
@ -38,11 +37,11 @@ describe('helpers', () => {
});
test('it returns false when `enrichedFieldMetadata` contains an @timestamp field', () => {
expect(showMissingTimestampCallout([timestamp, eventCategory, someField])).toBe(false);
expect(showMissingTimestampCallout([timestamp, eventCategory])).toBe(false);
});
test('it returns true when `enrichedFieldMetadata` does NOT contain an @timestamp field', () => {
expect(showMissingTimestampCallout([eventCategory, someField])).toBe(true);
expect(showMissingTimestampCallout([eventCategory])).toBe(true);
});
});

6
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/helpers.tsx
View file

@ -39,7 +39,7 @@ import { SameFamilyTab } from './same_family_tab';
import { SummaryTab } from './summary_tab';
import { getFillColor } from './summary_tab/helpers';
import type {
EnrichedFieldMetadata,
EcsBasedFieldMetadata,
IlmPhase,
MeteringStatsIndex,
PartitionedFieldMetadata,
@ -56,8 +56,8 @@ ${i18n.PAGES_MAY_NOT_DISPLAY_EVENTS}
});
export const showMissingTimestampCallout = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): boolean => !enrichedFieldMetadata.some((x) => x.name === '@timestamp');
ecsBasedFieldMetadata: EcsBasedFieldMetadata[]
): boolean => !ecsBasedFieldMetadata.some((x) => x.name === '@timestamp');
export const getEcsCompliantColor = (partitionedFieldMetadata: PartitionedFieldMetadata): string =>
showMissingTimestampCallout(partitionedFieldMetadata.ecsCompliant)

34
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/helpers.ts
View file

@ -19,7 +19,7 @@ import {
} from '../../index_properties/markdown/helpers';
import { getFillColor } from '../summary_tab/helpers';
import * as i18n from '../../index_properties/translations';
import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types';
import type { EcsBasedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types';
import {
INCOMPATIBLE_FIELD_MAPPINGS_TABLE_TITLE,
INCOMPATIBLE_FIELD_VALUES_TABLE_TITLE,
@ -44,17 +44,17 @@ ${i18n.MAPPINGS_THAT_CONFLICT_WITH_ECS}
title: i18n.INCOMPATIBLE_CALLOUT_TITLE(incompatible),
});
export const showInvalidCallout = (enrichedFieldMetadata: EnrichedFieldMetadata[]): boolean =>
enrichedFieldMetadata.length > 0;
export const showInvalidCallout = (ecsBasedFieldMetadata: EcsBasedFieldMetadata[]): boolean =>
ecsBasedFieldMetadata.length > 0;
export const getIncompatibleColor = (): string => getFillColor('incompatible');
export const getSameFamilyColor = (): string => getFillColor('same-family');
export const getIncompatibleMappings = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): EnrichedFieldMetadata[] =>
enrichedFieldMetadata.filter(
ecsBasedFieldMetadata: EcsBasedFieldMetadata[]
): EcsBasedFieldMetadata[] =>
ecsBasedFieldMetadata.filter(
(x) =>
!x.isEcsCompliant &&
x.type !== x.indexFieldType &&
@ -62,9 +62,9 @@ export const getIncompatibleMappings = (
);
export const getIncompatibleMappingsFields = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
ecsBasedFieldMetadata: EcsBasedFieldMetadata[]
): string[] =>
enrichedFieldMetadata.reduce<string[]>((acc, x) => {
ecsBasedFieldMetadata.reduce<string[]>((acc, x) => {
if (
!x.isEcsCompliant &&
x.type !== x.indexFieldType &&
@ -78,8 +78,8 @@ export const getIncompatibleMappingsFields = (
return acc;
}, []);
export const getSameFamilyFields = (enrichedFieldMetadata: EnrichedFieldMetadata[]): string[] =>
enrichedFieldMetadata.reduce<string[]>((acc, x) => {
export const getSameFamilyFields = (ecsBasedFieldMetadata: EcsBasedFieldMetadata[]): string[] =>
ecsBasedFieldMetadata.reduce<string[]>((acc, x) => {
if (!x.isEcsCompliant && x.type !== x.indexFieldType && x.isInSameFamily) {
const field = escape(x.indexFieldName);
if (field != null) {
@ -90,14 +90,14 @@ export const getSameFamilyFields = (enrichedFieldMetadata: EnrichedFieldMetadata
}, []);
export const getIncompatibleValues = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): EnrichedFieldMetadata[] =>
enrichedFieldMetadata.filter((x) => !x.isEcsCompliant && x.indexInvalidValues.length > 0);
ecsBasedFieldMetadata: EcsBasedFieldMetadata[]
): EcsBasedFieldMetadata[] =>
ecsBasedFieldMetadata.filter((x) => !x.isEcsCompliant && x.indexInvalidValues.length > 0);
export const getIncompatibleValuesFields = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
ecsBasedFieldMetadata: EcsBasedFieldMetadata[]
): string[] =>
enrichedFieldMetadata.reduce<string[]>((acc, x) => {
ecsBasedFieldMetadata.reduce<string[]>((acc, x) => {
if (!x.isEcsCompliant && x.indexInvalidValues.length > 0) {
const field = escape(x.indexFieldName);
if (field != null) {
@ -112,8 +112,8 @@ export const getIncompatibleFieldsMarkdownTablesComment = ({
incompatibleValues,
indexName,
}: {
incompatibleMappings: EnrichedFieldMetadata[];
incompatibleValues: EnrichedFieldMetadata[];
incompatibleMappings: EcsBasedFieldMetadata[];
incompatibleValues: EcsBasedFieldMetadata[];
indexName: string;
}): string => `
${

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/incompatible_tab/index.tsx
View file

@ -127,7 +127,7 @@ const IncompatibleTabComponent: React.FC<Props> = ({
<div data-test-subj="incompatibleTab">
{showInvalidCallout(partitionedFieldMetadata.incompatible) ? (
<>
<IncompatibleCallout enrichedFieldMetadata={partitionedFieldMetadata.incompatible}>
<IncompatibleCallout ecsBasedFieldMetadata={partitionedFieldMetadata.incompatible}>
<EuiFlexGroup alignItems="center" gutterSize="none">
<EuiFlexItem grow={false}>
<EuiButton

8
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/same_family_tab/helpers.ts
View file

@ -22,7 +22,7 @@ import {
} from '../../index_properties/markdown/helpers';
import * as i18n from '../../index_properties/translations';
import { SAME_FAMILY_FIELD_MAPPINGS_TABLE_TITLE } from './translations';
import type { EnrichedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types';
import type { EcsBasedFieldMetadata, IlmPhase, PartitionedFieldMetadata } from '../../../types';
export const getSameFamilyMarkdownComment = (fieldsInSameFamily: number): string =>
getMarkdownComment({
@ -37,14 +37,14 @@ ${i18n.FIELDS_WITH_MAPPINGS_SAME_FAMILY}
});
export const getSameFamilyMappings = (
enrichedFieldMetadata: EnrichedFieldMetadata[]
): EnrichedFieldMetadata[] => enrichedFieldMetadata.filter((x) => x.isInSameFamily);
enrichedFieldMetadata: EcsBasedFieldMetadata[]
): EcsBasedFieldMetadata[] => enrichedFieldMetadata.filter((x) => x.isInSameFamily);
export const getSameFamilyMarkdownTablesComment = ({
sameFamilyMappings,
indexName,
}: {
sameFamilyMappings: EnrichedFieldMetadata[];
sameFamilyMappings: EcsBasedFieldMetadata[];
indexName: string;
}): string => `
${

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/same_family_tab/index.tsx
View file

@ -83,7 +83,7 @@ const SameFamilyTabComponent: React.FC<Props> = ({
return (
<div data-test-subj="sameFamilyTab">
<SameFamilyCallout enrichedFieldMetadata={partitionedFieldMetadata.sameFamily}>
<SameFamilyCallout ecsBasedFieldMetadata={partitionedFieldMetadata.sameFamily}>
<EuiButtonEmpty aria-label={i18n.COPY_TO_CLIPBOARD} flush="both" onClick={onCopy}>
{i18n.COPY_TO_CLIPBOARD}
</EuiButtonEmpty>

2
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/callout_summary/index.tsx
View file

@ -109,7 +109,7 @@ const CalloutSummaryComponent: React.FC<Props> = ({
<>
{showInvalidCallout(partitionedFieldMetadata.incompatible) && (
<>
<IncompatibleCallout enrichedFieldMetadata={partitionedFieldMetadata.incompatible} />
<IncompatibleCallout ecsBasedFieldMetadata={partitionedFieldMetadata.incompatible} />
<EuiSpacer size="s" />
</>
)}

5
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/tabs/summary_tab/helpers.test.ts
View file

@ -35,6 +35,7 @@ import {
getSummaryData,
getTabId,
} from './helpers';
import { EcsFlatTyped } from '../../../constants';
describe('helpers', () => {
describe('getSummaryData', () => {
@ -216,15 +217,13 @@ describe('helpers', () => {
custom: [],
incompatible: [
{
description:
'Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events.',
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-',
indexInvalidValues: [],
isEcsCompliant: false,
isInSameFamily: false,
type: 'date',
},
],
sameFamily: [],

171
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts
View file

@ -7,7 +7,6 @@
import { IlmExplainLifecycleLifecycleExplain } from '@elastic/elasticsearch/lib/api/types';
import { euiThemeVars } from '@kbn/ui-theme';
import { EcsFlat } from '@elastic/ecs';
import { omit } from 'lodash/fp';
import {
@ -33,11 +32,11 @@ import {
getTotalPatternIndicesChecked,
getTotalPatternSameFamily,
getTotalSizeInBytes,
hasValidTimestampMapping,
isMappingCompatible,
postStorageResult,
getStorageResults,
StorageResult,
formatStorageResult,
} from './helpers';
import {
hostNameWithTextMapping,
@ -68,13 +67,11 @@ import {
COLD_DESCRIPTION,
FROZEN_DESCRIPTION,
HOT_DESCRIPTION,
TIMESTAMP_DESCRIPTION,
UNMANAGED_DESCRIPTION,
WARM_DESCRIPTION,
} from './translations';
import {
DataQualityCheckResult,
EcsMetadata,
EnrichedFieldMetadata,
PartitionedFieldMetadata,
PatternRollup,
@ -82,8 +79,8 @@ import {
} from './types';
import { httpServiceMock } from '@kbn/core-http-browser-mocks';
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
const ecsMetadata: Record<string, EcsMetadata> = EcsFlat as unknown as Record<string, EcsMetadata>;
import { EcsFlatTyped } from './constants';
import { mockPartitionedFieldMetadataWithSameFamily } from './mock/partitioned_field_metadata/mock_partitioned_field_metadata_with_same_family';
describe('helpers', () => {
describe('getTotalPatternSameFamily', () => {
@ -485,7 +482,7 @@ describe('helpers', () => {
test('it returns the happy path result when the index has no mapping conflicts, and no unallowed values', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index
unallowedValues: noUnallowedValues, // no unallowed values for `event.category` in this index
})
@ -501,7 +498,7 @@ describe('helpers', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index
unallowedValues: noEntryForEventCategory, // a lookup in this map for the `event.category` field will return undefined
})
@ -511,7 +508,7 @@ describe('helpers', () => {
test('it returns a result with the expected `indexInvalidValues` and `isEcsCompliant` when the index has no mapping conflict, but it has unallowed values', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: fieldMetadataCorrectMappingType, // no mapping conflicts for `event.category` in this index
unallowedValues, // this index has unallowed values for the event.category field
})
@ -537,7 +534,7 @@ describe('helpers', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field: 'event.category', // `event.category` is a `keyword`, per the ECS spec
type: indexFieldType, // this index has a mapping of `text` instead
@ -558,7 +555,7 @@ describe('helpers', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field: 'event.category', // `event.category` is a `keyword` per the ECS spec
type: indexFieldType, // this index has a mapping of `wildcard` instead
@ -579,7 +576,7 @@ describe('helpers', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field: 'event.category', // `event.category` is a `keyword` per the ECS spec
type: indexFieldType, // this index has a mapping of `text` instead
@ -611,7 +608,7 @@ describe('helpers', () => {
expect(
getEnrichedFieldMetadata({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
fieldMetadata: {
field,
type: indexFieldType, // no mapping conflict, because ECS doesn't define this field
@ -632,14 +629,13 @@ describe('helpers', () => {
describe('getMissingTimestampFieldMetadata', () => {
test('it returns the expected `EnrichedFieldMetadata`', () => {
expect(getMissingTimestampFieldMetadata()).toEqual({
description: TIMESTAMP_DESCRIPTION,
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-', // the index did NOT define a mapping for @timestamp
indexInvalidValues: [],
isEcsCompliant: false, // an index must define the @timestamp mapping
isInSameFamily: false, // `date` is not a member of any families
type: 'date',
});
});
});
@ -717,37 +713,6 @@ describe('helpers', () => {
});
});
describe('hasValidTimestampMapping', () => {
test('it returns true when the `enrichedFieldMetadata` has a valid @timestamp', () => {
const enrichedFieldMetadata: EnrichedFieldMetadata[] = [timestamp, sourcePort];
expect(hasValidTimestampMapping(enrichedFieldMetadata)).toBe(true);
});
test('it returns false when the `enrichedFieldMetadata` collection does NOT include a valid @timestamp', () => {
const enrichedFieldMetadata: EnrichedFieldMetadata[] = [sourcePort];
expect(hasValidTimestampMapping(enrichedFieldMetadata)).toBe(false);
});
test('it returns false when the `enrichedFieldMetadata` has an @timestamp with an invalid mapping', () => {
const timestampWithInvalidMapping: EnrichedFieldMetadata = {
...timestamp,
indexFieldType: 'text', // invalid mapping, should be "date"
};
const enrichedFieldMetadata: EnrichedFieldMetadata[] = [
timestampWithInvalidMapping,
sourcePort,
];
expect(hasValidTimestampMapping(enrichedFieldMetadata)).toBe(false);
});
test('it returns false when `enrichedFieldMetadata` is empty', () => {
expect(hasValidTimestampMapping([])).toBe(false);
});
});
describe('getDocsCount', () => {
test('it returns the expected docs count when `stats` contains the `indexName`', () => {
const indexName = '.ds-packetbeat-8.6.1-2023.02.04-000001';
@ -1429,6 +1394,120 @@ describe('helpers', () => {
});
});
describe('formatStorageResult', () => {
it('should correctly format the input data into a StorageResult object', () => {
const inputData: Parameters<typeof formatStorageResult>[number] = {
result: {
indexName: 'testIndex',
pattern: 'testPattern',
checkedAt: 1627545600000,
docsCount: 100,
incompatible: 3,
sameFamily: 1,
ilmPhase: 'hot',
markdownComments: ['test comments'],
error: null,
},
report: {
batchId: 'testBatch',
isCheckAll: true,
sameFamilyFields: ['agent.type'],
unallowedMappingFields: ['event.category', 'host.name', 'source.ip'],
unallowedValueFields: ['event.category'],
sizeInBytes: 5000,
ecsVersion: '1.0.0',
indexName: 'testIndex',
indexId: 'testIndexId',
},
partitionedFieldMetadata: mockPartitionedFieldMetadataWithSameFamily,
};
const expectedResult: StorageResult = {
batchId: 'testBatch',
indexName: 'testIndex',
indexPattern: 'testPattern',
isCheckAll: true,
checkedAt: 1627545600000,
docsCount: 100,
totalFieldCount: 10,
ecsFieldCount: 2,
customFieldCount: 4,
incompatibleFieldCount: 3,
incompatibleFieldMappingItems: [
{
fieldName: 'event.category',
expectedValue: 'keyword',
actualValue: 'constant_keyword',
description:
'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
},
{
fieldName: 'host.name',
expectedValue: 'keyword',
actualValue: 'text',
description:
'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
},
{
fieldName: 'source.ip',
expectedValue: 'ip',
actualValue: 'text',
description: 'IP address of the source (IPv4 or IPv6).',
},
],
incompatibleFieldValueItems: [
{
fieldName: 'event.category',
expectedValues: [
'authentication',
'configuration',
'database',
'driver',
'email',
'file',
'host',
'iam',
'intrusion_detection',
'malware',
'network',
'package',
'process',
'registry',
'session',
'threat',
'vulnerability',
'web',
],
actualValues: [{ name: 'an_invalid_category', count: 2 }],
description:
'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
},
],
sameFamilyFieldCount: 1,
sameFamilyFields: ['agent.type'],
sameFamilyFieldItems: [
{
fieldName: 'agent.type',
expectedValue: 'keyword',
actualValue: 'constant_keyword',
description:
'Type of the agent.\nThe agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.',
},
],
unallowedMappingFields: ['event.category', 'host.name', 'source.ip'],
unallowedValueFields: ['event.category'],
sizeInBytes: 5000,
ilmPhase: 'hot',
markdownComments: ['test comments'],
ecsVersion: '1.0.0',
indexId: 'testIndexId',
error: null,
};
expect(formatStorageResult(inputData)).toEqual(expectedResult);
});
});
describe('postStorageResult', () => {
const { fetch } = httpServiceMock.createStartContract();
const { toasts } = notificationServiceMock.createStartContract();

105
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts
View file

@ -17,16 +17,20 @@ import * as i18n from './translations';
import type {
DataQualityCheckResult,
DataQualityIndexCheckedParams,
EcsMetadata,
EcsBasedFieldMetadata,
EnrichedFieldMetadata,
ErrorSummary,
IlmPhase,
IncompatibleFieldMappingItem,
IncompatibleFieldValueItem,
MeteringStatsIndex,
PartitionedFieldMetadata,
PartitionedFieldMetadataStats,
PatternRollup,
SameFamilyFieldItem,
UnallowedValueCount,
} from './types';
import { EcsFlatTyped } from './constants';
const EMPTY_INDEX_NAMES: string[] = [];
export const INTERNAL_API_VERSION = '1';
@ -172,7 +176,7 @@ export const getEnrichedFieldMetadata = ({
fieldMetadata,
unallowedValues,
}: {
ecsMetadata: Record<string, EcsMetadata>;
ecsMetadata: EcsFlatTyped;
fieldMetadata: FieldType;
unallowedValues: Record<string, UnallowedValueCount[]>;
}): EnrichedFieldMetadata => {
@ -202,7 +206,7 @@ export const getEnrichedFieldMetadata = ({
return {
indexFieldName: field,
indexFieldType: type,
indexInvalidValues,
indexInvalidValues: [],
hasEcsMetadata: false,
isEcsCompliant: false,
isInSameFamily: false, // custom fields are never in the same family
@ -210,15 +214,14 @@ export const getEnrichedFieldMetadata = ({
}
};
export const getMissingTimestampFieldMetadata = (): EnrichedFieldMetadata => ({
description: i18n.TIMESTAMP_DESCRIPTION,
export const getMissingTimestampFieldMetadata = (): EcsBasedFieldMetadata => ({
...EcsFlatTyped['@timestamp'],
hasEcsMetadata: true,
indexFieldName: '@timestamp',
indexFieldType: '-',
indexInvalidValues: [],
isEcsCompliant: false,
isInSameFamily: false, // `date` is not a member of any families
type: 'date',
});
export const getPartitionedFieldMetadata = (
@ -258,11 +261,6 @@ export const getPartitionedFieldMetadataStats = (
};
};
export const hasValidTimestampMapping = (enrichedFieldMetadata: EnrichedFieldMetadata[]): boolean =>
enrichedFieldMetadata.some(
(x) => x.indexFieldName === '@timestamp' && x.indexFieldType === 'date'
);
export const getDocsCount = ({
indexName,
stats,
@ -471,8 +469,11 @@ export interface StorageResult {
ecsFieldCount: number;
customFieldCount: number;
incompatibleFieldCount: number;
incompatibleFieldMappingItems: IncompatibleFieldMappingItem[];
incompatibleFieldValueItems: IncompatibleFieldValueItem[];
sameFamilyFieldCount: number;
sameFamilyFields: string[];
sameFamilyFieldItems: SameFamilyFieldItem[];
unallowedMappingFields: string[];
unallowedValueFields: string[];
sizeInBytes: number;
@ -491,28 +492,66 @@ export const formatStorageResult = ({
result: DataQualityCheckResult;
report: DataQualityIndexCheckedParams;
partitionedFieldMetadata: PartitionedFieldMetadata;
}): StorageResult => ({
batchId: report.batchId,
indexName: result.indexName,
indexPattern: result.pattern,
isCheckAll: report.isCheckAll,
checkedAt: result.checkedAt ?? Date.now(),
docsCount: result.docsCount ?? 0,
totalFieldCount: partitionedFieldMetadata.all.length,
ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
customFieldCount: partitionedFieldMetadata.custom.length,
incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
sameFamilyFields: report.sameFamilyFields ?? [],
unallowedMappingFields: report.unallowedMappingFields ?? [],
unallowedValueFields: report.unallowedValueFields ?? [],
sizeInBytes: report.sizeInBytes ?? 0,
ilmPhase: result.ilmPhase,
markdownComments: result.markdownComments,
ecsVersion: report.ecsVersion,
indexId: report.indexId ?? '', // ---> we don't have this field when isILMAvailable is false
error: result.error,
});
}): StorageResult => {
const incompatibleFieldMappingItems: IncompatibleFieldMappingItem[] = [];
const incompatibleFieldValueItems: IncompatibleFieldValueItem[] = [];
const sameFamilyFieldItems: SameFamilyFieldItem[] = [];
partitionedFieldMetadata.incompatible.forEach((field) => {
if (field.type !== field.indexFieldType) {
incompatibleFieldMappingItems.push({
fieldName: field.indexFieldName,
expectedValue: field.type,
actualValue: field.indexFieldType,
description: field.description,
});
}
if (field.indexInvalidValues.length > 0) {
incompatibleFieldValueItems.push({
fieldName: field.indexFieldName,
expectedValues: field.allowed_values?.map((x) => x.name) ?? [],
actualValues: field.indexInvalidValues.map((v) => ({ name: v.fieldName, count: v.count })),
description: field.description,
});
}
});
partitionedFieldMetadata.sameFamily.forEach((field) => {
sameFamilyFieldItems.push({
fieldName: field.indexFieldName,
expectedValue: field.type,
actualValue: field.indexFieldType,
description: field.description,
});
});
return {
batchId: report.batchId,
indexName: result.indexName,
indexPattern: result.pattern,
isCheckAll: report.isCheckAll,
checkedAt: result.checkedAt ?? Date.now(),
docsCount: result.docsCount ?? 0,
totalFieldCount: partitionedFieldMetadata.all.length,
ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
customFieldCount: partitionedFieldMetadata.custom.length,
incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
incompatibleFieldMappingItems,
incompatibleFieldValueItems,
sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
sameFamilyFields: report.sameFamilyFields ?? [],
sameFamilyFieldItems,
unallowedMappingFields: report.unallowedMappingFields ?? [],
unallowedValueFields: report.unallowedValueFields ?? [],
sizeInBytes: report.sizeInBytes ?? 0,
ilmPhase: result.ilmPhase,
markdownComments: result.markdownComments,
ecsVersion: report.ecsVersion,
indexId: report.indexId ?? '',
error: result.error,
};
};
export const formatResultFromStorage = ({
storageResult,

26
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/enriched_field_metadata/mock_enriched_field_metadata.ts
View file

@ -5,9 +5,9 @@
* 2.0.
*/
import { EnrichedFieldMetadata } from '../../types';
import { CustomFieldMetadata, EcsBasedFieldMetadata } from '../../types';
export const timestamp: EnrichedFieldMetadata = {
export const timestamp: EcsBasedFieldMetadata = {
dashed_name: 'timestamp',
description:
'Date/time when the event originated.\nThis is the date/time extracted from the event, typically representing when the event was generated by the source.\nIf the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline.\nRequired field for all events.',
@ -27,7 +27,7 @@ export const timestamp: EnrichedFieldMetadata = {
isInSameFamily: false, // `date` is not a member of any families
};
export const eventCategory: EnrichedFieldMetadata = {
export const eventCategory: EcsBasedFieldMetadata = {
allowed_values: [
{
description:
@ -166,7 +166,7 @@ export const eventCategory: EnrichedFieldMetadata = {
isInSameFamily: false,
};
export const eventCategoryWithUnallowedValues: EnrichedFieldMetadata = {
export const eventCategoryWithUnallowedValues: EcsBasedFieldMetadata = {
...eventCategory,
indexInvalidValues: [
{
@ -181,7 +181,7 @@ export const eventCategoryWithUnallowedValues: EnrichedFieldMetadata = {
isEcsCompliant: false, // because this index has unallowed values
};
export const hostNameWithTextMapping: EnrichedFieldMetadata = {
export const hostNameWithTextMapping: EcsBasedFieldMetadata = {
dashed_name: 'host-name',
description:
'Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.',
@ -200,7 +200,7 @@ export const hostNameWithTextMapping: EnrichedFieldMetadata = {
isInSameFamily: false, // `keyword` and `text` are not in the family
};
export const hostNameKeyword: EnrichedFieldMetadata = {
export const hostNameKeyword: CustomFieldMetadata = {
indexFieldName: 'host.name.keyword',
indexFieldType: 'keyword',
indexInvalidValues: [],
@ -209,7 +209,7 @@ export const hostNameKeyword: EnrichedFieldMetadata = {
isInSameFamily: false, // custom fields are never in the same family
};
export const someField: EnrichedFieldMetadata = {
export const someField: CustomFieldMetadata = {
indexFieldName: 'some.field',
indexFieldType: 'text',
indexInvalidValues: [],
@ -218,7 +218,7 @@ export const someField: EnrichedFieldMetadata = {
isInSameFamily: false, // custom fields are never in the same family
};
export const someFieldKeyword: EnrichedFieldMetadata = {
export const someFieldKeyword: CustomFieldMetadata = {
indexFieldName: 'some.field.keyword',
indexFieldType: 'keyword',
indexInvalidValues: [],
@ -227,7 +227,7 @@ export const someFieldKeyword: EnrichedFieldMetadata = {
isInSameFamily: false, // custom fields are never in the same family
};
export const sourceIpWithTextMapping: EnrichedFieldMetadata = {
export const sourceIpWithTextMapping: EcsBasedFieldMetadata = {
dashed_name: 'source-ip',
description: 'IP address of the source (IPv4 or IPv6).',
flat_name: 'source.ip',
@ -244,7 +244,7 @@ export const sourceIpWithTextMapping: EnrichedFieldMetadata = {
isInSameFamily: false, // `ip` is not a member of any families
};
export const sourceIpKeyword: EnrichedFieldMetadata = {
export const sourceIpKeyword: CustomFieldMetadata = {
indexFieldName: 'source.ip.keyword',
indexFieldType: 'keyword',
indexInvalidValues: [],
@ -253,7 +253,7 @@ export const sourceIpKeyword: EnrichedFieldMetadata = {
isInSameFamily: false, // custom fields are never in the same family
};
export const sourcePort: EnrichedFieldMetadata = {
export const sourcePort: EcsBasedFieldMetadata = {
dashed_name: 'source-port',
description: 'Port of the source.',
flat_name: 'source.port',
@ -271,7 +271,7 @@ export const sourcePort: EnrichedFieldMetadata = {
isInSameFamily: false, // `long` is not a member of any families
};
export const mockCustomFields: EnrichedFieldMetadata[] = [
export const mockCustomFields: CustomFieldMetadata[] = [
{
indexFieldName: 'host.name.keyword',
indexFieldType: 'keyword',
@ -306,7 +306,7 @@ export const mockCustomFields: EnrichedFieldMetadata[] = [
},
];
export const mockIncompatibleMappings: EnrichedFieldMetadata[] = [
export const mockIncompatibleMappings: EcsBasedFieldMetadata[] = [
{
dashed_name: 'host-name',
description:

100
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts
View file

@ -16,43 +16,73 @@ export interface Mappings {
indexes: Record<string, IndicesGetMappingIndexMappingRecord>;
}
export interface AllowedValue {
description?: string;
expected_event_types?: string[];
name?: string;
}
export interface EcsFieldMetadata {
dashed_name: string;
description: string;
flat_name: string;
level: string;
name: string;
normalize: string[];
short: string;
type: string;
export interface EcsMetadata {
allowed_values?: AllowedValue[];
dashed_name?: string;
description?: string;
example?: string;
flat_name?: string;
beta?: string;
doc_values?: boolean;
example?: string | number | boolean;
expected_values?: string[];
format?: string;
ignore_above?: number;
level?: string;
name?: string;
normalize?: string[];
index?: boolean;
input_format?: string;
multi_fields?: MultiField[];
object_type?: string;
original_fieldset?: string;
output_format?: string;
output_precision?: number;
pattern?: string;
required?: boolean;
short?: string;
type?: string;
scaling_factor?: number;
}
export type EnrichedFieldMetadata = EcsMetadata & {
hasEcsMetadata: boolean;
export interface AllowedValue {
description: string;
name: string;
expected_event_types?: string[];
beta?: string;
}
export interface MultiField {
flat_name: string;
name: string;
type: string;
}
export interface CustomFieldMetadata {
hasEcsMetadata: false;
indexFieldName: string;
indexFieldType: string;
indexInvalidValues: [];
isEcsCompliant: false;
isInSameFamily: false;
}
export interface EcsBasedFieldMetadata extends EcsFieldMetadata {
hasEcsMetadata: true;
indexFieldName: string;
indexFieldType: string;
indexInvalidValues: UnallowedValueCount[];
isEcsCompliant: boolean;
isInSameFamily: boolean;
};
}
export type EnrichedFieldMetadata = EcsBasedFieldMetadata | CustomFieldMetadata;
export interface PartitionedFieldMetadata {
all: EnrichedFieldMetadata[];
custom: EnrichedFieldMetadata[];
ecsCompliant: EnrichedFieldMetadata[];
incompatible: EnrichedFieldMetadata[];
sameFamily: EnrichedFieldMetadata[];
custom: CustomFieldMetadata[];
ecsCompliant: EcsBasedFieldMetadata[];
incompatible: EcsBasedFieldMetadata[];
sameFamily: EcsBasedFieldMetadata[];
}
export interface PartitionedFieldMetadataStats {
@ -89,6 +119,32 @@ export interface UnallowedValueSearchResult {
export type IlmPhase = 'hot' | 'warm' | 'cold' | 'frozen' | 'unmanaged';
export interface IncompatibleFieldMappingItem {
fieldName: string;
expectedValue: string;
actualValue: string;
description: string;
}
export interface ActualIncompatibleValue {
name: string;
count: number;
}
export interface IncompatibleFieldValueItem {
fieldName: string;
expectedValues: string[];
actualValues: ActualIncompatibleValue[];
description: string;
}
export interface SameFamilyFieldItem {
fieldName: string;
expectedValue: string;
actualValue: string;
description: string;
}
export interface IlmExplainPhaseCounts {
hot: number;
warm: number;

7
x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx
View file

@ -5,7 +5,6 @@
* 2.0.
*/
import { EcsFlat } from '@elastic/ecs';
import { renderHook } from '@testing-library/react-hooks';
import React, { FC, PropsWithChildren } from 'react';
@ -13,9 +12,10 @@ import { getUnallowedValueRequestItems } from '../data_quality_panel/allowed_val
import { DataQualityProvider } from '../data_quality_panel/data_quality_context';
import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unallowed_values';
import { ERROR_LOADING_UNALLOWED_VALUES } from '../translations';
import { EcsMetadata, UnallowedValueRequestItem } from '../types';
import { UnallowedValueRequestItem } from '../types';
import { useUnallowedValues, UseUnallowedValues } from '.';
import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks';
import { EcsFlatTyped } from '../constants';
const mockHttpFetch = jest.fn();
const mockReportDataQualityIndexChecked = jest.fn();
@ -37,10 +37,9 @@ const ContextWrapper: FC<PropsWithChildren<unknown>> = ({ children }) => (
</DataQualityProvider>
);
const ecsMetadata = EcsFlat as unknown as Record<string, EcsMetadata>;
const indexName = 'auditbeat-custom-index-1';
const requestItems = getUnallowedValueRequestItems({
ecsMetadata,
ecsMetadata: EcsFlatTyped,
indexName,
});

18
x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts
View file

@ -18,12 +18,18 @@ export const resultsFieldMap: FieldMap = {
totalFieldCount: { type: 'long', required: true },
ecsFieldCount: { type: 'long', required: true },
customFieldCount: { type: 'long', required: true },
incompatibleFieldItems: { type: 'nested', required: true, array: true },
'incompatibleFieldItems.fieldName': { type: 'keyword', required: true },
'incompatibleFieldItems.expectedValue': { type: 'keyword', required: true },
'incompatibleFieldItems.actualValue': { type: 'keyword', required: true },
'incompatibleFieldItems.description': { type: 'keyword', required: true },
'incompatibleFieldItems.reason': { type: 'keyword', required: true },
incompatibleFieldMappingItems: { type: 'nested', required: true, array: true },
'incompatibleFieldMappingItems.fieldName': { type: 'keyword', required: true },
'incompatibleFieldMappingItems.expectedValue': { type: 'keyword', required: true },
'incompatibleFieldMappingItems.actualValue': { type: 'keyword', required: true },
'incompatibleFieldMappingItems.description': { type: 'keyword', required: true },
incompatibleFieldValueItems: { type: 'nested', required: true, array: true },
'incompatibleFieldValueItems.fieldName': { type: 'keyword', required: true },
'incompatibleFieldValueItems.expectedValues': { type: 'keyword', required: true },
'incompatibleFieldValueItems.actualValues': { type: 'nested', required: true, array: true },
'incompatibleFieldValueItems.actualValues.name': { type: 'keyword', required: true },
'incompatibleFieldValueItems.actualValues.count': { type: 'keyword', required: true },
'incompatibleFieldValueItems.description': { type: 'keyword', required: true },
incompatibleFieldCount: { type: 'long', required: true },
sameFamilyFieldCount: { type: 'long', required: true },
sameFamilyFieldItems: { type: 'nested', required: true, array: true },

30
x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts
View file

@ -19,8 +19,38 @@ export const resultDocument: ResultDocument = {
ecsFieldCount: 677,
customFieldCount: 904,
incompatibleFieldCount: 1,
incompatibleFieldMappingItems: [],
incompatibleFieldValueItems: [
{
fieldName: 'event.category',
expectedValues: [
`authentication`,
`configuration`,
`database`,
`driver`,
`email`,
`file`,
`host`,
`iam`,
`intrusion_detection`,
`malware`,
`network`,
`package`,
`process`,
`registry`,
`session`,
`threat`,
'vulnerability',
'web',
],
actualValues: [{ name: 'behavior', count: 6 }],
description:
'This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy.\n`event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory.\nThis field is an array. This will allow proper categorization of some events that fall in multiple categories.',
},
],
sameFamilyFieldCount: 0,
sameFamilyFields: [],
sameFamilyFieldItems: [],
unallowedMappingFields: [],
unallowedValueFields: ['event.category'],
sizeInBytes: 173796,

24
x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts
View file

@ -19,8 +19,32 @@ const ResultDocumentInterface = t.interface({
ecsFieldCount: t.number,
customFieldCount: t.number,
incompatibleFieldCount: t.number,
incompatibleFieldMappingItems: t.array(
t.type({
fieldName: t.string,
expectedValue: t.string,
actualValue: t.string,
description: t.string,
})
),
incompatibleFieldValueItems: t.array(
t.type({
fieldName: t.string,
expectedValues: t.array(t.string),
actualValues: t.array(t.type({ name: t.string, count: t.number })),
description: t.string,
})
),
sameFamilyFieldCount: t.number,
sameFamilyFields: t.array(t.string),
sameFamilyFieldItems: t.array(
t.type({
fieldName: t.string,
expectedValue: t.string,
actualValue: t.string,
description: t.string,
})
),
unallowedMappingFields: t.array(t.string),
unallowedValueFields: t.array(t.string),
sizeInBytes: t.number,