[Dataset Quality] Create the basic degraded fields flyout (#191597)

## Summary

Closes - https://github.com/elastic/kibana/issues/190328

Delivered as part of this PR

- [x] Added a new Degraded Field Flyout with a basic List of data point
for the degraded Field
- [x] A new endpoint to display possible values. This endpoint will
query to get the latest values, maximum 4
- [x] URL supports Flyout state
- [x] API Tests for the new endpoint
- [x] E2E tests for the flyout


## Screenshot

<img width="1903" alt="image"
src="https://github.com/user-attachments/assets/9bc20d15-d52b-4d1e-827f-ab1444e27128">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Achyut Jhunjhunwala 2024-09-03 19:25:09 +02:00 committed by GitHub
parent 62a163cc95
commit 0be5efd71b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 1047 additions and 47 deletions

View file

@ -40,4 +40,5 @@ export interface DataQualityDetailsLocatorParams extends SerializableRecord {
degradedFields?: {
table?: DegradedFieldsTable;
};
expandedDegradedField?: string;
}

View file

@ -66,6 +66,7 @@ export type LogDocument = Fields &
'event.duration': number;
'event.start': Date;
'event.end': Date;
test_field: string | string[];
date: Date;
severity: string;
msg: string;

View file

@ -45,9 +45,9 @@ export const degradedFieldRT = rt.exact(
export const dataStreamRT = new rt.Type<string, string, unknown>(
'dataStreamRT',
(input: unknown): input is string =>
typeof input === 'string' && (input.match(/-/g) || []).length === 2,
typeof input === 'string' && (input.match(/-/g) || []).length >= 2,
(input, context) =>
typeof input === 'string' && (input.match(/-/g) || []).length === 2
typeof input === 'string' && (input.match(/-/g) || []).length >= 2
? rt.success(input)
: rt.failure(input, context),
rt.identity

View file

@ -18,6 +18,7 @@ export const urlSchemaRT = rt.exact(
timeRange: timeRangeRT,
breakdownField: rt.string,
degradedFields: degradedFieldRT,
expandedDegradedField: rt.string,
}),
])
);

View file

@ -18,6 +18,7 @@ export const getStateFromUrlValue = (
timeRange: urlValue.timeRange,
degradedFields: urlValue.degradedFields,
breakdownField: urlValue.breakdownField,
expandedDegradedField: urlValue.expandedDegradedField,
});
export const getUrlValueFromState = (
@ -28,6 +29,7 @@ export const getUrlValueFromState = (
timeRange: state.timeRange,
degradedFields: state.degradedFields,
breakdownField: state.breakdownField,
expandedDegradedField: state.expandedDegradedField,
v: 1,
});

View file

@ -113,6 +113,13 @@ export const getDataStreamDegradedFieldsResponseRt = rt.type({
export type DegradedFieldResponse = rt.TypeOf<typeof getDataStreamDegradedFieldsResponseRt>;
export const degradedFieldValuesRt = rt.type({
field: rt.string,
values: rt.array(rt.string),
});
export type DegradedFieldValues = rt.TypeOf<typeof degradedFieldValuesRt>;
export const dataStreamSettingsRt = rt.partial({
createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
integration: rt.string,

View file

@ -34,6 +34,13 @@ export type GetDataStreamDegradedFieldsQueryParams =
export type GetDataStreamDegradedFieldsParams = GetDataStreamDegradedFieldsPathParams &
GetDataStreamDegradedFieldsQueryParams;
/*
Types for Degraded Field Values inside a DataStream
*/
export type GetDataStreamDegradedFieldValuesPathParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/degraded_field/{degradedField}/values`>['params']['path'];
/*
Types for DataStream Settings
*/

View file

@ -374,3 +374,31 @@ export const integrationVersionText = i18n.translate(
defaultMessage: 'Version',
}
);
export const fieldColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.field', {
defaultMessage: 'Field',
});
export const countColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.count', {
defaultMessage: 'Docs count',
});
export const lastOccurrenceColumnName = i18n.translate(
'xpack.datasetQuality.details.degradedField.lastOccurrence',
{
defaultMessage: 'Last occurrence',
}
);
export const degradedFieldValuesColumnName = i18n.translate(
'xpack.datasetQuality.details.degradedField.values',
{
defaultMessage: 'Values',
}
);
export const fieldIgnoredText = i18n.translate(
'xpack.datasetQuality.details.degradedField.fieldIgnored',
{
defaultMessage: 'field ignored',
}
);

View file

@ -16,7 +16,7 @@ import {
} from '@elastic/eui';
import { ScaleType, Settings, Tooltip, Chart, BarSeries } from '@elastic/charts';
import { i18n } from '@kbn/i18n';
import { Coordinate } from '../../../../../common/types';
import { Coordinate } from '../../../common/types';
export function SparkPlot({
valueLabel,

View file

@ -6,16 +6,20 @@
*/
import React, { useEffect } from 'react';
import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule } from '@elastic/eui';
import { dynamic } from '@kbn/shared-ux-utility';
import { useDatasetDetailsTelemetry, useDatasetQualityDetailsState } from '../../hooks';
import { DataStreamNotFoundPrompt } from './index_not_found_prompt';
import { Header } from './header';
import { Overview } from './overview';
import { Details } from './details';
const DegradedFieldFlyout = dynamic(() => import('./degraded_field_flyout'));
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function DatasetQualityDetails() {
const { isIndexNotFoundError, dataStream } = useDatasetQualityDetailsState();
const { isIndexNotFoundError, dataStream, expandedDegradedField } =
useDatasetQualityDetailsState();
const { startTracking } = useDatasetDetailsTelemetry();
useEffect(() => {
@ -24,14 +28,17 @@ export default function DatasetQualityDetails() {
return isIndexNotFoundError ? (
<DataStreamNotFoundPrompt dataStream={dataStream} />
) : (
<EuiFlexGroup direction="column" gutterSize="l" data-test-subj="datasetDetailsContainer">
<EuiFlexItem grow={false}>
<Header />
<EuiHorizontalRule />
<Overview />
<EuiHorizontalRule />
<Details />
</EuiFlexItem>
</EuiFlexGroup>
<>
<EuiFlexGroup direction="column" gutterSize="l" data-test-subj="datasetDetailsContainer">
<EuiFlexItem grow={false}>
<Header />
<EuiHorizontalRule />
<Overview />
<EuiHorizontalRule />
<Details />
</EuiFlexItem>
</EuiFlexGroup>
{expandedDegradedField && <DegradedFieldFlyout />}
</>
);
}

View file

@ -0,0 +1,108 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { useMemo } from 'react';
import {
EuiBadge,
EuiBadgeGroup,
EuiFlexGroup,
EuiFlexItem,
EuiHorizontalRule,
EuiSkeletonRectangle,
EuiTextColor,
EuiTitle,
formatNumber,
} from '@elastic/eui';
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
import { NUMBER_FORMAT } from '../../../../common/constants';
import {
countColumnName,
degradedFieldValuesColumnName,
lastOccurrenceColumnName,
} from '../../../../common/translations';
import { useDegradedFields } from '../../../hooks';
import { SparkPlot } from '../../common/spark_plot';
export const DegradedFieldInfo = () => {
const {
renderedItems,
fieldFormats,
expandedDegradedField,
degradedFieldValues,
isDegradedFieldsLoading,
isDegradedFieldsValueLoading,
} = useDegradedFields();
const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
ES_FIELD_TYPES.DATE,
]);
const fieldList = useMemo(() => {
return renderedItems.find((item) => {
return item.name === expandedDegradedField;
});
}, [renderedItems, expandedDegradedField]);
return (
<EuiFlexGroup direction="column" gutterSize="none">
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount`}>
<EuiFlexItem>
<EuiTitle size="xxs">
<span>{countColumnName}</span>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-docCount">
<SparkPlot
series={fieldList?.timeSeries}
valueLabel={formatNumber(fieldList?.count, NUMBER_FORMAT)}
isLoading={isDegradedFieldsLoading}
/>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule margin="s" />
<EuiFlexGroup
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-lastOccurrence`}
>
<EuiFlexItem>
<EuiTitle size="xxs">
<span>{lastOccurrenceColumnName}</span>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-lastOccurrence">
<span>{dateFormatter.convert(fieldList?.lastOccurrence)}</span>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule margin="s" />
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-values`}>
<EuiFlexItem>
<EuiTitle size="xxs">
<span>{degradedFieldValuesColumnName}</span>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-values"
grow={false}
css={{ maxWidth: '49%' }}
>
<EuiSkeletonRectangle isLoading={isDegradedFieldsValueLoading} width="300px">
<EuiBadgeGroup gutterSize="s">
{degradedFieldValues?.values.map((value) => (
<EuiBadge color="hollow">
<EuiTextColor color="#765B96">
<strong>{value}</strong>
</EuiTextColor>
</EuiBadge>
))}
</EuiBadgeGroup>
</EuiSkeletonRectangle>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule margin="s" />
</EuiFlexGroup>
);
};

View file

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import {
EuiBadge,
EuiFlyout,
EuiFlyoutHeader,
EuiFlyoutBody,
EuiSpacer,
EuiText,
EuiTitle,
useGeneratedHtmlId,
} from '@elastic/eui';
import { useDegradedFields } from '../../../hooks';
import {
fieldIgnoredText,
overviewDegradedFieldsSectionTitle,
} from '../../../../common/translations';
import { DegradedFieldInfo } from './field_info';
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function DegradedFieldFlyout() {
const { closeDegradedFieldFlyout, expandedDegradedField } = useDegradedFields();
const pushedFlyoutTitleId = useGeneratedHtmlId({
prefix: 'pushedFlyoutTitle',
});
return (
<EuiFlyout
type="push"
size="s"
onClose={closeDegradedFieldFlyout}
aria-labelledby={pushedFlyoutTitleId}
data-test-subj={'datasetQualityDetailsDegradedFieldFlyout'}
>
<EuiFlyoutHeader hasBorder>
<EuiBadge color="warning">{overviewDegradedFieldsSectionTitle}</EuiBadge>
<EuiSpacer size="s" />
<EuiTitle size="m">
<EuiText>
{expandedDegradedField} <span style={{ fontWeight: 400 }}>{fieldIgnoredText}</span>
</EuiText>
</EuiTitle>
</EuiFlyoutHeader>
<EuiFlyoutBody>
<DegradedFieldInfo />
</EuiFlyoutBody>
</EuiFlyout>
);
}

View file

@ -6,37 +6,74 @@
*/
import React from 'react';
import { css } from '@emotion/react';
import { i18n } from '@kbn/i18n';
import type { EuiBasicTableColumn } from '@elastic/eui';
import { EuiBasicTableColumn, EuiButtonIcon } from '@elastic/eui';
import type { FieldFormat } from '@kbn/field-formats-plugin/common';
import { formatNumber } from '@elastic/eui';
import { DegradedField } from '../../../../../common/api_types';
import { SparkPlot } from './spark_plot';
import { SparkPlot } from '../../../common/spark_plot';
import { NUMBER_FORMAT } from '../../../../../common/constants';
import {
countColumnName,
fieldColumnName,
lastOccurrenceColumnName,
} from '../../../../../common/translations';
const fieldColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.field', {
defaultMessage: 'Field',
});
const countColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.count', {
defaultMessage: 'Docs count',
});
const lastOccurrenceColumnName = i18n.translate(
'xpack.datasetQuality.details.degradedField.lastOccurrence',
const expandDatasetAriaLabel = i18n.translate(
'xpack.datasetQuality.details.degradedFieldTable.expandLabel',
{
defaultMessage: 'Last occurrence',
defaultMessage: 'Expand',
}
);
const collapseDatasetAriaLabel = i18n.translate(
'xpack.datasetQuality.details.degradedFieldTable.collapseLabel',
{
defaultMessage: 'Collapse',
}
);
export const getDegradedFieldsColumns = ({
dateFormatter,
isLoading,
expandedDegradedField,
openDegradedFieldFlyout,
}: {
dateFormatter: FieldFormat;
isLoading: boolean;
expandedDegradedField?: string;
openDegradedFieldFlyout: (name: string) => void;
}): Array<EuiBasicTableColumn<DegradedField>> => [
{
name: '',
field: 'name',
render: (_, { name }) => {
const isExpanded = name === expandedDegradedField;
const onExpandClick = () => {
openDegradedFieldFlyout(name);
};
return (
<EuiButtonIcon
data-test-subj="datasetQualityDetailsDegradedFieldsExpandButton"
size="xs"
color="text"
onClick={onExpandClick}
iconType={isExpanded ? 'minimize' : 'expand'}
title={!isExpanded ? expandDatasetAriaLabel : collapseDatasetAriaLabel}
aria-label={!isExpanded ? expandDatasetAriaLabel : collapseDatasetAriaLabel}
/>
);
},
width: '40px',
css: css`
&.euiTableCellContent {
padding: 0;
}
`,
},
{
name: fieldColumnName,
field: 'name',

View file

@ -16,19 +16,32 @@ import {
import { useDegradedFields } from '../../../../hooks/use_degraded_fields';
export const DegradedFieldTable = () => {
const { isLoading, pagination, renderedItems, onTableChange, sort, fieldFormats } =
useDegradedFields();
const {
isDegradedFieldsLoading,
pagination,
renderedItems,
onTableChange,
sort,
fieldFormats,
expandedDegradedField,
openDegradedFieldFlyout,
} = useDegradedFields();
const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
ES_FIELD_TYPES.DATE,
]);
const columns = getDegradedFieldsColumns({ dateFormatter, isLoading });
const columns = getDegradedFieldsColumns({
dateFormatter,
isLoading: isDegradedFieldsLoading,
expandedDegradedField,
openDegradedFieldFlyout,
});
return (
<EuiBasicTable
tableLayout="fixed"
columns={columns}
items={renderedItems ?? []}
loading={isLoading}
loading={isDegradedFieldsLoading}
sorting={sort}
onChange={onTableChange}
pagination={pagination}
@ -37,7 +50,7 @@ export const DegradedFieldTable = () => {
'data-test-subj': 'datasetQualityDetailsDegradedTableRow',
}}
noItemsMessage={
isLoading ? (
isDegradedFieldsLoading ? (
overviewDegradedFieldsTableLoadingText
) : (
<EuiEmptyPrompt

View file

@ -21,6 +21,7 @@ export const getPublicStateFromContext = (
timeRange: context.timeRange,
breakdownField: context.breakdownField,
integration: context.integration,
expandedDegradedField: context.expandedDegradedField,
};
};
@ -49,4 +50,5 @@ export const getContextFromPublicState = (
},
},
dataStream: publicState.dataStream,
expandedDegradedField: publicState.expandedDegradedField,
});

View file

@ -28,7 +28,7 @@ export type DatasetQualityDetailsPublicState = WithDefaultControllerState;
// a must and everything else can be optional. The table inside the
// degradedFields must accept field property as string
export type DatasetQualityDetailsPublicStateUpdate = Partial<
Pick<WithDefaultControllerState, 'timeRange' | 'breakdownField'>
Pick<WithDefaultControllerState, 'timeRange' | 'breakdownField' | 'expandedDegradedField'>
> & {
dataStream: string;
} & {

View file

@ -21,8 +21,14 @@ export const useDatasetQualityDetailsState = () => {
services: { fieldFormats },
} = useKibanaContextForPlugin();
const { dataStream, degradedFields, timeRange, breakdownField, isIndexNotFoundError } =
useSelector(service, (state) => state.context) ?? {};
const {
dataStream,
degradedFields,
timeRange,
breakdownField,
isIndexNotFoundError,
expandedDegradedField,
} = useSelector(service, (state) => state.context) ?? {};
const isNonAggregatable = useSelector(service, (state) =>
state.matches('initializing.nonAggregatableDataset.done')
@ -143,5 +149,6 @@ export const useDatasetQualityDetailsState = () => {
integrationDetails,
canUserAccessDashboards,
canUserViewIntegrations,
expandedDegradedField,
};
};

View file

@ -24,8 +24,8 @@ export function useDegradedFields() {
services: { fieldFormats },
} = useKibanaContextForPlugin();
const degradedFields = useSelector(service, (state) => state.context.degradedFields) ?? {};
const { data, table } = degradedFields;
const { degradedFields, expandedDegradedField } = useSelector(service, (state) => state.context);
const { data, table } = degradedFields ?? {};
const { page, rowsPerPage, sort } = table;
const totalItemCount = data?.length ?? 0;
@ -62,17 +62,48 @@ export function useDegradedFields() {
return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage);
}, [data, sort.field, sort.direction, page, rowsPerPage]);
const isLoading = useSelector(service, (state) =>
const isDegradedFieldsLoading = useSelector(service, (state) =>
state.matches('initializing.dataStreamDegradedFields.fetching')
);
const closeDegradedFieldFlyout = useCallback(
() => service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' }),
[service]
);
const openDegradedFieldFlyout = useCallback(
(fieldName: string) => {
if (expandedDegradedField === fieldName) {
service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' });
} else {
service.send({ type: 'OPEN_DEGRADED_FIELD_FLYOUT', fieldName });
}
},
[expandedDegradedField, service]
);
const degradedFieldValues = useSelector(service, (state) =>
state.matches('initializing.initializeFixItFlow.ignoredValues.done')
? state.context.degradedFieldValues
: undefined
);
const isDegradedFieldsValueLoading = useSelector(service, (state) => {
return !state.matches('initializing.initializeFixItFlow.ignoredValues.done');
});
return {
isLoading,
isDegradedFieldsLoading,
pagination,
onTableChange,
renderedItems,
sort: { sort },
fieldFormats,
totalItemCount,
expandedDegradedField,
openDegradedFieldFlyout,
closeDegradedFieldFlyout,
degradedFieldValues,
isDegradedFieldsValueLoading,
};
}

View file

@ -8,6 +8,8 @@
import { HttpStart } from '@kbn/core/public';
import { decodeOrThrow } from '@kbn/io-ts-utils';
import {
DegradedFieldValues,
degradedFieldValuesRt,
getDataStreamDegradedFieldsResponseRt,
getDataStreamsDetailsResponseRt,
getDataStreamsSettingsResponseRt,
@ -21,6 +23,7 @@ import {
DataStreamSettings,
DegradedFieldResponse,
GetDataStreamDegradedFieldsParams,
GetDataStreamDegradedFieldValuesPathParams,
GetDataStreamDetailsParams,
GetDataStreamDetailsResponse,
GetDataStreamSettingsParams,
@ -102,6 +105,30 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient {
)(response);
}
public async getDataStreamDegradedFieldValues({
dataStream,
degradedField,
}: GetDataStreamDegradedFieldValuesPathParams): Promise<DegradedFieldValues> {
const response = await this.http
.get<DegradedFieldValues>(
`/internal/dataset_quality/data_streams/${dataStream}/degraded_field/${degradedField}/values`
)
.catch((error) => {
throw new DatasetQualityError(
`Failed to fetch data stream degraded field Value": ${error}`,
error
);
});
return decodeOrThrow(
degradedFieldValuesRt,
(message: string) =>
new DatasetQualityError(
`Failed to decode data stream degraded field values response: ${message}"`
)
)(response);
}
public async getIntegrationDashboards({ integration }: GetIntegrationDashboardsParams) {
const response = await this.http
.get<IntegrationDashboardsResponse>(

View file

@ -15,9 +15,10 @@ import {
GetIntegrationDashboardsParams,
GetDataStreamDegradedFieldsParams,
DegradedFieldResponse,
GetDataStreamDegradedFieldValuesPathParams,
} from '../../../common/data_streams_stats';
import { GetDataStreamIntegrationParams } from '../../../common/data_stream_details/types';
import { Dashboard } from '../../../common/api_types';
import { Dashboard, DegradedFieldValues } from '../../../common/api_types';
export type DataStreamDetailsServiceSetup = void;
@ -35,6 +36,9 @@ export interface IDataStreamDetailsClient {
getDataStreamDegradedFields(
params: GetDataStreamDegradedFieldsParams
): Promise<DegradedFieldResponse>;
getDataStreamDegradedFieldValues(
params: GetDataStreamDegradedFieldValuesPathParams
): Promise<DegradedFieldValues>;
getIntegrationDashboards(params: GetIntegrationDashboardsParams): Promise<Dashboard[]>;
getDataStreamIntegration(
params: GetDataStreamIntegrationParams

View file

@ -22,6 +22,7 @@ import {
DataStreamDetails,
DataStreamSettings,
DegradedFieldResponse,
DegradedFieldValues,
NonAggregatableDatasets,
} from '../../../common/api_types';
import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications';
@ -175,6 +176,15 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
target: 'done',
actions: ['storeDegradedFieldTableOptions'],
},
OPEN_DEGRADED_FIELD_FLYOUT: {
target:
'#DatasetQualityDetailsController.initializing.initializeFixItFlow.ignoredValues',
actions: ['storeExpandedDegradedField'],
},
CLOSE_DEGRADED_FIELD_FLYOUT: {
target: 'done',
actions: ['storeExpandedDegradedField'],
},
},
},
},
@ -267,6 +277,42 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
},
},
},
initializeFixItFlow: {
initial: 'closed',
type: 'parallel',
states: {
ignoredValues: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'loadDegradedFieldValues',
onDone: {
target: 'done',
actions: ['storeDegradedFieldValues'],
},
onError: [
{
target: '#DatasetQualityDetailsController.indexNotFound',
cond: 'isIndexNotFoundError',
},
{
target: 'done',
},
],
},
},
done: {
on: {
UPDATE_TIME_RANGE: {
target: 'fetching',
},
},
},
},
},
},
},
},
},
indexNotFound: {
@ -317,6 +363,13 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
}
: {};
}),
storeDegradedFieldValues: assign((_, event: DoneInvokeEvent<DegradedFieldValues>) => {
return 'data' in event
? {
degradedFieldValues: event.data,
}
: {};
}),
storeDegradedFieldTableOptions: assign((context, event) => {
return 'degraded_field_criteria' in event
? {
@ -327,6 +380,11 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
}
: {};
}),
storeExpandedDegradedField: assign((context, event) => {
return {
expandedDegradedField: 'fieldName' in event ? event.fieldName : undefined,
};
}),
resetDegradedFieldPageAndRowsPerPage: assign((context, _event) => ({
degradedFields: {
...context.degradedFields,
@ -472,6 +530,13 @@ export const createDatasetQualityDetailsControllerStateMachine = ({
end,
});
},
loadDegradedFieldValues: (context) => {
return dataStreamDetailsClient.getDataStreamDegradedFieldValues({
dataStream: context.dataStream,
degradedField: context.expandedDegradedField!,
});
},
loadDataStreamSettings: (context) => {
return dataStreamDetailsClient.getDataStreamSettings({
dataStream: context.dataStream,

View file

@ -13,6 +13,7 @@ import {
DataStreamSettings,
DegradedField,
DegradedFieldResponse,
DegradedFieldValues,
NonAggregatableDatasets,
} from '../../../common/api_types';
import { TableCriteria, TimeRangeConfig } from '../../../common/types';
@ -43,6 +44,7 @@ export interface WithDefaultControllerState {
isBreakdownFieldEcs?: boolean;
isIndexNotFoundError?: boolean;
integration?: Integration;
expandedDegradedField?: string;
}
export interface WithDataStreamDetails {
@ -74,6 +76,10 @@ export interface WithIntegration {
integrationDashboards?: Dashboard[];
}
export interface WithDegradedFieldValues {
degradedFieldValues: DegradedFieldValues;
}
export type DefaultDatasetQualityDetailsContext = Pick<
WithDefaultControllerState,
'degradedFields' | 'timeRange' | 'isIndexNotFoundError'
@ -110,6 +116,14 @@ export type DatasetQualityDetailsControllerTypeState =
value: 'initializing.dataStreamDegradedFields.done';
context: WithDefaultControllerState & WithDegradedFieldsData;
}
| {
value: 'initializing.initializeFixItFlow.ignoredValues.fetching';
context: WithDefaultControllerState & WithDegradedFieldsData;
}
| {
value: 'initializing.initializeFixItFlow.ignoredValues.done';
context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues;
}
| {
value:
| 'initializing.dataStreamSettings.initializeIntegrations'
@ -133,6 +147,13 @@ export type DatasetQualityDetailsControllerEvent =
type: 'UPDATE_TIME_RANGE';
timeRange: TimeRangeConfig;
}
| {
type: 'OPEN_DEGRADED_FIELD_FLYOUT';
fieldName: string | undefined;
}
| {
type: 'CLOSE_DEGRADED_FIELD_FLYOUT';
}
| {
type: 'BREAKDOWN_FIELD_CHANGE';
breakdownField: string | undefined;
@ -146,6 +167,7 @@ export type DatasetQualityDetailsControllerEvent =
| DoneInvokeEvent<Error>
| DoneInvokeEvent<boolean>
| DoneInvokeEvent<DegradedFieldResponse>
| DoneInvokeEvent<DegradedFieldValues>
| DoneInvokeEvent<DataStreamSettings>
| DoneInvokeEvent<Integration>
| DoneInvokeEvent<Dashboard[]>;

View file

@ -0,0 +1,65 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { SearchHit } from '@kbn/es-types';
import { DegradedFieldValues } from '../../../../common/api_types';
import { createDatasetQualityESClient } from '../../../utils';
import { _IGNORED, TIMESTAMP } from '../../../../common/es_fields';
export async function getDegradedFieldValues({
esClient,
dataStream,
degradedField,
}: {
esClient: ElasticsearchClient;
dataStream: string;
degradedField: string;
}): Promise<DegradedFieldValues> {
const datasetQualityESClient = createDatasetQualityESClient(esClient);
const response = await datasetQualityESClient.search({
index: dataStream,
size: 4,
fields: [degradedField],
query: { term: { [_IGNORED]: degradedField } },
sort: [
{
[TIMESTAMP]: {
order: 'desc',
},
},
],
});
const values = extractAndDeduplicateValues(response.hits.hits, degradedField);
return {
field: degradedField,
values,
};
}
function extractAndDeduplicateValues(searchHits: SearchHit[], key: string): string[] {
const values: string[] = [];
searchHits.forEach((hit: any) => {
const fieldValue = hit.ignored_field_values?.[key];
if (fieldValue) {
if (Array.isArray(fieldValue)) {
values.push(...fieldValue);
} else {
values.push(fieldValue);
}
}
});
// Flatten and deduplicate the array
const deduplicatedValues = Array.from(new Set(values.flat()));
return deduplicatedValues;
}

View file

@ -15,6 +15,7 @@ import {
NonAggregatableDatasets,
DegradedFieldResponse,
DatasetUserPrivileges,
DegradedFieldValues,
} from '../../../common/api_types';
import { rangeRt, typeRt, typesRt } from '../../types/default_api_types';
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
@ -25,6 +26,7 @@ import { getDataStreamsStats } from './get_data_streams_stats';
import { getDegradedDocsPaginated } from './get_degraded_docs';
import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams';
import { getDegradedFields } from './get_degraded_fields';
import { getDegradedFieldValues } from './get_degraded_field_values';
const statsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/stats',
@ -192,6 +194,33 @@ const degradedFieldsRoute = createDatasetQualityServerRoute({
},
});
const degradedFieldValuesRoute = createDatasetQualityServerRoute({
endpoint:
'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_field/{degradedField}/values',
params: t.type({
path: t.type({
dataStream: t.string,
degradedField: t.string,
}),
}),
options: {
tags: [],
},
async handler(resources): Promise<DegradedFieldValues> {
const { context, params } = resources;
const { dataStream, degradedField } = params.path;
const coreContext = await context.core;
const esClient = coreContext.elasticsearch.client.asCurrentUser;
return await getDegradedFieldValues({
esClient,
dataStream,
degradedField,
});
},
});
const dataStreamSettingsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings',
params: t.type({
@ -258,6 +287,7 @@ export const dataStreamsRouteRepository = {
...nonAggregatableDatasetsRoute,
...nonAggregatableDatasetRoute,
...degradedFieldsRoute,
...degradedFieldValuesRoute,
...dataStreamDetailsRoute,
...dataStreamSettingsRoute,
};

View file

@ -0,0 +1,99 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { DatasetQualityApiClientKey } from '../../common/config';
import { FtrProviderContext } from '../../common/ftr_provider_context';
const MORE_THAN_1024_CHARS =
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
const ANOTHER_1024_CHARS =
'grape fig tangerine tangerine kiwi lemon papaya cherry nectarine papaya mango cherry nectarine fig cherry fig grape mango mango quince fig strawberry mango quince date kiwi quince raspberry apple kiwi banana quince fig papaya grape mango cherry banana mango cherry lemon cherry tangerine fig quince quince papaya tangerine grape strawberry banana kiwi grape mango papaya nectarine banana nectarine kiwi papaya lemon apple lemon orange fig cherry grape apple nectarine papaya orange fig papaya date mango papaya mango cherry tangerine papaya apple banana papaya cherry strawberry grape raspberry lemon date papaya mango kiwi cherry fig banana banana apple date strawberry mango tangerine date lemon kiwi quince date orange orange papaya date apple fig tangerine quince tangerine date papaya banana banana orange raspberry papaya apple nectarine lemon raspberry raspberry mango cherry kiwi cherry cherry nectarine cherry date strawberry banana orange mango mango tangerine quince papaya papaya kiwi papaya strawberry date mango';
export default function ApiTest({ getService }: FtrProviderContext) {
const registry = getService('registry');
const synthtrace = getService('logSynthtraceEsClient');
const datasetQualityApiClient = getService('datasetQualityApiClient');
const start = '2024-08-28T08:00:00.000Z';
const end = '2024-08-28T08:02:00.000Z';
const degradedFieldDataset = 'nginx.error';
const degradedFieldsDatastream = 'logs-nginx.error-default';
const degradedFieldName = 'test_field';
const regularFieldName = 'service.name';
const serviceName = 'my-service';
async function callApiAs({
user,
dataStream,
degradedField,
}: {
user: DatasetQualityApiClientKey;
dataStream: string;
degradedField: string;
}) {
return await datasetQualityApiClient[user]({
endpoint:
'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_field/{degradedField}/values',
params: {
path: {
dataStream,
degradedField,
},
},
});
}
registry.when('Degraded Fields Values per field', { config: 'basic' }, () => {
describe('gets the degraded fields values for a given field', () => {
before(async () => {
await synthtrace.index([
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a error message')
.logLevel(MORE_THAN_1024_CHARS)
.timestamp(timestamp)
.dataset(degradedFieldDataset)
.defaults({
'log.file.path': '/error.log',
'service.name': serviceName + 1,
'trace.id': MORE_THAN_1024_CHARS,
test_field: [ANOTHER_1024_CHARS, 'hello world', MORE_THAN_1024_CHARS],
})
),
]);
});
after(async () => {
await synthtrace.clean();
});
it('returns no values when provided field has no degraded values', async () => {
const resp = await callApiAs({
user: 'datasetQualityLogsUser',
dataStream: degradedFieldsDatastream,
degradedField: regularFieldName,
});
expect(resp.body.values.length).to.be(0);
});
it('returns values when provided field has degraded values', async () => {
const resp = await callApiAs({
user: 'datasetQualityLogsUser',
dataStream: degradedFieldsDatastream,
degradedField: degradedFieldName,
});
expect(resp.body.values.length).to.be(2);
});
});
});
}

View file

@ -169,7 +169,7 @@ export function createDegradedFieldsRecord({
.defaults({
'trace.id': generateShortId(),
'agent.name': 'synth-agent',
'cloud.availability_zone': MORE_THAN_1024_CHARS,
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
})
.timestamp(timestamp),
log
@ -213,5 +213,7 @@ const CLUSTER = [
const SERVICE_NAMES = [`synth-service-0`, `synth-service-1`, `synth-service-2`];
const MORE_THAN_1024_CHARS =
export const MORE_THAN_1024_CHARS =
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
export const ANOTHER_1024_CHARS =
'grape fig tangerine tangerine kiwi lemon papaya cherry nectarine papaya mango cherry nectarine fig cherry fig grape mango mango quince fig strawberry mango quince date kiwi quince raspberry apple kiwi banana quince fig papaya grape mango cherry banana mango cherry lemon cherry tangerine fig quince quince papaya tangerine grape strawberry banana kiwi grape mango papaya nectarine banana nectarine kiwi papaya lemon apple lemon orange fig cherry grape apple nectarine papaya orange fig papaya date mango papaya mango cherry tangerine papaya apple banana papaya cherry strawberry grape raspberry lemon date papaya mango kiwi cherry fig banana banana apple date strawberry mango tangerine date lemon kiwi quince date orange orange papaya date apple fig tangerine quince tangerine date papaya banana banana orange raspberry papaya apple nectarine lemon raspberry raspberry mango cherry kiwi cherry cherry nectarine cherry date strawberry banana orange mango mango tangerine quince papaya papaya kiwi papaya strawberry date mango';

View file

@ -371,7 +371,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
const rows =
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
expect(rows.length).to.eql(2);
expect(rows.length).to.eql(3);
});
it('should display Spark Plot for every row of degraded fields', async () => {

View file

@ -0,0 +1,108 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { DatasetQualityFtrProviderContext } from './config';
import {
createDegradedFieldsRecord,
datasetNames,
defaultNamespace,
getInitialTestLogs,
ANOTHER_1024_CHARS,
MORE_THAN_1024_CHARS,
} from './data';
export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) {
const PageObjects = getPageObjects([
'common',
'navigationalSearch',
'observabilityLogsExplorer',
'datasetQuality',
]);
const testSubjects = getService('testSubjects');
const synthtrace = getService('logSynthtraceEsClient');
const retry = getService('retry');
const to = '2024-01-01T12:00:00.000Z';
const degradedDatasetName = datasetNames[2];
const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`;
describe('Degraded fields flyout', () => {
before(async () => {
await synthtrace.index([
// Ingest basic logs
getInitialTestLogs({ to, count: 4 }),
// Ingest Degraded Logs
createDegradedFieldsRecord({
to: new Date().toISOString(),
count: 2,
dataset: degradedDatasetName,
}),
]);
});
after(async () => {
await synthtrace.clean();
});
describe('degraded field flyout open-close', () => {
it('should open and close the flyout when user clicks on the expand button', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
});
await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field');
await testSubjects.existOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
await PageObjects.datasetQuality.closeFlyout();
await testSubjects.missingOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
});
it('should open the flyout when navigating to the page with degradedField in URL State', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
expandedDegradedField: 'test_field',
});
await testSubjects.existOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
await PageObjects.datasetQuality.closeFlyout();
});
});
describe('values exist', () => {
it('should display the degraded field values', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
expandedDegradedField: 'test_field',
});
await retry.tryForTime(5000, async () => {
const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist(
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
ANOTHER_1024_CHARS
);
const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist(
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
MORE_THAN_1024_CHARS
);
expect(cloudAvailabilityZoneValueExists).to.be(true);
expect(cloudAvailabilityZoneValue2Exists).to.be(true);
});
await PageObjects.datasetQuality.closeFlyout();
});
});
});
}

View file

@ -15,5 +15,6 @@ export default function ({ loadTestFile }: DatasetQualityFtrProviderContext) {
loadTestFile(require.resolve('./dataset_quality_table_filters'));
loadTestFile(require.resolve('./dataset_quality_privileges'));
loadTestFile(require.resolve('./dataset_quality_details'));
loadTestFile(require.resolve('./dataset_quality_details_degraded_field_flyout'));
});
}

View file

@ -5,6 +5,7 @@
* 2.0.
*/
import expect from '@kbn/expect';
import querystring from 'querystring';
import rison from '@kbn/rison';
import { WebElementWrapper } from '@kbn/ftr-common-functional-ui-services';
@ -91,6 +92,9 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
datasetQualityTable: 'datasetQualityTable',
datasetQualityFiltersContainer: 'datasetQualityFiltersContainer',
datasetQualityExpandButton: 'datasetQualityExpandButton',
datasetQualityDetailsDegradedFieldsExpandButton:
'datasetQualityDetailsDegradedFieldsExpandButton',
datasetQualityDetailsDegradedFieldFlyout: 'datasetQualityDetailsDegradedFieldFlyout',
datasetDetailsContainer: 'datasetDetailsContainer',
datasetQualityDetailsTitle: 'datasetQualityDetailsTitle',
datasetQualityDetailsDegradedFieldTable: 'datasetQualityDetailsDegradedFieldTable',
@ -127,6 +131,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
'unifiedHistogramBreakdownSelectorSelectorSearch',
unifiedHistogramBreakdownSelectorSelectable: 'unifiedHistogramBreakdownSelectorSelectable',
managementHome: 'managementHome',
euiFlyoutCloseButton: 'euiFlyoutCloseButton',
};
return {
@ -193,6 +198,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
}
},
async waitUntilDegradedFieldFlyoutLoaded() {
await testSubjects.existOrFail(testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout);
},
async parseSummaryPanel(excludeKeys: string[] = []): Promise<SummaryPanelKpi> {
const isStateful = !excludeKeys.includes('estimatedData');
@ -282,7 +291,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
async parseDegradedFieldTable() {
await this.waitUntilTableLoaded();
const table = await this.getDatasetQualityDetailsDegradedFieldTable();
return this.parseTable(table, ['Field', 'Docs count', 'Last Occurrence']);
return this.parseTable(table, ['0', 'Field', 'Docs count', 'Last Occurrence']);
},
async filterForIntegrations(integrations: string[]) {
@ -398,6 +407,39 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
);
},
async openDegradedFieldFlyout(fieldName: string) {
await this.waitUntilTableLoaded();
const cols = await this.parseDegradedFieldTable();
const fieldNameCol = cols.Field;
const fieldNameColCellTexts = await fieldNameCol.getCellTexts();
const testDatasetRowIndex = fieldNameColCellTexts.findIndex((dName) => dName === fieldName);
expect(testDatasetRowIndex).to.be.greaterThan(-1);
const expandColumn = cols['0'];
const expandButtons = await expandColumn.getCellChildren(
`[data-test-subj=${testSubjectSelectors.datasetQualityDetailsDegradedFieldsExpandButton}]`
);
expect(expandButtons.length).to.be.greaterThan(0);
const fieldExpandButton = expandButtons[testDatasetRowIndex];
// Check if 'title' attribute is "Expand" or "Collapse"
const isCollapsed = (await fieldExpandButton.getAttribute('title')) === 'Expand';
// Open if collapsed
if (isCollapsed) {
await fieldExpandButton.click();
}
await this.waitUntilDegradedFieldFlyoutLoaded();
},
async closeFlyout() {
return testSubjects.click(testSubjectSelectors.euiFlyoutCloseButton);
},
async parseTable(tableWrapper: WebElementWrapper, columnNamesOrIndexes: string[]) {
const headerElementWrappers = await tableWrapper.findAllByCssSelector('thead th, thead td');

View file

@ -0,0 +1,110 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { DatasetQualityFtrContextProvider } from './common/services';
import type { InternalRequestHeader, RoleCredentials } from '../../../../shared/services';
const MORE_THAN_1024_CHARS =
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
const ANOTHER_1024_CHARS =
'grape fig tangerine tangerine kiwi lemon papaya cherry nectarine papaya mango cherry nectarine fig cherry fig grape mango mango quince fig strawberry mango quince date kiwi quince raspberry apple kiwi banana quince fig papaya grape mango cherry banana mango cherry lemon cherry tangerine fig quince quince papaya tangerine grape strawberry banana kiwi grape mango papaya nectarine banana nectarine kiwi papaya lemon apple lemon orange fig cherry grape apple nectarine papaya orange fig papaya date mango papaya mango cherry tangerine papaya apple banana papaya cherry strawberry grape raspberry lemon date papaya mango kiwi cherry fig banana banana apple date strawberry mango tangerine date lemon kiwi quince date orange orange papaya date apple fig tangerine quince tangerine date papaya banana banana orange raspberry papaya apple nectarine lemon raspberry raspberry mango cherry kiwi cherry cherry nectarine cherry date strawberry banana orange mango mango tangerine quince papaya papaya kiwi papaya strawberry date mango';
export default function ApiTest({ getService }: DatasetQualityFtrContextProvider) {
const synthtrace = getService('logSynthtraceEsClient');
const datasetQualityApiClient = getService('datasetQualityApiClient');
const svlUserManager = getService('svlUserManager');
const svlCommonApi = getService('svlCommonApi');
const start = '2024-08-28T08:00:00.000Z';
const end = '2024-08-28T08:02:00.000Z';
const degradedFieldDataset = 'nginx.error';
const degradedFieldsDatastream = 'logs-nginx.error-default';
const degradedFieldName = 'test_field';
const regularFieldName = 'service.name';
const serviceName = 'my-service';
async function callApiAs({
dataStream,
degradedField,
roleAuthc,
internalReqHeader,
}: {
dataStream: string;
degradedField: string;
roleAuthc: RoleCredentials;
internalReqHeader: InternalRequestHeader;
}) {
return await datasetQualityApiClient.slsUser({
endpoint:
'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_field/{degradedField}/values',
params: {
path: {
dataStream,
degradedField,
},
},
roleAuthc,
internalReqHeader,
});
}
describe('Degraded Fields Values per field', () => {
let roleAuthc: RoleCredentials;
let internalReqHeader: InternalRequestHeader;
before(async () => {
roleAuthc = await svlUserManager.createM2mApiKeyWithRoleScope('admin');
internalReqHeader = svlCommonApi.getInternalRequestHeader();
await synthtrace.index([
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp) =>
log
.create()
.message('This is a error message')
.logLevel(MORE_THAN_1024_CHARS)
.timestamp(timestamp)
.dataset(degradedFieldDataset)
.defaults({
'log.file.path': '/error.log',
'service.name': serviceName + 1,
'trace.id': MORE_THAN_1024_CHARS,
test_field: [ANOTHER_1024_CHARS, 'hello world', MORE_THAN_1024_CHARS],
})
),
]);
});
after(async () => {
await synthtrace.clean();
await svlUserManager.invalidateM2mApiKeyWithRoleScope(roleAuthc);
});
it('returns no values when provided field has no degraded values', async () => {
const resp = await callApiAs({
dataStream: degradedFieldsDatastream,
degradedField: regularFieldName,
roleAuthc,
internalReqHeader,
});
expect(resp.body.values.length).to.be(0);
});
it('returns values when provided field has degraded values', async () => {
const resp = await callApiAs({
dataStream: degradedFieldsDatastream,
degradedField: degradedFieldName,
roleAuthc,
internalReqHeader,
});
expect(resp.body.values.length).to.be(2);
});
});
}

View file

@ -10,5 +10,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('Dataset Quality', function () {
loadTestFile(require.resolve('./data_stream_details'));
loadTestFile(require.resolve('./data_stream_settings'));
loadTestFile(require.resolve('./degraded_field_values'));
});
}

View file

@ -169,7 +169,7 @@ export function createDegradedFieldsRecord({
.defaults({
'trace.id': generateShortId(),
'agent.name': 'synth-agent',
'cloud.availability_zone': MORE_THAN_1024_CHARS,
test_field: [MORE_THAN_1024_CHARS, ANOTHER_1024_CHARS],
})
.timestamp(timestamp),
log
@ -213,5 +213,7 @@ const CLUSTER = [
const SERVICE_NAMES = [`synth-service-0`, `synth-service-1`, `synth-service-2`];
const MORE_THAN_1024_CHARS =
export const MORE_THAN_1024_CHARS =
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
export const ANOTHER_1024_CHARS =
'grape fig tangerine tangerine kiwi lemon papaya cherry nectarine papaya mango cherry nectarine fig cherry fig grape mango mango quince fig strawberry mango quince date kiwi quince raspberry apple kiwi banana quince fig papaya grape mango cherry banana mango cherry lemon cherry tangerine fig quince quince papaya tangerine grape strawberry banana kiwi grape mango papaya nectarine banana nectarine kiwi papaya lemon apple lemon orange fig cherry grape apple nectarine papaya orange fig papaya date mango papaya mango cherry tangerine papaya apple banana papaya cherry strawberry grape raspberry lemon date papaya mango kiwi cherry fig banana banana apple date strawberry mango tangerine date lemon kiwi quince date orange orange papaya date apple fig tangerine quince tangerine date papaya banana banana orange raspberry papaya apple nectarine lemon raspberry raspberry mango cherry kiwi cherry cherry nectarine cherry date strawberry banana orange mango mango tangerine quince papaya papaya kiwi papaya strawberry date mango';

View file

@ -28,7 +28,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
'navigationalSearch',
'observabilityLogsExplorer',
'datasetQuality',
'svlCommonNavigation',
'svlCommonPage',
]);
const testSubjects = getService('testSubjects');
@ -375,7 +374,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
const rows =
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
expect(rows.length).to.eql(2);
expect(rows.length).to.eql(3);
});
it('should display Spark Plot for every row of degraded fields', async () => {

View file

@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import {
createDegradedFieldsRecord,
datasetNames,
defaultNamespace,
getInitialTestLogs,
ANOTHER_1024_CHARS,
MORE_THAN_1024_CHARS,
} from './data';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects([
'common',
'navigationalSearch',
'observabilityLogsExplorer',
'datasetQuality',
'svlCommonPage',
]);
const testSubjects = getService('testSubjects');
const synthtrace = getService('svlLogsSynthtraceClient');
const retry = getService('retry');
const to = '2024-01-01T12:00:00.000Z';
const degradedDatasetName = datasetNames[2];
const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`;
describe('Degraded fields flyout', () => {
before(async () => {
await synthtrace.index([
// Ingest basic logs
getInitialTestLogs({ to, count: 4 }),
// Ingest Degraded Logs
createDegradedFieldsRecord({
to: new Date().toISOString(),
count: 2,
dataset: degradedDatasetName,
}),
]);
await PageObjects.svlCommonPage.loginWithPrivilegedRole();
});
after(async () => {
// await synthtrace.clean();
});
describe('degraded field flyout open-close', () => {
it('should open and close the flyout when user clicks on the expand button', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
});
await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field');
await testSubjects.existOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
await PageObjects.datasetQuality.closeFlyout();
await testSubjects.missingOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
});
it('should open the flyout when navigating to the page with degradedField in URL State', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
expandedDegradedField: 'test_field',
});
await testSubjects.existOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
await PageObjects.datasetQuality.closeFlyout();
await testSubjects.missingOrFail(
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
);
});
});
describe('values exist', () => {
it('should display the degraded field values', async () => {
await PageObjects.datasetQuality.navigateToDetails({
dataStream: degradedDataStreamName,
expandedDegradedField: 'test_field',
});
await retry.tryForTime(5000, async () => {
const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist(
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
ANOTHER_1024_CHARS
);
const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist(
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
MORE_THAN_1024_CHARS
);
expect(cloudAvailabilityZoneValueExists).to.be(true);
expect(cloudAvailabilityZoneValue2Exists).to.be(true);
});
await PageObjects.datasetQuality.closeFlyout();
});
});
});
}

View file

@ -15,5 +15,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./dataset_quality_table_filters'));
loadTestFile(require.resolve('./dataset_quality_privileges'));
loadTestFile(require.resolve('./dataset_quality_details'));
loadTestFile(require.resolve('./dataset_quality_details_degraded_field_flyout'));
});
}