github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Dataset quality] Warning for datasets not supporting _ignored aggregation (#183183)

Browse source 
Closes https://github.com/elastic/kibana/issues/179227.

## 📝  Summary

This PR adds a warning to main page and flyout whenever a dataStream has
indices that doesn't support `_ignored` aggregation

## 🎥 Demo


c6d5fd81-d9a9-4fcc-92d0-8e65b996df9c

#### Flyout


2972e104-8b10-413a-a430-3efc5252b757
This commit is contained in:
Yngrid Coello 2024-05-14 21:07:18 +02:00 • committed by GitHub
parent e2dd5ee912
commit 3caf266cf8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 503 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

9
x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts
View file

@ -119,3 +119,12 @@ export type DataStreamsEstimatedDataInBytes = rt.TypeOf<typeof dataStreamsEstima
export const getDataStreamsEstimatedDataInBytesResponseRt = rt.exact(
dataStreamsEstimatedDataInBytesRT
);
export const getNonAggregatableDatasetsRt = rt.exact(
rt.type({
aggregatable: rt.boolean,
datasets: rt.array(rt.string),
})
);
export type NonAggregatableDatasets = rt.TypeOf<typeof getNonAggregatableDatasetsRt>;

5
x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts
View file

@ -43,6 +43,11 @@ export type GetDataStreamDetailsParams = GetDataStreamDetailsPathParams &
export type GetDataStreamDetailsResponse =
APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>;
export type GetNonAggregatableDataStreamsParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/non_aggregatable`>['params']['query'];
export type GetNonAggregatableDataStreamsResponse =
APIReturnType<`GET /internal/dataset_quality/data_streams/non_aggregatable`>;
export type GetDataStreamsEstimatedDataInBytesParams =
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/estimated_data`>['params'];
export type GetDataStreamsEstimatedDataInBytesResponse =

26
x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.test.ts
View file

@ -55,27 +55,33 @@ describe('dataset_name', () => {
});
describe('extractIndexNameFromBackingIndex', () => {
it('returns the correct index name if backing index provieded', () => {
it('returns the correct index name if backing index provided', () => {
expect(
extractIndexNameFromBackingIndex(
'.ds-logs-apm.app.adservice-default-2024.04.29-000001',
'logs'
)
extractIndexNameFromBackingIndex('.ds-logs-apm.app.adservice-default-2024.04.29-000001')
).toEqual('logs-apm.app.adservice-default');
});
it('returns the correct index name if index name is passed', () => {
expect(extractIndexNameFromBackingIndex('logs-nginx.access-default', 'logs')).toEqual(
expect(extractIndexNameFromBackingIndex('logs-nginx.access-default')).toEqual(
'logs-nginx.access-default'
);
});
it('returns the correct index name if backing index contains _', () => {
expect(
extractIndexNameFromBackingIndex('.ds-logs-elastic_agent-default-2024.04.29-000001')
).toEqual('logs-elastic_agent-default');
});
it('returns the correct index name if backing index contains only -', () => {
expect(
extractIndexNameFromBackingIndex('.ds-logs-generic-pods-default-2024.04.29-000001')
).toEqual('logs-generic-pods-default');
});
it('handles different types', () => {
expect(
extractIndexNameFromBackingIndex(
'.ds-metrics-apm.app.adservice-default-2024.04.29-000001',
'metrics'
)
extractIndexNameFromBackingIndex('.ds-metrics-apm.app.adservice-default-2024.04.29-000001')
).toEqual('metrics-apm.app.adservice-default');
});
});

10
x-pack/plugins/observability_solution/dataset_quality/common/utils/dataset_name.ts
View file

@ -39,14 +39,8 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => {
};
};
export const extractIndexNameFromBackingIndex = (
indexString: string,
type: DataStreamType
): string => {
const pattern: RegExp = new RegExp(
`(?:\\.ds-)?(${type}-(?:[^-.]+(?:\\.[^.]+)+)-[^-]+)-\\d{4}\\.\\d{2}\\.\\d{2}-\\d{6}`
);
export const extractIndexNameFromBackingIndex = (indexString: string): string => {
const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
const match = indexString.match(pattern);
return match ? match[1] : indexString;

4
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/dataset_quality.tsx
View file

@ -56,6 +56,7 @@ export const createDatasetQuality = ({
};
const Header = dynamic(() => import('./header'));
const Warnings = dynamic(() => import('./warnings/warnings'));
const Table = dynamic(() => import('./table/table'));
const Filters = dynamic(() => import('./filters/filters'));
const SummaryPanel = dynamic(() => import('./summary_panel/summary_panel'));
@ -66,6 +67,9 @@ function DatasetQuality() {
<EuiFlexItem grow={false}>
<Header />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<Warnings />
</EuiFlexItem>
<EuiFlexItem grow={false}>
<Filters />
</EuiFlexItem>

97
x-pack/plugins/observability_solution/dataset_quality/public/components/dataset_quality/warnings/warnings.tsx Normal file
View file

@ -0,0 +1,97 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiAccordion, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiLink } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import React from 'react';
import { useDatasetQualityWarnings } from '../../../hooks/use_dataset_quality_warnings';
const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', {
defaultMessage: 'Your request may take longer to complete',
});
const nonAggregatableWarningDescription = (nonAggregatableDatasets: string[]) => (
<FormattedMessage
id="xpack.datasetQuality.nonAggregatable.description"
defaultMessage="{description}"
values={{
description: (
<FormattedMessage
id="xpack.datasetQuality.nonAggregatable.warning"
defaultMessage="Some of your datasets do not support _ignored aggregation and may cause delays when querying data. {howToFixIt} {showDatasets}"
values={{
showDatasets: (
<FormattedMessage
id="xpack.datasetQuality.nonAggregatable.warning.description."
defaultMessage="{accordion}"
values={{
accordion: (
<EuiAccordion
style={{ marginTop: '12px ' }}
buttonContent={i18n.translate(
'xpack.datasetQuality.nonAggregatable.showAffectedDatasets',
{
defaultMessage: 'Show affected datasets',
}
)}
id={''}
>
<ul>
{nonAggregatableDatasets.map((dataset) => (
<li key={dataset}>{dataset}</li>
))}
</ul>
</EuiAccordion>
),
}}
/>
),
howToFixIt: (
<FormattedMessage
id="xpack.datasetQuality.nonAggregatable.howToFixIt"
defaultMessage="Manually {rolloverLink} these datasets to prevent future delays."
values={{
rolloverLink: (
<EuiLink
external
target="_blank"
data-test-subj="datasetQualityNonAggregatableHowToFixItLink"
href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-rollover-index.html"
>
{i18n.translate('xpack.datasetQuality.nonAggregatableDatasets.link.title', {
defaultMessage: 'rollover',
})}
</EuiLink>
),
}}
/>
),
}}
/>
),
}}
/>
);
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function Warnings() {
const { loading, nonAggregatableDatasets } = useDatasetQualityWarnings();
return (
<EuiFlexGroup data-test-subj="datasetQualityWarningsContainer" gutterSize="s" wrap>
{!loading && nonAggregatableDatasets.length > 0 && (
<EuiFlexItem>
<EuiCallOut title={nonAggregatableWarningTitle} color="warning" iconType="warning">
<p>{nonAggregatableWarningDescription(nonAggregatableDatasets)}</p>
</EuiCallOut>
</EuiFlexItem>
)}
</EuiFlexGroup>
);
}

2
x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout.tsx
View file

@ -34,6 +34,7 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) {
dataStreamStat,
dataStreamSettings,
dataStreamDetails,
isNonAggregatable,
fieldFormats,
timeRange,
loadingState,
@ -60,6 +61,7 @@ export default function Flyout({ dataset, closeFlyout }: FlyoutProps) {
dataStreamDetails={dataStreamDetails}
dataStreamDetailsLoading={loadingState.dataStreamDetailsLoading}
timeRange={timeRange}
isNonAggregatable={isNonAggregatable}
/>
</EuiPanel>

76
x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx
View file

@ -6,8 +6,19 @@
*/
import React, { useCallback, useState } from 'react';
import { OnRefreshProps, OnTimeChangeProps, EuiSpacer } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import {
OnRefreshProps,
OnTimeChangeProps,
EuiSpacer,
EuiFlexGroup,
EuiFlexItem,
EuiCallOut,
EuiLink,
EuiCode,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
import { DegradedDocs } from '../degraded_docs_trend/degraded_docs';
import { DataStreamDetails } from '../../../../common/api_types';
import { DEFAULT_TIME_RANGE, DEFAULT_DATEPICKER_REFRESH } from '../../../../common/constants';
@ -16,10 +27,60 @@ import { FlyoutDataset, TimeRangeConfig } from '../../../state_machines/dataset_
import { FlyoutSummaryHeader } from './flyout_summary_header';
import { FlyoutSummaryKpis, FlyoutSummaryKpisLoading } from './flyout_summary_kpis';
const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', {
defaultMessage: 'Your request may take longer to complete',
});
const nonAggregatableWarningDescription = (dataset: string) => (
<FormattedMessage
id="xpack.datasetQuality.flyout.nonAggregatable.description"
defaultMessage="{description}"
values={{
description: (
<FormattedMessage
id="xpack.datasetQuality.flyout.nonAggregatable.warning"
defaultMessage="{dataset}does not support _ignored aggregation and may cause delays when querying data. {howToFixIt}"
values={{
dataset: (
<EuiCode language="json" transparentBackground>
{dataset}
</EuiCode>
),
howToFixIt: (
<FormattedMessage
id="xpack.datasetQuality.flyout.nonAggregatable.howToFixIt"
defaultMessage="Manually {rolloverLink} this dataset to prevent future delays."
values={{
rolloverLink: (
<EuiLink
external
target="_blank"
data-test-subj="datasetQualityFlyoutNonAggregatableHowToFixItLink"
href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-rollover-index.html"
>
{i18n.translate(
'xpack.datasetQuality.flyout.nonAggregatableDatasets.link.title',
{
defaultMessage: 'rollover',
}
)}
</EuiLink>
),
}}
/>
),
}}
/>
),
}}
/>
);
export function FlyoutSummary({
dataStream,
dataStreamStat,
dataStreamDetails,
isNonAggregatable,
dataStreamDetailsLoading,
timeRange = { ...DEFAULT_TIME_RANGE, refresh: DEFAULT_DATEPICKER_REFRESH },
}: {
@ -28,6 +89,7 @@ export function FlyoutSummary({
dataStreamDetails?: DataStreamDetails;
dataStreamDetailsLoading: boolean;
timeRange?: TimeRangeConfig;
isNonAggregatable?: boolean;
}) {
const { service } = useDatasetQualityContext();
const [lastReloadTime, setLastReloadTime] = useState<number>(Date.now());
@ -72,6 +134,18 @@ export function FlyoutSummary({
return (
<>
{isNonAggregatable && (
<EuiFlexGroup
data-test-subj="datasetQualityFlyoutNonAggregatableWarning"
style={{ marginBottom: '24px' }}
>
<EuiFlexItem>
<EuiCallOut title={nonAggregatableWarningTitle} color="warning" iconType="warning">
<p>{nonAggregatableWarningDescription(dataStream)}</p>
</EuiCallOut>
</EuiFlexItem>
</EuiFlexGroup>
)}
<FlyoutSummaryHeader
timeRange={timeRange}
onTimeChange={handleTimeChange}

2
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx
View file

@ -22,6 +22,7 @@ export const useDatasetQualityFlyout = () => {
datasetDetails: dataStreamDetails,
insightsTimeRange,
breakdownField,
isNonAggregatable,
} = useSelector(service, (state) => state.context.flyout) ?? {};
const { timeRange } = useSelector(service, (state) => state.context.filters);
@ -36,6 +37,7 @@ export const useDatasetQualityFlyout = () => {
dataStreamStat,
dataStreamSettings,
dataStreamDetails,
isNonAggregatable,
fieldFormats,
timeRange: insightsTimeRange ?? timeRange,
breakdownField,

23
x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_warnings.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { useSelector } from '@xstate/react';
import { useDatasetQualityContext } from '../components/dataset_quality/context';
export function useDatasetQualityWarnings() {
const { service } = useDatasetQualityContext();
const nonAggregatableDatasets = useSelector(
service,
(state) => state.context.nonAggregatableDatasets
);
const isNonAggregatableDatasetsLoading = useSelector(service, (state) =>
state.matches('nonAggregatableDatasets.fetching')
);
return { loading: isNonAggregatableDatasetsLoading, nonAggregatableDatasets };
}

35
x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts
View file

@ -7,26 +7,29 @@
import { HttpStart } from '@kbn/core/public';
import { decodeOrThrow } from '@kbn/io-ts-utils';
import { Integration } from '../../../common/data_streams_stats/integration';
import {
getDataStreamsDegradedDocsStatsResponseRt,
getDataStreamsStatsResponseRt,
getDataStreamsEstimatedDataInBytesResponseRt,
getDataStreamsStatsResponseRt,
getIntegrationsResponseRt,
getNonAggregatableDatasetsRt,
} from '../../../common/api_types';
import { DEFAULT_DATASET_TYPE } from '../../../common/constants';
import {
DataStreamStatServiceResponse,
GetDataStreamsDegradedDocsStatsQuery,
GetDataStreamsDegradedDocsStatsResponse,
GetDataStreamsEstimatedDataInBytesParams,
GetDataStreamsEstimatedDataInBytesResponse,
GetDataStreamsStatsError,
GetDataStreamsStatsQuery,
GetDataStreamsStatsResponse,
GetDataStreamsEstimatedDataInBytesParams,
GetDataStreamsEstimatedDataInBytesResponse,
GetIntegrationsParams,
GetNonAggregatableDataStreamsParams,
GetNonAggregatableDataStreamsResponse,
IntegrationsResponse,
} from '../../../common/data_streams_stats';
import { Integration } from '../../../common/data_streams_stats/integration';
import { IDataStreamsStatsClient } from './types';
export class DataStreamsStatsClient implements IDataStreamsStatsClient {
@ -78,6 +81,30 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
return degradedDocs;
}
public async getNonAggregatableDatasets(params: GetNonAggregatableDataStreamsParams) {
const response = await this.http
.get<GetNonAggregatableDataStreamsResponse>(
'/internal/dataset_quality/data_streams/non_aggregatable',
{
query: {
...params,
type: DEFAULT_DATASET_TYPE,
},
}
)
.catch((error) => {
throw new GetDataStreamsStatsError(`Failed to fetch non aggregatable datasets: ${error}`);
});
const nonAggregatableDatasets = decodeOrThrow(
getNonAggregatableDatasetsRt,
(message: string) =>
new GetDataStreamsStatsError(`Failed to fetch non aggregatable datasets: ${message}`)
)(response);
return nonAggregatableDatasets;
}
public async getDataStreamsEstimatedDataInBytes(
params: GetDataStreamsEstimatedDataInBytesParams
) {

7
x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts
View file

@ -10,10 +10,12 @@ import {
DataStreamDegradedDocsStatServiceResponse,
DataStreamStatServiceResponse,
GetDataStreamsDegradedDocsStatsQuery,
GetDataStreamsStatsQuery,
GetDataStreamsEstimatedDataInBytesParams,
GetDataStreamsEstimatedDataInBytesResponse,
GetDataStreamsStatsQuery,
GetIntegrationsParams,
GetNonAggregatableDataStreamsParams,
GetNonAggregatableDataStreamsResponse,
IntegrationsResponse,
} from '../../../common/data_streams_stats';
@ -36,4 +38,7 @@ export interface IDataStreamsStatsClient {
params: GetDataStreamsEstimatedDataInBytesParams
): Promise<GetDataStreamsEstimatedDataInBytesResponse>;
getIntegrations(params: GetIntegrationsParams['query']): Promise<IntegrationsResponse>;
getNonAggregatableDatasets(
params: GetNonAggregatableDataStreamsParams
): Promise<GetNonAggregatableDataStreamsResponse>;
}

1
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts
View file

@ -41,4 +41,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
},
flyout: {},
datasets: [],
nonAggregatableDatasets: [],
};

9
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts
View file

@ -44,6 +44,15 @@ export const fetchDegradedStatsFailedNotifier = (toasts: IToasts, error: Error)
});
};
export const fetchNonAggregatableDatasetsFailedNotifier = (toasts: IToasts, error: Error) => {
toasts.addDanger({
title: i18n.translate('xpack.datasetQuality.fetchNonAggregatableDatasetsFailed', {
defaultMessage: "We couldn't get non aggregatable datasets information.",
}),
text: error.message,
});
};
export const fetchIntegrationDashboardsFailedNotifier = (toasts: IToasts, error: Error) => {
toasts.addDanger({
title: i18n.translate('xpack.datasetQuality.fetchIntegrationDashboardsFailed', {

122
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts
View file

@ -17,6 +17,8 @@ import {
DataStreamDetails,
GetDataStreamsStatsQuery,
GetIntegrationsParams,
GetNonAggregatableDataStreamsParams,
GetNonAggregatableDataStreamsResponse,
} from '../../../../common/data_streams_stats';
import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat';
import { DataStreamType } from '../../../../common/types';
@ -32,11 +34,13 @@ import {
fetchIntegrationDashboardsFailedNotifier,
fetchIntegrationsFailedNotifier,
noDatasetSelected,
fetchNonAggregatableDatasetsFailedNotifier,
} from './notifications';
import {
DatasetQualityControllerContext,
DatasetQualityControllerEvent,
DatasetQualityControllerTypeState,
DefaultDatasetQualityControllerState,
FlyoutDataset,
} from './types';
@ -173,12 +177,69 @@ export const createPureDatasetQualityControllerStateMachine = (
},
},
},
nonAggregatableDatasets: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'loadNonAggregatableDatasets',
onDone: {
target: 'loaded',
actions: ['storeNonAggregatableDatasets'],
},
onError: {
target: 'loaded',
actions: ['notifyFetchNonAggregatableDatasetsFailed'],
},
},
},
loaded: {},
},
on: {
UPDATE_TIME_RANGE: {
target: 'nonAggregatableDatasets.fetching',
},
REFRESH_DATA: {
target: 'nonAggregatableDatasets.fetching',
},
},
},
flyout: {
initial: 'closed',
states: {
initializing: {
type: 'parallel',
states: {
nonAggregatableDataset: {
initial: 'fetching',
states: {
fetching: {
invoke: {
src: 'loadDatasetIsNonAggregatable',
onDone: {
target: 'done',
actions: ['storeDatasetIsNonAggregatable'],
},
onError: {
target: 'done',
actions: ['notifyFetchNonAggregatableDatasetsFailed'],
},
},
},
done: {
on: {
UPDATE_INSIGHTS_TIME_RANGE: {
target: 'fetching',
actions: ['storeFlyoutOptions'],
},
SELECT_DATASET: {
target: 'fetching',
actions: ['storeFlyoutOptions'],
},
},
},
},
},
dataStreamSettings: {
initial: 'fetching',
states: {
@ -402,6 +463,18 @@ export const createPureDatasetQualityControllerStateMachine = (
}
: {};
}),
storeNonAggregatableDatasets: assign(
(
_context: DefaultDatasetQualityControllerState,
event: DoneInvokeEvent<GetNonAggregatableDataStreamsResponse>
) => {
return 'data' in event
? {
nonAggregatableDatasets: event.data.datasets,
}
: {};
}
),
storeDataStreamSettings: assign((context, event) => {
return 'data' in event
? {
@ -422,6 +495,21 @@ export const createPureDatasetQualityControllerStateMachine = (
}
: {};
}),
storeDatasetIsNonAggregatable: assign(
(
context: DefaultDatasetQualityControllerState,
event: DoneInvokeEvent<GetNonAggregatableDataStreamsResponse>
) => {
return 'data' in event
? {
flyout: {
...context.flyout,
isNonAggregatable: !event.data.aggregatable,
},
}
: {};
}
),
storeIntegrations: assign((_context, event) => {
return 'data' in event
? {
@ -484,6 +572,8 @@ export const createDatasetQualityControllerStateMachine = ({
fetchDatasetStatsFailedNotifier(toasts, event.data),
notifyFetchDegradedStatsFailed: (_context, event: DoneInvokeEvent<Error>) =>
fetchDegradedStatsFailedNotifier(toasts, event.data),
notifyFetchNonAggregatableDatasetsFailed: (_context, event: DoneInvokeEvent<Error>) =>
fetchNonAggregatableDatasetsFailedNotifier(toasts, event.data),
notifyFetchDatasetSettingsFailed: (_context, event: DoneInvokeEvent<Error>) =>
fetchDatasetSettingsFailedNotifier(toasts, event.data),
notifyFetchDatasetDetailsFailed: (_context, event: DoneInvokeEvent<Error>) =>
@ -514,6 +604,15 @@ export const createDatasetQualityControllerStateMachine = ({
type: context.type as GetIntegrationsParams['query']['type'],
});
},
loadNonAggregatableDatasets: (context) => {
const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange);
return dataStreamStatsClient.getNonAggregatableDatasets({
type: context.type as GetNonAggregatableDataStreamsParams['type'],
start,
end,
});
},
loadDataStreamSettings: (context) => {
if (!context.flyout.dataset) {
fetchDatasetSettingsFailedNotifier(toasts, new Error(noDatasetSelected));
@ -566,6 +665,29 @@ export const createDatasetQualityControllerStateMachine = ({
? dataStreamDetailsClient.getIntegrationDashboards({ integration: integration.name })
: Promise.resolve({});
},
loadDatasetIsNonAggregatable: async (context) => {
if (!context.flyout.dataset || !context.flyout.insightsTimeRange) {
fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected));
return Promise.resolve({});
}
const { type, name: dataset, namespace } = context.flyout.dataset;
const { startDate: start, endDate: end } = getDateISORange(
context.flyout.insightsTimeRange
);
return dataStreamStatsClient.getNonAggregatableDatasets({
type: context.type as GetNonAggregatableDataStreamsParams['type'],
start,
end,
dataStream: dataStreamPartsToIndexName({
type: type as DataStreamType,
dataset,
namespace,
}),
});
},
},
});

12
x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts
View file

@ -20,6 +20,7 @@ import {
IntegrationsResponse,
DataStreamStat,
DataStreamStatType,
GetNonAggregatableDataStreamsResponse,
} from '../../../../common/data_streams_stats';
export type FlyoutDataset = Omit<
@ -61,6 +62,7 @@ export interface WithFlyoutOptions {
datasetDetails?: DataStreamDetails;
insightsTimeRange?: TimeRangeConfig;
breakdownField?: string;
isNonAggregatable?: boolean;
};
}
@ -76,6 +78,10 @@ export interface WithDegradedDocs {
degradedDocStats: DegradedDocsStat[];
}
export interface WithNonAggregatableDatasets {
nonAggregatableDatasets: string[];
}
export interface WithDatasets {
datasets: DataStreamStat[];
}
@ -90,6 +96,7 @@ export type DefaultDatasetQualityControllerState = { type: string } & WithTableO
WithFlyoutOptions &
WithDatasets &
WithFilters &
WithNonAggregatableDatasets &
Partial<WithIntegrations>;
type DefaultDatasetQualityStateContext = DefaultDatasetQualityControllerState &
@ -120,6 +127,10 @@ export type DatasetQualityControllerTypeState =
value: 'integrations.fetching';
context: DefaultDatasetQualityStateContext;
}
| {
value: 'nonAggregatableDatasets.fetching';
context: DefaultDatasetQualityStateContext;
}
| {
value: 'flyout.initializing.dataStreamSettings.fetching';
context: DefaultDatasetQualityStateContext;
@ -189,6 +200,7 @@ export type DatasetQualityControllerEvent =
query: string;
}
| DoneInvokeEvent<DataStreamDegradedDocsStatServiceResponse>
| DoneInvokeEvent<GetNonAggregatableDataStreamsResponse>
| DoneInvokeEvent<DashboardType>
| DoneInvokeEvent<DataStreamDetails>
| DoneInvokeEvent<DataStreamSettings>

53
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_non_aggregatable_data_streams.ts Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
import { rangeQuery } from '@kbn/observability-plugin/server/utils/queries';
import { extractIndexNameFromBackingIndex } from '../../../common/utils';
import { DEFAULT_DATASET_TYPE } from '../../../common/constants';
import { _IGNORED } from '../../../common/es_fields';
import { DataStreamType } from '../../../common/types';
import { createDatasetQualityESClient } from '../../utils';
export async function getNonAggregatableDataStreams({
esClient,
type = DEFAULT_DATASET_TYPE,
start,
end,
dataStream,
}: {
esClient: ElasticsearchClient;
type?: DataStreamType;
start: number;
end: number;
dataStream?: string;
}) {
const datasetQualityESClient = createDatasetQualityESClient(esClient);
const response = await datasetQualityESClient.fieldCaps({
index: dataStream ?? `${type}-*`,
fields: [_IGNORED],
index_filter: {
...rangeQuery(start, end)[0],
},
});
const ignoredField = response.fields._ignored?._ignored;
const nonAggregatableIndices = ignoredField?.non_aggregatable_indices ?? [];
const nonAggregatableDatasets = new Set(
(Array.isArray(nonAggregatableIndices) ? nonAggregatableIndices : [nonAggregatableIndices]).map(
extractIndexNameFromBackingIndex
)
);
return {
aggregatable: ignoredField?.aggregatable ?? true,
datasets: Array.from(nonAggregatableDatasets),
};
}

30
x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts
View file

@ -13,6 +13,7 @@ import {
DataStreamSettings,
DataStreamStat,
DegradedDocs,
NonAggregatableDatasets,
} from '../../../common/api_types';
import { indexNameToDataStreamParts } from '../../../common/utils';
import { rangeRt, typeRt } from '../../types/default_api_types';
@ -22,6 +23,7 @@ import { getDataStreams } from './get_data_streams';
import { getDataStreamsStats } from './get_data_streams_stats';
import { getDegradedDocsPaginated } from './get_degraded_docs';
import { getEstimatedDataInBytes } from './get_estimated_data_in_bytes';
import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams';
const statsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/stats',
@ -95,6 +97,33 @@ const degradedDocsRoute = createDatasetQualityServerRoute({
},
});
const nonAggregatableDatasetsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/non_aggregatable',
params: t.type({
query: t.intersection([
rangeRt,
typeRt,
t.partial({
dataStream: t.string,
}),
]),
}),
options: {
tags: [],
},
async handler(resources): Promise<NonAggregatableDatasets> {
const { context, params } = resources;
const coreContext = await context.core;
const esClient = coreContext.elasticsearch.client.asCurrentUser;
return await getNonAggregatableDataStreams({
esClient,
...params.query,
});
},
});
const dataStreamSettingsRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings',
params: t.type({
@ -200,6 +229,7 @@ const estimatedDataInBytesRoute = createDatasetQualityServerRoute({
export const dataStreamsRouteRepository = {
...statsRoute,
...degradedDocsRoute,
...nonAggregatableDatasetsRoute,
...dataStreamDetailsRoute,
...dataStreamSettingsRoute,
...estimatedDataInBytesRoute,

5
x-pack/plugins/observability_solution/dataset_quality/server/utils/create_dataset_quality_es_client.ts
View file

@ -7,7 +7,7 @@
import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
import { ElasticsearchClient } from '@kbn/core/server';
import { Indices } from '@elastic/elasticsearch/lib/api/types';
import { FieldCapsRequest, FieldCapsResponse, Indices } from '@elastic/elasticsearch/lib/api/types';
type DatasetQualityESSearchParams = ESSearchRequest & {
size: number;
@ -32,5 +32,8 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) {
searches: searches.map((search) => [index, search]).flat(),
}) as Promise<any>;
},
async fieldCaps(params: FieldCapsRequest): Promise<FieldCapsResponse> {
return esClient.fieldCaps(params) as Promise<any>;
},
};
}