github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 18:51:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Dataset Quality] Indicate if failure store isn't enabled for data stream (#221644)

Browse source 
Added a tooltip and a link with documentation for Failed docs column
when dataset does not have failure store enabled.



https://github.com/user-attachments/assets/be65db9a-15c8-4087-b175-752b2fabab6e




For now it awaits for the documentation PR to be merged :
https://github.com/elastic/docs-content/pull/1368

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Yngrid Coello <yngrid.coello@elastic.co>
This commit is contained in:
Robert Stelmach 2025-06-24 09:49:36 +02:00 • committed by GitHub
parent 853c4bde73
commit 36cbbb9bf8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 264 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

2
x-pack/platform/plugins/shared/dataset_quality/common/api_types.ts
View file

@ -34,6 +34,7 @@ export const dataStreamStatRt = rt.intersection([
integration: rt.string,
totalDocs: rt.number,
creationDate: rt.number,
hasFailureStore: rt.boolean,
}),
]);
@ -236,6 +237,7 @@ export const dataStreamSettingsRt = rt.partial({
export type DataStreamSettings = rt.TypeOf<typeof dataStreamSettingsRt>;
export const dataStreamDetailsRt = rt.partial({
hasFailureStore: rt.boolean,
lastActivity: rt.number,
degradedDocsCount: rt.number,
failedDocsCount: rt.number,

6
x-pack/platform/plugins/shared/dataset_quality/common/data_streams_stats/data_stream_stat.ts
View file

@ -32,6 +32,7 @@ export class DataStreamStat {
docsInTimeRange?: number;
degradedDocs: QualityStat;
failedDocs: QualityStat;
hasFailureStore?: DataStreamStatType['hasFailureStore'];
private constructor(dataStreamStat: DataStreamStat) {
this.rawName = dataStreamStat.rawName;
@ -49,6 +50,7 @@ export class DataStreamStat {
this.docsInTimeRange = dataStreamStat.docsInTimeRange;
this.degradedDocs = dataStreamStat.degradedDocs;
this.failedDocs = dataStreamStat.failedDocs;
this.hasFailureStore = dataStreamStat.hasFailureStore;
}
public static create(dataStreamStat: DataStreamStatType) {
@ -56,6 +58,7 @@ export class DataStreamStat {
const dataStreamStatProps = {
rawName: dataStreamStat.name,
hasFailureStore: dataStreamStat.hasFailureStore,
type,
name: dataset,
title: dataset,
@ -79,17 +82,20 @@ export class DataStreamStat {
failedDocStat,
datasetIntegrationMap,
totalDocs,
hasFailureStore,
}: {
datasetName: string;
degradedDocStat: QualityStat;
failedDocStat: QualityStat;
datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
totalDocs: number;
hasFailureStore?: boolean;
}) {
const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName);
const dataStreamStatProps = {
rawName: datasetName,
hasFailureStore,
type,
name: dataset,
title: datasetIntegrationMap[dataset]?.title || dataset,

86
x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/columns.tsx
View file

@ -355,29 +355,69 @@ export const getDatasetQualityTableColumns = ({
),
field: 'failedDocs.percentage',
sortable: true,
render: (_: any, dataStreamStat: DataStreamStat) => (
<PrivilegesWarningIconWrapper
title={`sizeBytes-${dataStreamStat.title}`}
hasPrivileges={dataStreamStat.userPrivileges?.canReadFailureStore ?? true}
>
<QualityStatPercentageLink
isLoading={loadingFailedStats}
dataStreamStat={dataStreamStat}
timeRange={timeRange}
accessor="failedDocs"
selector={FAILURE_STORE_SELECTOR}
fewDocStatsTooltip={(failedDocsCount: number) =>
i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', {
defaultMessage: '{failedDocsCount} failed docs in this data set.',
values: {
failedDocsCount,
},
})
}
dataTestSubj="datasetQualityFailedDocsPercentageLink"
/>
</PrivilegesWarningIconWrapper>
),
render: (_: any, dataStreamStat: DataStreamStat) => {
if (!dataStreamStat.hasFailureStore) {
const FailureStoreHoverLink = () => {
const [hovered, setHovered] = React.useState(false);
const locator = urlService.locators.get('INDEX_MANAGEMENT_LOCATOR_ID');
const params = {
page: 'data_streams_details',
dataStreamName: dataStreamStat.rawName,
} as const;
return (
<EuiToolTip
content={i18n.translate('xpack.datasetQuality.failureStore.notEnabled', {
defaultMessage:
'Failure store is not enabled for this data stream. Enable failure store.',
})}
>
<EuiLink
href={locator?.getRedirectUrl(params)}
target="_blank"
external={false}
data-test-subj="datasetQualitySetFailureStoreLink"
onMouseEnter={() => setHovered(true)}
onMouseLeave={() => setHovered(false)}
css={{ fontWeight: 'normal' }}
>
{hovered
? i18n.translate('xpack.datasetQuality.failureStore.enable', {
defaultMessage: 'Set failure store',
})
: i18n.translate('xpack.datasetQuality.failureStore.notAvailable', {
defaultMessage: 'N/A',
})}
</EuiLink>
</EuiToolTip>
);
};
return <FailureStoreHoverLink />;
}
return (
<PrivilegesWarningIconWrapper
title={`sizeBytes-${dataStreamStat.title}`}
hasPrivileges={dataStreamStat.userPrivileges?.canReadFailureStore ?? true}
>
<QualityStatPercentageLink
isLoading={loadingFailedStats}
dataStreamStat={dataStreamStat}
timeRange={timeRange}
accessor="failedDocs"
selector={FAILURE_STORE_SELECTOR}
fewDocStatsTooltip={(failedDocsCount: number) =>
i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', {
defaultMessage: '{failedDocsCount} failed docs in this data set.',
values: {
failedDocsCount,
},
})
}
dataTestSubj="datasetQualityFailedDocsPercentageLink"
/>
</PrivilegesWarningIconWrapper>
);
},
width: '140px',
},
]

2
x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality/table/table.tsx
View file

@ -75,7 +75,7 @@ export const Table = () => {
</EuiFlexGroup>
</EuiFlexGroup>
<EuiSpacer size="s" />
<EuiHorizontalRule margin="none" style={{ height: 2 }} />
<EuiHorizontalRule margin="none" css={{ height: 2 }} />
<EuiBasicTable
tableLayout="auto"
sorting={sort}

6
x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/document_trends/index.tsx
View file

@ -60,7 +60,7 @@ const degradedDocsTooltip = (
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: number }) {
const { timeRange, updateTimeRange, docsTrendChart, canUserReadFailureStore } =
const { timeRange, updateTimeRange, docsTrendChart, canShowFailureStoreInfo } =
useDatasetQualityDetailsState();
const {
dataView,
@ -81,7 +81,7 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num
[updateTimeRange, timeRange.refresh]
);
const accordionTitle = !canUserReadFailureStore ? (
const accordionTitle = !canShowFailureStoreInfo ? (
<EuiFlexItem
css={css`
flex-direction: row;
@ -127,7 +127,7 @@ export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: num
<EuiSpacer size="m" />
<EuiFlexGroup justifyContent="flexEnd" gutterSize="s">
<EuiFlexItem>
{canUserReadFailureStore && (
{canShowFailureStoreInfo && (
<EuiButtonGroup
data-test-subj="datasetQualityDetailsChartTypeButtonGroup"
legend={i18n.translate('xpack.datasetQuality.details.chartTypeLegend', {

40
x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/index.tsx
View file

@ -7,11 +7,13 @@
import React, { useCallback, useState } from 'react';
import { dynamic } from '@kbn/shared-ux-utility';
import { EuiFlexItem, EuiSpacer, OnRefreshProps } from '@elastic/eui';
import { EuiCallOut, EuiFlexItem, EuiLink, EuiSpacer, OnRefreshProps } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { useDatasetQualityDetailsState } from '../../../hooks';
import { AggregationNotSupported } from './aggregation_not_supported';
import { QualityIssues } from './quality_issues';
import { FailureStoreWarning } from '../../failure_store/failure_store_warning';
import { useKibanaContextForPlugin } from '../../../utils/use_kibana';
const OverviewHeader = dynamic(() => import('./header'));
const Summary = dynamic(() => import('./summary'));
@ -22,9 +24,20 @@ export function Overview() {
dataStream,
isNonAggregatable,
canUserReadFailureStore,
hasFailureStore,
updateTimeRange,
loadingState: { dataStreamSettingsLoading },
} = useDatasetQualityDetailsState();
const {
services: {
share: { url: urlService },
},
} = useKibanaContextForPlugin();
const locator = urlService.locators.get('INDEX_MANAGEMENT_LOCATOR_ID');
const locatorParams = { page: 'data_streams_details', dataStreamName: dataStream } as const;
const [lastReloadTime, setLastReloadTime] = useState<number>(Date.now());
const handleRefresh = useCallback(
@ -39,6 +52,31 @@ export function Overview() {
{isNonAggregatable && <AggregationNotSupported dataStream={dataStream} />}
<OverviewHeader handleRefresh={handleRefresh} />
<EuiSpacer size="m" />
{!dataStreamSettingsLoading && !hasFailureStore && canUserReadFailureStore && (
<div style={{ marginBottom: 16 }}>
<EuiCallOut
color="warning"
title={
<>
{i18n.translate('xpack.datasetQuality.noFailureStoreTitle', {
defaultMessage: 'Failure store is not enabled for this data stream. ',
})}
<EuiLink
href={locator?.getRedirectUrl(locatorParams)}
target="_blank"
external={false}
css={{ textDecoration: 'underline' }}
>
{i18n.translate('xpack.datasetQuality.enableFailureStore', {
defaultMessage: 'Enable failure store',
})}
</EuiLink>
</>
}
/>
</div>
)}
{!dataStreamSettingsLoading && !canUserReadFailureStore && (
<EuiFlexItem>
<FailureStoreWarning />

4
x-pack/platform/plugins/shared/dataset_quality/public/components/dataset_quality_details/overview/summary/index.tsx
View file

@ -48,7 +48,7 @@ const failedDocsColumnTooltip = (
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function Summary() {
const { canUserReadFailureStore } = useDatasetQualityDetailsState();
const { canShowFailureStoreInfo } = useDatasetQualityDetailsState();
const {
isSummaryPanelLoading,
totalDocsCount,
@ -103,7 +103,7 @@ export default function Summary() {
isLoading={isSummaryPanelLoading}
tooltip={degradedDocsTooltip}
/>
{canUserReadFailureStore && (
{canShowFailureStoreInfo && (
<PanelIndicator
label={overviewPanelDatasetQualityIndicatorFailedDocs}
value={totalFailedDocsCount}

5
x-pack/platform/plugins/shared/dataset_quality/public/hooks/use_dataset_quality_details_state.ts
View file

@ -160,6 +160,9 @@ export const useDatasetQualityDetailsState = () => {
[service]
);
const hasFailureStore = Boolean(dataStreamDetails?.hasFailureStore);
const canShowFailureStoreInfo = canUserReadFailureStore && hasFailureStore;
return {
service,
telemetryClient,
@ -182,6 +185,8 @@ export const useDatasetQualityDetailsState = () => {
canUserAccessDashboards,
canUserViewIntegrations,
canUserReadFailureStore,
hasFailureStore,
canShowFailureStoreInfo,
expandedQualityIssue,
isQualityIssueFlyoutOpen,
};

15
x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.test.ts
View file

@ -36,6 +36,7 @@ describe('generateDatasets', () => {
const dataStreamStats: DataStreamStatType[] = [
{
hasFailureStore: true,
name: 'logs-system.application-default',
lastActivity: 1712911241117,
size: '82.1kb',
@ -48,6 +49,7 @@ describe('generateDatasets', () => {
},
},
{
hasFailureStore: false,
name: 'logs-synth-default',
lastActivity: 1712911241117,
size: '62.5kb',
@ -123,6 +125,7 @@ describe('generateDatasets', () => {
percentage: 1.9607843137254901,
count: 2,
},
hasFailureStore: true,
},
{
name: 'synth',
@ -149,6 +152,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
]);
});
@ -188,6 +192,7 @@ describe('generateDatasets', () => {
percentage: 100,
count: 2,
},
hasFailureStore: true,
},
{
name: 'synth',
@ -214,6 +219,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
]);
});
@ -244,6 +250,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
{
name: 'synth',
@ -267,6 +274,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
]);
});
@ -300,6 +308,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
{
name: 'synth',
@ -323,6 +332,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
{
name: 'another',
@ -346,6 +356,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
]);
});
@ -379,6 +390,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: true,
},
{
name: 'synth',
@ -405,12 +417,14 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
]);
});
it('merges integration information with dataStreamStats when dataset is not an integration default one', () => {
const nonDefaultDataset = {
hasFailureStore: false,
name: 'logs-system.custom-default',
lastActivity: 1712911241117,
size: '82.1kb',
@ -451,6 +465,7 @@ describe('generateDatasets', () => {
percentage: 0,
count: 0,
},
hasFailureStore: false,
},
]);
});

9
x-pack/platform/plugins/shared/dataset_quality/public/utils/generate_datasets.ts
View file

@ -74,6 +74,10 @@ export function generateDatasets(
{}
);
const datasetsWithFailureStore = new Set(
dataStreamStats.filter(({ hasFailureStore }) => hasFailureStore).map(({ name }) => name)
);
const degradedMap: Record<
DataStreamDocsStat['dataset'],
{
@ -110,12 +114,15 @@ export function generateDatasets(
failedDocStat: failedMap[dataset] || DEFAULT_QUALITY_DOC_STATS,
datasetIntegrationMap,
totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0),
hasFailureStore: datasetsWithFailureStore.has(dataset),
})
);
}
return dataStreamStats?.map((dataStream) => {
const dataset = DataStreamStat.create(dataStream);
const dataset = DataStreamStat.create({
...dataStream,
});
const degradedDocs = degradedMap[dataset.rawName] || dataset.degradedDocs;
const failedDocs = failedMap[dataset.rawName] || dataset.failedDocs;
const qualityStats = [degradedDocs.percentage, failedDocs.percentage];

1
x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts
View file

@ -86,6 +86,7 @@ export async function getDataStreamDetails({
...dataStreamSummaryStats,
failedDocsCount: failedDocs?.count,
sizeBytes,
hasFailureStore: esDataStream?.hasFailureStore,
lastActivity: esDataStream?.lastActivity,
userPrivileges: {
canMonitor: dataStreamPrivileges.monitor,

1
x-pack/platform/plugins/shared/dataset_quality/server/routes/data_streams/get_data_streams/index.ts
View file

@ -68,6 +68,7 @@ export async function getDataStreams(options: {
canMonitor: dataStreamsPrivileges[dataStream.name].monitor,
canReadFailureStore: dataStreamsPrivileges[dataStream.name][FAILURE_STORE_PRIVILEGE],
},
hasFailureStore: dataStream.failure_store?.enabled,
}));
return {

86
x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/data_streams_failure_store.ts Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
const synthtrace = getService('synthtrace');
const roleScopedSupertest = getService('roleScopedSupertest');
const es = getService('es');
const start = '2025-01-01T00:00:00.000Z';
const end = '2025-01-01T00:01:00.000Z';
const enabledDs = 'logs-synth.fs-default';
const disabledDs = 'logs-synth.no-default';
async function callDetails(supertestApi: any, ds: string) {
return supertestApi
.get(`/internal/dataset_quality/data_streams/${encodeURIComponent(ds)}/details`)
.query({ start, end });
}
describe('Failure-store flag on data-streams', () => {
let client: LogsSynthtraceEsClient;
let supertestAdmin: any;
before(async () => {
client = await synthtrace.createLogsSynthtraceEsClient();
await client.createComponentTemplate({
name: 'logs-failure-enabled@mappings',
dataStreamOptions: { failure_store: { enabled: true } },
});
await es.indices.putIndexTemplate({
name: enabledDs,
index_patterns: [enabledDs],
composed_of: [
'logs-failure-enabled@mappings',
'logs@mappings',
'logs@settings',
'ecs@mappings',
],
priority: 500,
allow_auto_create: true,
data_stream: { hidden: false },
});
await client.index([
timerange(start, end)
.interval('1m')
.rate(1)
.generator((ts) => log.create().timestamp(ts).dataset('synth.fs')),
timerange(start, end)
.interval('1m')
.rate(1)
.generator((ts) => log.create().timestamp(ts).dataset('synth.no')),
]);
await client.refresh();
supertestAdmin = await roleScopedSupertest.getSupertestWithRoleScope('admin', {
useCookieHeader: true,
withInternalHeaders: true,
});
});
after(async () => {
await es.indices.deleteIndexTemplate({ name: enabledDs });
await client.deleteComponentTemplate('logs-failure-enabled@mappings');
await client.clean();
});
it('details API reports correct hasFailureStore flag', async () => {
const enabled = await callDetails(supertestAdmin, enabledDs);
const disabled = await callDetails(supertestAdmin, disabledDs);
expect(enabled.body.hasFailureStore).to.be(true);
expect(disabled.body.hasFailureStore).to.be(false);
});
});
}

1
x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/index.ts
View file

@ -25,5 +25,6 @@ export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext)
loadTestFile(require.resolve('./integration_dashboards'));
loadTestFile(require.resolve('./integrations'));
loadTestFile(require.resolve('./update_field_limit'));
loadTestFile(require.resolve('./data_streams_failure_store'));
});
}

21
x-pack/test/functional/apps/dataset_quality/dataset_quality_table.ts
View file

@ -28,6 +28,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
'datasetQuality',
]);
const retry = getService('retry');
const testSubjects = getService('testSubjects');
const synthtrace = getService('logSynthtraceEsClient');
const to = '2024-01-01T12:00:00.000Z';
const apacheAccessDatasetName = 'apache.access';
@ -203,7 +204,25 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
const failedDocsCol = cols[PageObjects.datasetQuality.texts.datasetFailedDocsColumn];
const failedDocsColCellTexts = await failedDocsCol.getCellTexts();
expect(failedDocsColCellTexts).to.eql(['0%', '0%', '20%', '0%']);
expect(failedDocsColCellTexts).to.eql(['N/A', 'N/A', '20%', 'N/A']);
});
it('changes link text on hover when failure store is not enabled', async () => {
const linkSelector = 'datasetQualitySetFailureStoreLink';
const links = await testSubjects.findAll(linkSelector);
expect(links.length).to.be.greaterThan(0);
const link = links[links.length - 1];
expect(await link.getVisibleText()).to.eql('N/A');
await link.moveMouseTo();
await retry.try(async () => {
expect(await link.getVisibleText()).to.eql('Set failure store');
});
const table = await PageObjects.datasetQuality.getDatasetsTable();
await table.moveMouseTo();
});
});
});

25
x-pack/test/functional/page_objects/dataset_quality.ts
View file

@ -411,6 +411,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
},
async toggleShowFullDatasetNames() {
await find.waitForDeletedByCssSelector('.euiToolTipPopover', 5 * 1000);
return find.clickByCssSelector(selectors.showFullDatasetNamesSwitch);
},
@ -473,12 +474,16 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
].filter((item) => !excludeKeys.includes(item.key));
const kpiTexts = await Promise.all(
kpiTitleAndKeys.map(async ({ title, key }) => ({
key,
value: await testSubjects.getVisibleText(
`${testSubjectSelectors.datasetQualityDetailsSummaryKpiValue}-${title}`
),
}))
kpiTitleAndKeys.map(async ({ title, key }) => {
const selector = `${testSubjectSelectors.datasetQualityDetailsSummaryKpiValue}-${title}`;
const exists = await testSubjects.exists(selector);
if (!exists) {
return { key, value: undefined } as { key: string; value: string | undefined };
}
return { key, value: await testSubjects.getVisibleText(selector) };
})
);
return kpiTexts.reduce(
@ -490,10 +495,6 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
);
},
/**
* Selects a breakdown field from the unified histogram breakdown selector
* @param fieldText The text of the field to select. Use 'No breakdown' to clear the selection
*/
async selectBreakdownField(fieldText: string) {
return euiSelectable.searchAndSelectOption(
testSubjectSelectors.unifiedHistogramBreakdownSelectorButton,
@ -532,10 +533,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
const fieldExpandButton = expandButtons[testDatasetRowIndex];
// Check if 'title' attribute is "Expand" or "Collapse"
const isCollapsed = (await fieldExpandButton.getAttribute('title')) === 'Expand';
// Open if collapsed
if (isCollapsed) {
await fieldExpandButton.click();
}
@ -563,10 +562,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
const fieldExpandButton = expandButtons[testDatasetRowIndex];
// Check if 'title' attribute is "Expand" or "Collapse"
const isCollapsed = (await fieldExpandButton.getAttribute('title')) === 'Expand';
// Open if collapsed
if (isCollapsed) {
await fieldExpandButton.click();
}