mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[Dataset quality] Failure store support (#206758)
Closes https://github.com/elastic/logs-dev/issues/183, https://github.com/elastic/logs-dev/issues/184 and https://github.com/elastic/logs-dev/issues/185. ## Summary This PR aims to support failure store in dataset quality page. The following acceptance criteria items were resolved ### Dataset quality page - [x] A column for Failed docs is included in the table - [x] A tooltip is placed in the title of the column - [x] A % of documents inside Failure store is calculated for every dataStream - [x] If % is lesser than 0.0001 but greater than 0 we should show ⚠ symbol next to the ~0 value (as we do with degraded docs) - [x] Failed docs percentages greater than 0 should link to discover 🎥 Demo https://github.com/user-attachments/assets/6d9e3f4c-02d9-43ab-88cb-ae70716b05d9 ### Dataset details page - [x] A metric, Failed docs, is included in the Overview panel under Data set quality. This metric includes the number of documents inside the failure store for the specific dataStream. - [x] A tooltip is placed in the title of the Failed docs metric with message: `The percentage of docs sent to failure store due to an issue during ingestion.` - [x] Degraded docs graph section is transformed to Document trends allowing the users to switch between Degraded docs and Failed docs trends over time. - [x] A new chart for failed documents is created with links to discover/Logs explorer using the right dataView 🎥 Demo https://github.com/user-attachments/assets/6a3a1f09-2668-4e83-938e-ecdda798c199 ### Failed docs ingestion issue flyout - [x] Whenever documents are found in failure store we should list Document indexing failed in Quality issues table - [x] User should be able to expand Document indexing failed and see more information in the flyout - [x] The flyout will show Docs count, an aggregation of the number of documents inside failure store for the selected timeframe - [x] The flyout will show Last ocurrence, the datetime registered for the most recent document in the failure store. - [x] The flyout will contain a section called Error messages where a list of unique error messages should be shown, exposing Content (error message) and Type (Error Type). - [x] Type should contain a tooltip where message (`Error message category`) explain users how we are categorising the errors. - [x] Other issues inside Quality issues table will be appended by field ignored and the field will be shown in bold. https://github.com/user-attachments/assets/94dc81f0-9720-4596-b256-c9d289cefd94 Note: This PR was reconstructed from https://github.com/elastic/kibana/pull/199806 which it supersedes. ## How to test 1. Execute `failed_logs` synthtrace scenario 2. Open dataset quality page ## Follow ups - Enable in serverless - Deployment agnostic tests cannot be added until we enable this in serverless - FTR tests will be added as part of https://github.com/elastic/logs-dev/issues/182 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Yngrid Coello
GitHub
parent
5e8188a960
commit
511f77c231
107 changed files with 3372 additions and 1177 deletions
|
|
@ -52,7 +52,19 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
|
|||
}
|
||||
}
|
||||
|
||||
async createComponentTemplate(name: string, mappings: MappingTypeMapping) {
|
||||
async createComponentTemplate({
|
||||
name,
|
||||
mappings,
|
||||
dataStreamOptions,
|
||||
}: {
|
||||
name: string;
|
||||
mappings?: MappingTypeMapping;
|
||||
dataStreamOptions?: {
|
||||
failure_store: {
|
||||
enabled: boolean;
|
||||
};
|
||||
};
|
||||
}) {
|
||||
const isTemplateExisting = await this.client.cluster.existsComponentTemplate({ name });
|
||||
|
||||
if (isTemplateExisting) return this.logger.info(`Component template already exists: ${name}`);
|
||||
|
|
@ -61,7 +73,8 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
|
|||
await this.client.cluster.putComponentTemplate({
|
||||
name,
|
||||
template: {
|
||||
mappings,
|
||||
...((mappings && { mappings }) || {}),
|
||||
...((dataStreamOptions && { data_stream_options: dataStreamOptions }) || {}),
|
||||
},
|
||||
});
|
||||
this.logger.info(`Component template successfully created: ${name}`);
|
||||
|
|
@ -124,16 +137,16 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient<LogDocument> {
|
|||
}
|
||||
}
|
||||
|
||||
async createCustomPipeline(processors: IngestProcessorContainer[]) {
|
||||
async createCustomPipeline(processors: IngestProcessorContainer[], id = LogsCustom) {
|
||||
try {
|
||||
this.client.ingest.putPipeline({
|
||||
id: LogsCustom,
|
||||
id,
|
||||
processors,
|
||||
version: 1,
|
||||
});
|
||||
this.logger.info(`Custom pipeline created: ${LogsCustom}`);
|
||||
this.logger.info(`Custom pipeline created: ${id}`);
|
||||
} catch (err) {
|
||||
this.logger.error(`Custom pipeline creation failed: ${LogsCustom} - ${err.message}`);
|
||||
this.logger.error(`Custom pipeline creation failed: ${id} - ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,20 +7,19 @@
|
|||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import { LogDocument, log, generateShortId, generateLongId } from '@kbn/apm-synthtrace-client';
|
||||
import { merge } from 'lodash';
|
||||
import { LogDocument, generateLongId, generateShortId, log } from '@kbn/apm-synthtrace-client';
|
||||
import { Scenario } from '../cli/scenario';
|
||||
import { IndexTemplateName } from '../lib/logs/custom_logsdb_index_templates';
|
||||
import { LogsCustom } from '../lib/logs/logs_synthtrace_es_client';
|
||||
import { withClient } from '../lib/utils/with_client';
|
||||
import {
|
||||
getServiceName,
|
||||
getCluster,
|
||||
getCloudRegion,
|
||||
getCloudProvider,
|
||||
MORE_THAN_1024_CHARS,
|
||||
getCloudProvider,
|
||||
getCloudRegion,
|
||||
getCluster,
|
||||
getServiceName,
|
||||
} from './helpers/logs_mock_data';
|
||||
import { parseLogsScenarioOpts } from './helpers/logs_scenario_opts_parser';
|
||||
import { LogsIndex } from '../lib/logs/logs_synthtrace_es_client';
|
||||
|
||||
const processors = [
|
||||
{
|
||||
|
|
@ -61,24 +60,20 @@ const MESSAGE_LOG_LEVELS = [
|
|||
|
||||
const scenario: Scenario<LogDocument> = async (runOptions) => {
|
||||
const { isLogsDb } = parseLogsScenarioOpts(runOptions.scenarioOpts);
|
||||
|
||||
return {
|
||||
bootstrap: async ({ logsEsClient }) => {
|
||||
await logsEsClient.createCustomPipeline(processors);
|
||||
if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb);
|
||||
|
||||
await logsEsClient.updateIndexTemplate(
|
||||
isLogsDb ? IndexTemplateName.LogsDb : LogsIndex,
|
||||
(template) => {
|
||||
const next = {
|
||||
name: LogsIndex,
|
||||
data_stream: {
|
||||
failure_store: true,
|
||||
},
|
||||
};
|
||||
|
||||
return merge({}, template, next);
|
||||
}
|
||||
);
|
||||
await logsEsClient.createComponentTemplate({
|
||||
name: LogsCustom,
|
||||
dataStreamOptions: {
|
||||
failure_store: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
generate: ({ range, clients: { logsEsClient } }) => {
|
||||
const { logger } = runOptions;
|
||||
|
|
|
|||
|
|
@ -14983,7 +14983,6 @@
|
|||
"xpack.datasetQuality.degradedDocsColumnName": "Documents dégradés (%)",
|
||||
"xpack.datasetQuality.degradedDocsColumnTooltip": "Le pourcentage de documents avec la propriété {ignoredProperty} dans votre ensemble de données.",
|
||||
"xpack.datasetQuality.degradedDocsQualityDescription": "{quality} -{comparator} {minimimPercentage}%",
|
||||
"xpack.datasetQuality.detail.degradedFieldsSectionTitle": "Problèmes de qualité",
|
||||
"xpack.datasetQuality.details.chartOpenInLensText": "Ouvrir dans Lens",
|
||||
"xpack.datasetQuality.details.checkBreakdownFieldEcsFailed": "Nous n'avons pas pu récupérer les métadonnées du champ de répartition.",
|
||||
"xpack.datasetQuality.details.datasetCreatedOnText": "Créé le",
|
||||
|
|
@ -14996,22 +14995,13 @@
|
|||
"xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip": "Le nombre de champs dans cet index a dépassé la limite maximale autorisée.",
|
||||
"xpack.datasetQuality.details.degradedField.cause.fieldMalformed": "champ mal formé",
|
||||
"xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip": "Le type de données du champ n'est pas défini correctement.",
|
||||
"xpack.datasetQuality.details.degradedField.count": "Nombre de documents",
|
||||
"xpack.datasetQuality.details.degradedField.currentFieldLimit": "Limite de champ",
|
||||
"xpack.datasetQuality.details.degradedField.field": "Champ",
|
||||
"xpack.datasetQuality.details.degradedField.fieldIgnored": "champ ignoré",
|
||||
"xpack.datasetQuality.details.degradedField.lastOccurrence": "Dernière occurrence",
|
||||
"xpack.datasetQuality.details.degradedField.maximumCharacterLimit": "Longueur maximale des caractères",
|
||||
"xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex": "Ce problème a été détecté dans une ancienne version de l'ensemble de données, mais pas dans la version la plus récente.",
|
||||
"xpack.datasetQuality.details.degradedField.potentialCause": "Cause potentielle",
|
||||
"xpack.datasetQuality.details.degradedField.values": "Valeurs",
|
||||
"xpack.datasetQuality.details.degradedFieldsSectionTooltip": "Une liste partielle des problèmes de qualité détectés dans votre ensemble de données.",
|
||||
"xpack.datasetQuality.details.degradedFieldsTableLoadingText": "Chargement des champs dégradés",
|
||||
"xpack.datasetQuality.details.degradedFieldsTableNoData": "Aucun champ dégradé n’a été trouvé",
|
||||
"xpack.datasetQuality.details.degradedFieldTable.collapseLabel": "Réduire",
|
||||
"xpack.datasetQuality.details.degradedFieldTable.expandLabel": "Développer",
|
||||
"xpack.datasetQuality.details.degradedFieldToggleSwitch": "Problèmes de qualité actuels uniquement",
|
||||
"xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip": "Activez cette option pour afficher uniquement les problèmes détectés dans la version la plus récente de l'ensemble de données. Désactivez cette option pour afficher tous les problèmes détectés dans la plage temporelle configurée.",
|
||||
"xpack.datasetQuality.details.detailsTitle": "Détails",
|
||||
"xpack.datasetQuality.details.discoverAriaText": "Discover",
|
||||
"xpack.datasetQuality.details.emptyPromptBody": "Flux de données introuvable : {dataStream}",
|
||||
|
|
@ -15045,7 +15035,6 @@
|
|||
"xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "Nous n'avons pas pu obtenir d'informations sur les ensembles de données non agrégés.",
|
||||
"xpack.datasetQuality.fewDegradedDocsTooltip": "{degradedDocsCount} documents dégradés dans cet ensemble de données.",
|
||||
"xpack.datasetQuality.filterBar.placeholder": "Filtrer les ensembles de données",
|
||||
"xpack.datasetQuality.flyout.degradedDocsTitle": "Documents dégradés",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.description": "{description}",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "{rolloverLink} manuellement cet ensemble de données pour empêcher des délais à l'avenir.",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset} est incompatible avec l'agrégation _ignored, ce qui peut entraîner des délais lors de la recherche de données. {howToFixIt}",
|
||||
|
|
|
|||
|
|
@ -14848,7 +14848,6 @@
|
|||
"xpack.datasetQuality.degradedDocsColumnName": "劣化したドキュメント（%）",
|
||||
"xpack.datasetQuality.degradedDocsColumnTooltip": "データセットにおける{ignoredProperty}プロパティのドキュメントの割合。",
|
||||
"xpack.datasetQuality.degradedDocsQualityDescription": "{quality} -{comparator} {minimimPercentage}%",
|
||||
"xpack.datasetQuality.detail.degradedFieldsSectionTitle": "品質の問題",
|
||||
"xpack.datasetQuality.details.chartOpenInLensText": "Lensで開く",
|
||||
"xpack.datasetQuality.details.checkBreakdownFieldEcsFailed": "内訳フィールドメタデータを取得できませんでした。",
|
||||
"xpack.datasetQuality.details.datasetCreatedOnText": "作成日時",
|
||||
|
|
@ -14861,22 +14860,13 @@
|
|||
"xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip": "このインデックスのフィールド数が許可された最大上限を超えています。",
|
||||
"xpack.datasetQuality.details.degradedField.cause.fieldMalformed": "フィールドの形式が正しくありません",
|
||||
"xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip": "フィールドのデータ型が正しく設定されていません。",
|
||||
"xpack.datasetQuality.details.degradedField.count": "ドキュメント数",
|
||||
"xpack.datasetQuality.details.degradedField.currentFieldLimit": "フィールド上限",
|
||||
"xpack.datasetQuality.details.degradedField.field": "フィールド",
|
||||
"xpack.datasetQuality.details.degradedField.fieldIgnored": "無視されたフィールド",
|
||||
"xpack.datasetQuality.details.degradedField.lastOccurrence": "前回の発生",
|
||||
"xpack.datasetQuality.details.degradedField.maximumCharacterLimit": "最大文字長",
|
||||
"xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex": "この問題は、古いバージョンのデータセットでは検出されてましたが、最新のバージョンでは検出されませんでした。",
|
||||
"xpack.datasetQuality.details.degradedField.potentialCause": "潜在的な原因",
|
||||
"xpack.datasetQuality.details.degradedField.values": "値",
|
||||
"xpack.datasetQuality.details.degradedFieldsSectionTooltip": "データセットで見つかった品質の問題の部分的なリスト。",
|
||||
"xpack.datasetQuality.details.degradedFieldsTableLoadingText": "劣化したフィールドを読み込み中",
|
||||
"xpack.datasetQuality.details.degradedFieldsTableNoData": "劣化したフィールドが見つかりません",
|
||||
"xpack.datasetQuality.details.degradedFieldTable.collapseLabel": "縮小",
|
||||
"xpack.datasetQuality.details.degradedFieldTable.expandLabel": "拡張",
|
||||
"xpack.datasetQuality.details.degradedFieldToggleSwitch": "現在の品質の問題のみ",
|
||||
"xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip": "有効にすると、データセットの最新バージョンで検出された問題のみが表示されます。無効にすると、設定した時間範囲内で検出されたすべての問題が表示されます。",
|
||||
"xpack.datasetQuality.details.detailsTitle": "詳細",
|
||||
"xpack.datasetQuality.details.discoverAriaText": "Discover",
|
||||
"xpack.datasetQuality.details.emptyPromptBody": "データストリームが見つかりません：{dataStream}",
|
||||
|
|
@ -14910,7 +14900,6 @@
|
|||
"xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "集約可能なデータセット情報以外を取得できませんでした。",
|
||||
"xpack.datasetQuality.fewDegradedDocsTooltip": "このデータセットの{degradedDocsCount}個の劣化したドキュメント。",
|
||||
"xpack.datasetQuality.filterBar.placeholder": "データセットのフィルタリング",
|
||||
"xpack.datasetQuality.flyout.degradedDocsTitle": "劣化したドキュメント",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.description": "{description}",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "今後の遅れを防止するには、手動でこのデータを{rolloverLink}してください。",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset}は_ignored集約をサポートしていません。データのクエリを実行するときに遅延が生じる可能性があります。{howToFixIt}",
|
||||
|
|
|
|||
|
|
@ -14581,7 +14581,6 @@
|
|||
"xpack.datasetQuality.degradedDocsColumnName": "已降级文档 (%)",
|
||||
"xpack.datasetQuality.degradedDocsColumnTooltip": "您的数据集中包含 {ignoredProperty} 属性的文档的百分比。",
|
||||
"xpack.datasetQuality.degradedDocsQualityDescription": "{quality} -{comparator} {minimimPercentage}%",
|
||||
"xpack.datasetQuality.detail.degradedFieldsSectionTitle": "质量问题",
|
||||
"xpack.datasetQuality.details.chartOpenInLensText": "在 Lens 中打开",
|
||||
"xpack.datasetQuality.details.checkBreakdownFieldEcsFailed": "无法检索细目字段元数据。",
|
||||
"xpack.datasetQuality.details.datasetCreatedOnText": "创建日期",
|
||||
|
|
@ -14594,22 +14593,13 @@
|
|||
"xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip": "此索引中的字段数已超出允许的最大限制。",
|
||||
"xpack.datasetQuality.details.degradedField.cause.fieldMalformed": "字段格式不正确",
|
||||
"xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip": "未正确设置此字段的数据类型。",
|
||||
"xpack.datasetQuality.details.degradedField.count": "文档计数",
|
||||
"xpack.datasetQuality.details.degradedField.currentFieldLimit": "字段限制",
|
||||
"xpack.datasetQuality.details.degradedField.field": "字段",
|
||||
"xpack.datasetQuality.details.degradedField.fieldIgnored": "字段已忽略",
|
||||
"xpack.datasetQuality.details.degradedField.lastOccurrence": "最后一次发生",
|
||||
"xpack.datasetQuality.details.degradedField.maximumCharacterLimit": "最大字符长度",
|
||||
"xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex": "在较早版本而不是最新版本的数据集中检测到此问题。",
|
||||
"xpack.datasetQuality.details.degradedField.potentialCause": "潜在原因",
|
||||
"xpack.datasetQuality.details.degradedField.values": "值",
|
||||
"xpack.datasetQuality.details.degradedFieldsSectionTooltip": "在数据集中发现的质量问题的部分列表。",
|
||||
"xpack.datasetQuality.details.degradedFieldsTableLoadingText": "正在加载已降级字段",
|
||||
"xpack.datasetQuality.details.degradedFieldsTableNoData": "找不到已降级字段",
|
||||
"xpack.datasetQuality.details.degradedFieldTable.collapseLabel": "折叠",
|
||||
"xpack.datasetQuality.details.degradedFieldTable.expandLabel": "展开",
|
||||
"xpack.datasetQuality.details.degradedFieldToggleSwitch": "仅限当前的质量问题",
|
||||
"xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip": "启用以仅显示在最新版本的数据集中检测到的问题。禁用以显示在配置的时间范围内检测到的所有问题。",
|
||||
"xpack.datasetQuality.details.detailsTitle": "详情",
|
||||
"xpack.datasetQuality.details.discoverAriaText": "Discover",
|
||||
"xpack.datasetQuality.details.emptyPromptBody": "找不到数据流：{dataStream}",
|
||||
|
|
@ -14643,7 +14633,6 @@
|
|||
"xpack.datasetQuality.fetchNonAggregatableDatasetsFailed": "无法获取非可聚合数据集信息。",
|
||||
"xpack.datasetQuality.fewDegradedDocsTooltip": "此数据集中的 {degradedDocsCount} 个已降级文档。",
|
||||
"xpack.datasetQuality.filterBar.placeholder": "筛选数据集",
|
||||
"xpack.datasetQuality.flyout.degradedDocsTitle": "已降级文档",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.description": "{description}",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.howToFixIt": "手动 {rolloverLink} 此数据集以防止未来出现延迟。",
|
||||
"xpack.datasetQuality.flyout.nonAggregatable.warning": "{dataset} 不支持 _ignored 聚合，在查询数据时可能会导致延迟。{howToFixIt}",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,11 @@ import * as rt from 'io-ts';
|
|||
|
||||
export const DATA_QUALITY_URL_STATE_KEY = 'pageState';
|
||||
|
||||
export const qualityIssuesRT = rt.keyof({
|
||||
degraded: null,
|
||||
failed: null,
|
||||
});
|
||||
|
||||
export const directionRT = rt.keyof({
|
||||
asc: null,
|
||||
desc: null,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import * as rt from 'io-ts';
|
||||
import { dataStreamRT, degradedFieldRT, qualityIssuesRT, timeRangeRT } from './common';
|
||||
|
||||
export const urlSchemaRT = rt.exact(
|
||||
rt.intersection([
|
||||
rt.type({
|
||||
dataStream: dataStreamRT,
|
||||
}),
|
||||
rt.partial({
|
||||
v: rt.literal(2),
|
||||
timeRange: timeRangeRT,
|
||||
qualityIssuesChart: qualityIssuesRT,
|
||||
breakdownField: rt.string,
|
||||
qualityIssues: degradedFieldRT,
|
||||
expandedQualityIssue: rt.type({
|
||||
name: rt.string,
|
||||
type: qualityIssuesRT,
|
||||
}),
|
||||
showCurrentQualityIssues: rt.boolean,
|
||||
}),
|
||||
])
|
||||
);
|
||||
|
||||
export type UrlSchema = rt.TypeOf<typeof urlSchemaRT>;
|
||||
|
|
@ -8,3 +8,4 @@
|
|||
export { DATA_QUALITY_URL_STATE_KEY } from './common';
|
||||
export * as datasetQualityUrlSchemaV1 from './dataset_quality_url_schema_v1';
|
||||
export * as datasetQualityDetailsUrlSchemaV1 from './dataset_quality_details_url_schema_v1';
|
||||
export * as datasetQualityDetailsUrlSchemaV2 from './dataset_quality_details_url_schema_v2';
|
||||
|
|
|
|||
|
|
@ -16,10 +16,16 @@ export const getStateFromUrlValue = (
|
|||
deepCompactObject<DatasetQualityDetailsPublicStateUpdate>({
|
||||
dataStream: urlValue.dataStream,
|
||||
timeRange: urlValue.timeRange,
|
||||
degradedFields: urlValue.degradedFields,
|
||||
qualityIssues: urlValue.degradedFields,
|
||||
breakdownField: urlValue.breakdownField,
|
||||
expandedDegradedField: urlValue.expandedDegradedField,
|
||||
showCurrentQualityIssues: urlValue.showCurrentQualityIssues,
|
||||
qualityIssuesChart: 'degraded',
|
||||
expandedQualityIssue: urlValue.expandedDegradedField
|
||||
? {
|
||||
name: urlValue.expandedDegradedField,
|
||||
type: 'degraded',
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
|
||||
export const getUrlValueFromState = (
|
||||
|
|
@ -28,9 +34,9 @@ export const getUrlValueFromState = (
|
|||
deepCompactObject<datasetQualityDetailsUrlSchemaV1.UrlSchema>({
|
||||
dataStream: state.dataStream,
|
||||
timeRange: state.timeRange,
|
||||
degradedFields: state.degradedFields,
|
||||
degradedFields: state.qualityIssues,
|
||||
breakdownField: state.breakdownField,
|
||||
expandedDegradedField: state.expandedDegradedField,
|
||||
expandedDegradedField: state.expandedQualityIssue?.name,
|
||||
showCurrentQualityIssues: state.showCurrentQualityIssues,
|
||||
v: 1,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DatasetQualityDetailsPublicStateUpdate } from '@kbn/dataset-quality-plugin/public/controller/dataset_quality_details';
|
||||
import * as rt from 'io-ts';
|
||||
import { deepCompactObject } from '../../../common/utils/deep_compact_object';
|
||||
import { datasetQualityDetailsUrlSchemaV2 } from '../../../common/url_schema';
|
||||
|
||||
export const getStateFromUrlValue = (
|
||||
urlValue: datasetQualityDetailsUrlSchemaV2.UrlSchema
|
||||
): DatasetQualityDetailsPublicStateUpdate =>
|
||||
deepCompactObject<DatasetQualityDetailsPublicStateUpdate>({
|
||||
dataStream: urlValue.dataStream,
|
||||
timeRange: urlValue.timeRange,
|
||||
qualityIssues: urlValue.qualityIssues,
|
||||
breakdownField: urlValue.breakdownField,
|
||||
showCurrentQualityIssues: urlValue.showCurrentQualityIssues,
|
||||
qualityIssuesChart: urlValue.qualityIssuesChart,
|
||||
expandedQualityIssue: urlValue.expandedQualityIssue,
|
||||
});
|
||||
|
||||
export const getUrlValueFromState = (
|
||||
state: DatasetQualityDetailsPublicStateUpdate
|
||||
): datasetQualityDetailsUrlSchemaV2.UrlSchema =>
|
||||
deepCompactObject<datasetQualityDetailsUrlSchemaV2.UrlSchema>({
|
||||
dataStream: state.dataStream,
|
||||
timeRange: state.timeRange,
|
||||
qualityIssues: state.qualityIssues,
|
||||
breakdownField: state.breakdownField,
|
||||
showCurrentQualityIssues: state.showCurrentQualityIssues,
|
||||
qualityIssuesChart: state.qualityIssuesChart,
|
||||
expandedQualityIssue: state.expandedQualityIssue,
|
||||
v: 2,
|
||||
});
|
||||
|
||||
const stateFromUrlSchemaRT = new rt.Type<
|
||||
DatasetQualityDetailsPublicStateUpdate,
|
||||
datasetQualityDetailsUrlSchemaV2.UrlSchema,
|
||||
datasetQualityDetailsUrlSchemaV2.UrlSchema
|
||||
>(
|
||||
'stateFromUrlSchemaRT',
|
||||
rt.never.is,
|
||||
(urlSchema, _context) => rt.success(getStateFromUrlValue(urlSchema)),
|
||||
getUrlValueFromState
|
||||
);
|
||||
|
||||
export const stateFromUntrustedUrlRT =
|
||||
datasetQualityDetailsUrlSchemaV2.urlSchemaRT.pipe(stateFromUrlSchemaRT);
|
||||
|
|
@ -14,6 +14,7 @@ import { DatasetQualityDetailsPublicStateUpdate } from '@kbn/dataset-quality-plu
|
|||
import * as rt from 'io-ts';
|
||||
import { DATA_QUALITY_URL_STATE_KEY } from '../../../common/url_schema';
|
||||
import * as urlSchemaV1 from './url_schema_v1';
|
||||
import * as urlSchemaV2 from './url_schema_v2';
|
||||
|
||||
export const updateUrlFromDatasetQualityDetailsState = ({
|
||||
urlStateStorageContainer,
|
||||
|
|
@ -26,7 +27,8 @@ export const updateUrlFromDatasetQualityDetailsState = ({
|
|||
return;
|
||||
}
|
||||
|
||||
const encodedUrlStateValues = urlSchemaV1.stateFromUntrustedUrlRT.encode(
|
||||
// we want to use always the newest schema version
|
||||
const encodedUrlStateValues = urlSchemaV2.stateFromUntrustedUrlRT.encode(
|
||||
datasetQualityDetailsState
|
||||
);
|
||||
|
||||
|
|
@ -50,7 +52,7 @@ export const getDatasetQualityDetailsStateFromUrl = ({
|
|||
urlStateStorageContainer.get<unknown>(DATA_QUALITY_URL_STATE_KEY) ?? undefined;
|
||||
|
||||
const stateValuesE = rt
|
||||
.union([rt.undefined, urlSchemaV1.stateFromUntrustedUrlRT])
|
||||
.union([rt.undefined, urlSchemaV1.stateFromUntrustedUrlRT, urlSchemaV2.stateFromUntrustedUrlRT])
|
||||
.decode(urlStateValues);
|
||||
|
||||
if (Either.isLeft(stateValuesE)) {
|
||||
|
|
|
|||
|
|
@ -56,6 +56,12 @@ export const getDataStreamDegradedDocsResponseRt = rt.type({
|
|||
|
||||
export type DataStreamDegradedDocsResponse = rt.TypeOf<typeof getDataStreamDegradedDocsResponseRt>;
|
||||
|
||||
export const getDataStreamFailedDocsResponseRt = rt.type({
|
||||
failedDocs: rt.array(dataStreamDocsStatRt),
|
||||
});
|
||||
|
||||
export type DataStreamFailedDocsResponse = rt.TypeOf<typeof getDataStreamFailedDocsResponseRt>;
|
||||
|
||||
export const integrationDashboardRT = rt.type({
|
||||
id: rt.string,
|
||||
title: rt.string,
|
||||
|
|
@ -114,19 +120,56 @@ export const getIntegrationsResponseRt = rt.exact(
|
|||
|
||||
export type IntegrationsResponse = rt.TypeOf<typeof getIntegrationsResponseRt>;
|
||||
|
||||
export const degradedFieldRt = rt.type({
|
||||
name: rt.string,
|
||||
export const qualityIssueBaseRT = rt.type({
|
||||
count: rt.number,
|
||||
lastOccurrence: rt.union([rt.null, rt.number]),
|
||||
lastOccurrence: rt.union([rt.undefined, rt.null, rt.number]),
|
||||
timeSeries: rt.array(
|
||||
rt.type({
|
||||
x: rt.number,
|
||||
y: rt.number,
|
||||
})
|
||||
),
|
||||
indexFieldWasLastPresentIn: rt.string,
|
||||
});
|
||||
|
||||
export const qualityIssueRT = rt.intersection([
|
||||
qualityIssueBaseRT,
|
||||
rt.partial({
|
||||
indexFieldWasLastPresentIn: rt.string,
|
||||
}),
|
||||
rt.type({
|
||||
name: rt.string,
|
||||
type: rt.keyof({
|
||||
degraded: null,
|
||||
failed: null,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
export type QualityIssue = rt.TypeOf<typeof qualityIssueRT>;
|
||||
|
||||
export type FailedDocsDetails = rt.TypeOf<typeof qualityIssueBaseRT>;
|
||||
|
||||
export const failedDocsErrorRt = rt.type({
|
||||
message: rt.string,
|
||||
type: rt.string,
|
||||
});
|
||||
|
||||
export type FailedDocsError = rt.TypeOf<typeof failedDocsErrorRt>;
|
||||
|
||||
export const failedDocsErrorsRt = rt.type({
|
||||
errors: rt.array(failedDocsErrorRt),
|
||||
});
|
||||
|
||||
export type FailedDocsErrorsResponse = rt.TypeOf<typeof failedDocsErrorsRt>;
|
||||
|
||||
export const degradedFieldRt = rt.intersection([
|
||||
qualityIssueBaseRT,
|
||||
rt.type({
|
||||
name: rt.string,
|
||||
indexFieldWasLastPresentIn: rt.string,
|
||||
}),
|
||||
]);
|
||||
|
||||
export type DegradedField = rt.TypeOf<typeof degradedFieldRt>;
|
||||
|
||||
export const getDataStreamDegradedFieldsResponseRt = rt.type({
|
||||
|
|
@ -193,6 +236,7 @@ export type DataStreamSettings = rt.TypeOf<typeof dataStreamSettingsRt>;
|
|||
export const dataStreamDetailsRt = rt.partial({
|
||||
lastActivity: rt.number,
|
||||
degradedDocsCount: rt.number,
|
||||
failedDocsCount: rt.number,
|
||||
docsCount: rt.number,
|
||||
sizeBytes: rt.number,
|
||||
services: rt.record(rt.string, rt.array(rt.string)),
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { _IGNORED } from './es_fields';
|
||||
import { DataStreamType, QualityIndicators } from './types';
|
||||
|
||||
export const DATASET_QUALITY_APP_ID = 'dataset_quality';
|
||||
|
|
@ -18,15 +19,18 @@ export const DEGRADED_QUALITY_MINIMUM_PERCENTAGE = 0;
|
|||
export const DEFAULT_SORT_FIELD = 'title';
|
||||
export const DEFAULT_SORT_DIRECTION = 'asc';
|
||||
|
||||
export const DEFAULT_DEGRADED_FIELD_SORT_FIELD = 'count';
|
||||
export const DEFAULT_DEGRADED_FIELD_SORT_DIRECTION = 'desc';
|
||||
export const DEFAULT_QUALITY_ISSUE_SORT_FIELD = 'count';
|
||||
export const DEFAULT_QUALITY_ISSUE_SORT_DIRECTION = 'desc';
|
||||
|
||||
export const DEFAULT_FAILED_DOCS_ERROR_SORT_FIELD = 'type';
|
||||
export const DEFAULT_FAILED_DOCS_ERROR_SORT_DIRECTION = 'desc';
|
||||
|
||||
export const NONE = 'none';
|
||||
|
||||
export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' };
|
||||
export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false };
|
||||
|
||||
export const DEFAULT_DEGRADED_DOCS = {
|
||||
export const DEFAULT_QUALITY_DOC_STATS = {
|
||||
count: 0,
|
||||
percentage: 0,
|
||||
};
|
||||
|
|
@ -42,3 +46,9 @@ export const MASKED_FIELD_PLACEHOLDER = '<custom field>';
|
|||
export const UNKOWN_FIELD_PLACEHOLDER = '<unkwon>';
|
||||
|
||||
export const KNOWN_TYPES: DataStreamType[] = ['logs', 'metrics', 'traces', 'synthetics'];
|
||||
|
||||
export const DEGRADED_DOCS_QUERY = `${_IGNORED}: *`;
|
||||
|
||||
export const ALL_PATTERNS_SELECTOR = '::*';
|
||||
export const FAILURE_STORE_SELECTOR = '::failures';
|
||||
export const DATA_SELECTOR = '::data';
|
||||
|
|
|
|||
|
|
@ -5,13 +5,17 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DataStreamDocsStat } from '../api_types';
|
||||
import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants';
|
||||
import { DEFAULT_DATASET_QUALITY, DEFAULT_QUALITY_DOC_STATS } from '../constants';
|
||||
import { DataStreamType, QualityIndicators } from '../types';
|
||||
import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils';
|
||||
import { Integration } from './integration';
|
||||
import { DataStreamStatType } from './types';
|
||||
|
||||
interface QualityStat {
|
||||
percentage: number;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export class DataStreamStat {
|
||||
rawName: string;
|
||||
type: DataStreamType;
|
||||
|
|
@ -26,10 +30,8 @@ export class DataStreamStat {
|
|||
integration?: Integration;
|
||||
quality: QualityIndicators;
|
||||
docsInTimeRange?: number;
|
||||
degradedDocs: {
|
||||
percentage: number;
|
||||
count: number;
|
||||
};
|
||||
degradedDocs: QualityStat;
|
||||
failedDocs: QualityStat;
|
||||
|
||||
private constructor(dataStreamStat: DataStreamStat) {
|
||||
this.rawName = dataStreamStat.rawName;
|
||||
|
|
@ -46,6 +48,7 @@ export class DataStreamStat {
|
|||
this.quality = dataStreamStat.quality;
|
||||
this.docsInTimeRange = dataStreamStat.docsInTimeRange;
|
||||
this.degradedDocs = dataStreamStat.degradedDocs;
|
||||
this.failedDocs = dataStreamStat.failedDocs;
|
||||
}
|
||||
|
||||
public static create(dataStreamStat: DataStreamStatType) {
|
||||
|
|
@ -63,36 +66,45 @@ export class DataStreamStat {
|
|||
userPrivileges: dataStreamStat.userPrivileges,
|
||||
totalDocs: dataStreamStat.totalDocs,
|
||||
quality: DEFAULT_DATASET_QUALITY,
|
||||
degradedDocs: DEFAULT_DEGRADED_DOCS,
|
||||
degradedDocs: DEFAULT_QUALITY_DOC_STATS,
|
||||
failedDocs: DEFAULT_QUALITY_DOC_STATS,
|
||||
};
|
||||
|
||||
return new DataStreamStat(dataStreamStatProps);
|
||||
}
|
||||
|
||||
public static fromDegradedDocStat({
|
||||
public static fromQualityStats({
|
||||
datasetName,
|
||||
degradedDocStat,
|
||||
failedDocStat,
|
||||
datasetIntegrationMap,
|
||||
totalDocs,
|
||||
}: {
|
||||
degradedDocStat: DataStreamDocsStat & { percentage: number };
|
||||
datasetName: string;
|
||||
degradedDocStat: QualityStat;
|
||||
failedDocStat: QualityStat;
|
||||
datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
|
||||
totalDocs: number;
|
||||
}) {
|
||||
const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset);
|
||||
const { type, dataset, namespace } = indexNameToDataStreamParts(datasetName);
|
||||
|
||||
const dataStreamStatProps = {
|
||||
rawName: degradedDocStat.dataset,
|
||||
rawName: datasetName,
|
||||
type,
|
||||
name: dataset,
|
||||
title: datasetIntegrationMap[dataset]?.title || dataset,
|
||||
namespace,
|
||||
integration: datasetIntegrationMap[dataset]?.integration,
|
||||
quality: mapPercentageToQuality(degradedDocStat.percentage),
|
||||
quality: mapPercentageToQuality([degradedDocStat.percentage, failedDocStat.percentage]),
|
||||
docsInTimeRange: totalDocs,
|
||||
degradedDocs: {
|
||||
percentage: degradedDocStat.percentage,
|
||||
count: degradedDocStat.count,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: failedDocStat.percentage,
|
||||
count: failedDocStat.count,
|
||||
},
|
||||
};
|
||||
|
||||
return new DataStreamStat(dataStreamStatProps);
|
||||
|
|
@ -102,8 +114,4 @@ export class DataStreamStat {
|
|||
const avgDocSize = sizeBytes && totalDocs ? sizeBytes / totalDocs : 0;
|
||||
return avgDocSize * (docsInTimeRange ?? 0);
|
||||
}
|
||||
|
||||
public static calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) {
|
||||
return totalDocs && count ? (count / totalDocs) * 100 : 0;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ export type GetDataStreamsStatsResponse =
|
|||
APIReturnType<`GET /internal/dataset_quality/data_streams/stats`>;
|
||||
export type DataStreamStatType = GetDataStreamsStatsResponse['dataStreamsStats'][0];
|
||||
export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse;
|
||||
|
||||
export type GetDataStreamsDegradedDocsStatsParams =
|
||||
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params'];
|
||||
export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query'];
|
||||
|
|
@ -80,3 +79,20 @@ export type {
|
|||
DegradedField,
|
||||
DegradedFieldResponse,
|
||||
} from '../api_types';
|
||||
|
||||
/*
|
||||
Types for Failure store information
|
||||
*/
|
||||
export type GetDataStreamsFailedDocsStatsParams =
|
||||
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/failed_docs`>['params'];
|
||||
export type GetDataStreamsFailedDocsStatsQuery = GetDataStreamsFailedDocsStatsParams['query'];
|
||||
export type GetDataStreamsFailedDocsDetailsParams =
|
||||
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs`>['params'];
|
||||
export type GetDataStreamsFailedDocsDetailsQuery = GetDataStreamsFailedDocsDetailsParams['query'];
|
||||
export type GetDataStreamFailedDocsDetailsParams = GetDataStreamsFailedDocsDetailsParams['path'] &
|
||||
GetDataStreamsFailedDocsDetailsQuery;
|
||||
export type GetDataStreamsFailedDocsErrorsParams =
|
||||
APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs/errors`>['params'];
|
||||
export type GetDataStreamsFailedDocsErrorsQuery = GetDataStreamsFailedDocsErrorsParams['query'];
|
||||
export type GetDataStreamFailedDocsErrorsParams = GetDataStreamsFailedDocsErrorsParams['path'] &
|
||||
GetDataStreamsFailedDocsErrorsQuery;
|
||||
|
|
|
|||
|
|
@ -79,12 +79,9 @@ export const flyoutSummaryText = i18n.translate('xpack.datasetQuality.flyoutSumm
|
|||
defaultMessage: 'Summary',
|
||||
});
|
||||
|
||||
export const overviewDegradedDocsText = i18n.translate(
|
||||
'xpack.datasetQuality.flyout.degradedDocsTitle',
|
||||
{
|
||||
defaultMessage: 'Degraded docs',
|
||||
}
|
||||
);
|
||||
export const overviewTrendsDocsText = i18n.translate('xpack.datasetQuality.flyout.trendDocsTitle', {
|
||||
defaultMessage: 'Document trends',
|
||||
});
|
||||
|
||||
export const flyoutDegradedDocsTrendText = i18n.translate(
|
||||
'xpack.datasetQuality.flyoutDegradedDocsViz',
|
||||
|
|
@ -93,6 +90,13 @@ export const flyoutDegradedDocsTrendText = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const flyoutFailedDocsTrendText = i18n.translate(
|
||||
'xpack.datasetQuality.flyoutFailedDocsViz',
|
||||
{
|
||||
defaultMessage: 'Failed documents trend',
|
||||
}
|
||||
);
|
||||
|
||||
export const flyoutDegradedDocsPercentageText = i18n.translate(
|
||||
'xpack.datasetQuality.flyoutDegradedDocsPercentage',
|
||||
{
|
||||
|
|
@ -101,6 +105,14 @@ export const flyoutDegradedDocsPercentageText = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const flyoutFailedDocsPercentageText = i18n.translate(
|
||||
'xpack.datasetQuality.flyoutFailedDocsPercentage',
|
||||
{
|
||||
defaultMessage: 'Failed docs %',
|
||||
description: 'Tooltip label for the percentage of failed documents chart.',
|
||||
}
|
||||
);
|
||||
|
||||
export const flyoutDocsCountTotalText = i18n.translate(
|
||||
'xpack.datasetQuality.flyoutDocsCountTotal',
|
||||
{
|
||||
|
|
@ -137,14 +149,14 @@ export const summaryPanelLast24hText = i18n.translate(
|
|||
export const summaryPanelQualityText = i18n.translate(
|
||||
'xpack.datasetQuality.summaryPanelQualityText',
|
||||
{
|
||||
defaultMessage: 'Data Sets Quality',
|
||||
defaultMessage: 'Data Set Quality',
|
||||
}
|
||||
);
|
||||
|
||||
export const summaryPanelQualityTooltipText = i18n.translate(
|
||||
'xpack.datasetQuality.summaryPanelQualityTooltipText',
|
||||
{
|
||||
defaultMessage: 'Quality is based on the percentage of degraded docs in a data set.',
|
||||
defaultMessage: 'Quality is based on the percentage of degraded and failed docs in a data set.',
|
||||
}
|
||||
);
|
||||
|
||||
|
|
@ -305,6 +317,13 @@ export const overviewPanelDatasetQualityIndicatorDegradedDocs = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const overviewPanelDatasetQualityIndicatorFailedDocs = i18n.translate(
|
||||
'xpack.datasetQuality.details.overviewPanel.datasetQuality.failedDocs',
|
||||
{
|
||||
defaultMessage: 'Failed docs',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldsTableLoadingText = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldsTableLoadingText',
|
||||
{
|
||||
|
|
@ -312,37 +331,37 @@ export const overviewDegradedFieldsTableLoadingText = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldsTableNoData = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldsTableNoData',
|
||||
export const qualityIssuesTableNoData = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssuesTableNoData',
|
||||
{
|
||||
defaultMessage: 'No degraded fields found',
|
||||
defaultMessage: 'No quality issues found',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldsSectionTitle = i18n.translate(
|
||||
'xpack.datasetQuality.detail.degradedFieldsSectionTitle',
|
||||
export const overviewQualityIssuesSectionTitle = i18n.translate(
|
||||
'xpack.datasetQuality.detail.qualityIssuesSectionTitle',
|
||||
{
|
||||
defaultMessage: 'Quality issues',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldToggleSwitch = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldToggleSwitch',
|
||||
export const currentIssuesToggleSwitch = i18n.translate(
|
||||
'xpack.datasetQuality.details.currentIssuesToggleSwitch',
|
||||
{
|
||||
defaultMessage: 'Current quality issues only',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldToggleSwitchTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip',
|
||||
export const currentIssuesToggleSwitchTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.currentIssuesToggleSwitchTooltip',
|
||||
{
|
||||
defaultMessage:
|
||||
'Enable to only show issues detected in the most recent version of the data set. Disable to show all issues detected within the configured time range.',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldsSectionTitleTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldsSectionTooltip',
|
||||
export const overviewQualityIssueSectionTitleTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssueSectionTitleTooltip',
|
||||
{
|
||||
defaultMessage: 'A partial list of quality issues found in your data set.',
|
||||
}
|
||||
|
|
@ -386,21 +405,31 @@ export const integrationVersionText = i18n.translate(
|
|||
defaultMessage: 'Version',
|
||||
}
|
||||
);
|
||||
export const fieldColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.field', {
|
||||
defaultMessage: 'Field',
|
||||
export const issueColumnName = i18n.translate('xpack.datasetQuality.details.qualityIssues.issue', {
|
||||
defaultMessage: 'Issue',
|
||||
});
|
||||
|
||||
export const countColumnName = i18n.translate('xpack.datasetQuality.details.degradedField.count', {
|
||||
defaultMessage: 'Docs count',
|
||||
});
|
||||
export const countColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssues.docsCount',
|
||||
{
|
||||
defaultMessage: 'Docs count',
|
||||
}
|
||||
);
|
||||
|
||||
export const lastOccurrenceColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.lastOccurrence',
|
||||
'xpack.datasetQuality.details.qualityIssues.lastOccurrence',
|
||||
{
|
||||
defaultMessage: 'Last occurrence',
|
||||
}
|
||||
);
|
||||
|
||||
export const documentIndexFailed = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssues.documentIndexFailed',
|
||||
{
|
||||
defaultMessage: 'Documents indexing failed',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldValuesColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.values',
|
||||
{
|
||||
|
|
@ -665,3 +694,10 @@ export const manualMitigationCustomPipelineCreateEditPipelineLink = i18n.transla
|
|||
defaultMessage: 'create or edit the pipeline',
|
||||
}
|
||||
);
|
||||
|
||||
export const failedDocsErrorsColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.failedDocs.errors',
|
||||
{
|
||||
defaultMessage: 'Error messages',
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
// https://github.com/gcanti/io-ts/blob/master/index.md#union-of-string-literals
|
||||
import * as t from 'io-ts';
|
||||
import { ALL_PATTERNS_SELECTOR, DATA_SELECTOR, FAILURE_STORE_SELECTOR } from '../constants';
|
||||
|
||||
export const dataStreamTypesRt = t.keyof({
|
||||
logs: null,
|
||||
|
|
@ -17,3 +18,11 @@ export const dataStreamTypesRt = t.keyof({
|
|||
});
|
||||
|
||||
export type DataStreamType = t.TypeOf<typeof dataStreamTypesRt>;
|
||||
|
||||
export const dataStreamSelectorsRt = t.keyof({
|
||||
[ALL_PATTERNS_SELECTOR]: null,
|
||||
[FAILURE_STORE_SELECTOR]: null,
|
||||
[DATA_SELECTOR]: null,
|
||||
});
|
||||
|
||||
export type DataStreamSelector = t.TypeOf<typeof dataStreamSelectorsRt>;
|
||||
|
|
|
|||
|
|
@ -85,4 +85,10 @@ describe('dataset_name', () => {
|
|||
).toEqual('metrics-apm.app.adservice-default');
|
||||
});
|
||||
});
|
||||
|
||||
it('returns the correct index name if backing index is a failure store index', () => {
|
||||
expect(
|
||||
extractIndexNameFromBackingIndex('.fs-logs-elastic_agent-default-2024.11.11-000001')
|
||||
).toEqual('logs-elastic_agent-default');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ export const indexNameToDataStreamParts = (dataStreamName: string) => {
|
|||
};
|
||||
|
||||
export const extractIndexNameFromBackingIndex = (indexString: string): string => {
|
||||
const pattern = /.ds-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
|
||||
const pattern = /.(?:ds|fs)-(.*?)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-[0-9]{6}/;
|
||||
const match = indexString.match(pattern);
|
||||
|
||||
return match ? match[1] : indexString;
|
||||
|
|
|
|||
|
|
@ -8,10 +8,14 @@
|
|||
import { POOR_QUALITY_MINIMUM_PERCENTAGE, DEGRADED_QUALITY_MINIMUM_PERCENTAGE } from '../constants';
|
||||
import { QualityIndicators } from '../types';
|
||||
|
||||
export const mapPercentageToQuality = (percentage: number): QualityIndicators => {
|
||||
return percentage > POOR_QUALITY_MINIMUM_PERCENTAGE
|
||||
? 'poor'
|
||||
: percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE
|
||||
? 'degraded'
|
||||
: 'good';
|
||||
export const mapPercentageToQuality = (percentages: number[]): QualityIndicators => {
|
||||
if (percentages.some((percentage) => percentage > POOR_QUALITY_MINIMUM_PERCENTAGE)) {
|
||||
return 'poor';
|
||||
}
|
||||
|
||||
if (percentages.some((percentage) => percentage > DEGRADED_QUALITY_MINIMUM_PERCENTAGE)) {
|
||||
return 'degraded';
|
||||
}
|
||||
|
||||
return 'good';
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { ITelemetryClient } from '../../services/telemetry';
|
|||
export interface DatasetQualityContextValue {
|
||||
service: DatasetQualityControllerStateService;
|
||||
telemetryClient: ITelemetryClient;
|
||||
isServerless: boolean;
|
||||
}
|
||||
|
||||
export const DatasetQualityContext = createContext({} as DatasetQualityContextValue);
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ export interface CreateDatasetQualityArgs {
|
|||
core: CoreStart;
|
||||
plugins: DatasetQualityStartDeps;
|
||||
telemetryClient: ITelemetryClient;
|
||||
isServerless: boolean;
|
||||
}
|
||||
|
||||
export const DatasetQuality = ({
|
||||
|
|
@ -36,6 +37,7 @@ export const DatasetQuality = ({
|
|||
core,
|
||||
plugins,
|
||||
telemetryClient,
|
||||
isServerless,
|
||||
}: DatasetQualityProps & CreateDatasetQualityArgs) => {
|
||||
const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider(core, plugins);
|
||||
|
||||
|
|
@ -43,8 +45,9 @@ export const DatasetQuality = ({
|
|||
() => ({
|
||||
service: controller.service,
|
||||
telemetryClient,
|
||||
isServerless,
|
||||
}),
|
||||
[controller.service, telemetryClient]
|
||||
[controller.service, isServerless, telemetryClient]
|
||||
);
|
||||
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export const createDatasetQuality = ({
|
|||
core,
|
||||
plugins,
|
||||
telemetryClient,
|
||||
isServerless,
|
||||
}: CreateDatasetQualityArgs) => {
|
||||
return ({ controller }: DatasetQualityProps) => {
|
||||
return (
|
||||
|
|
@ -27,6 +28,7 @@ export const createDatasetQuality = ({
|
|||
core={core}
|
||||
plugins={plugins}
|
||||
telemetryClient={telemetryClient}
|
||||
isServerless={isServerless}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -26,23 +26,14 @@ import {
|
|||
summaryPanelQualityText,
|
||||
summaryPanelQualityTooltipText,
|
||||
} from '../../../../common/translations';
|
||||
import { mapPercentagesToQualityCounts } from '../../quality_indicator';
|
||||
import { useDatasetQualityFilters } from '../../../hooks/use_dataset_quality_filters';
|
||||
import { VerticalRule } from '../../common/vertical_rule';
|
||||
|
||||
export function DatasetsQualityIndicators() {
|
||||
const { onPageReady } = usePerformanceContext();
|
||||
const { timeRange } = useDatasetQualityFilters();
|
||||
const {
|
||||
datasetsQuality,
|
||||
isDatasetsQualityLoading,
|
||||
datasetsActivity,
|
||||
numberOfDatasets,
|
||||
numberOfDocuments,
|
||||
} = useSummaryPanelContext();
|
||||
const qualityCounts = mapPercentagesToQualityCounts(datasetsQuality.percentages);
|
||||
const datasetsWithoutIgnoredField =
|
||||
datasetsActivity.total > 0 ? datasetsActivity.total - datasetsQuality.percentages.length : 0;
|
||||
const { datasetsQuality, isDatasetsQualityLoading, numberOfDatasets, numberOfDocuments } =
|
||||
useSummaryPanelContext();
|
||||
|
||||
if (!isDatasetsQualityLoading && (numberOfDatasets || numberOfDocuments)) {
|
||||
onPageReady({
|
||||
|
|
@ -72,21 +63,21 @@ export function DatasetsQualityIndicators() {
|
|||
</EuiFlexGroup>
|
||||
<EuiFlexGroup gutterSize="m" alignItems="flexEnd">
|
||||
<QualityIndicator
|
||||
value={qualityCounts.poor}
|
||||
value={datasetsQuality.poor}
|
||||
quality="danger"
|
||||
description={summaryPanelQualityPoorText}
|
||||
isLoading={isDatasetsQualityLoading}
|
||||
/>
|
||||
<VerticalRule />
|
||||
<QualityIndicator
|
||||
value={qualityCounts.degraded}
|
||||
value={datasetsQuality.degraded}
|
||||
quality="warning"
|
||||
description={summaryPanelQualityDegradedText}
|
||||
isLoading={isDatasetsQualityLoading}
|
||||
/>
|
||||
<VerticalRule />
|
||||
<QualityIndicator
|
||||
value={qualityCounts.good + datasetsWithoutIgnoredField}
|
||||
value={datasetsQuality.good}
|
||||
quality="success"
|
||||
description={summaryPanelQualityGoodText}
|
||||
isLoading={isDatasetsQualityLoading}
|
||||
|
|
|
|||
|
|
@ -13,33 +13,35 @@ import {
|
|||
EuiFlexItem,
|
||||
EuiIcon,
|
||||
EuiLink,
|
||||
EuiToolTip,
|
||||
EuiText,
|
||||
formatNumber,
|
||||
EuiSkeletonRectangle,
|
||||
EuiTableHeader,
|
||||
EuiText,
|
||||
EuiToolTip,
|
||||
formatNumber,
|
||||
} from '@elastic/eui';
|
||||
import { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
|
||||
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import React from 'react';
|
||||
import { BrowserUrlService } from '@kbn/share-plugin/public';
|
||||
import React from 'react';
|
||||
import {
|
||||
DEGRADED_QUALITY_MINIMUM_PERCENTAGE,
|
||||
POOR_QUALITY_MINIMUM_PERCENTAGE,
|
||||
BYTE_NUMBER_FORMAT,
|
||||
DEGRADED_DOCS_QUERY,
|
||||
DEGRADED_QUALITY_MINIMUM_PERCENTAGE,
|
||||
FAILURE_STORE_SELECTOR,
|
||||
POOR_QUALITY_MINIMUM_PERCENTAGE,
|
||||
} from '../../../../common/constants';
|
||||
import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat';
|
||||
import { DatasetQualityIndicator, QualityIndicator } from '../../quality_indicator';
|
||||
import { PrivilegesWarningIconWrapper, IntegrationIcon } from '../../common';
|
||||
import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks';
|
||||
import { DegradedDocsPercentageLink } from './degraded_docs_percentage_link';
|
||||
import { TimeRangeConfig } from '../../../../common/types';
|
||||
import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks';
|
||||
import { IntegrationIcon, PrivilegesWarningIconWrapper } from '../../common';
|
||||
import { DatasetQualityIndicator, QualityIndicator } from '../../quality_indicator';
|
||||
import { DatasetQualityDetailsLink } from './dataset_quality_details_link';
|
||||
import { QualityStatPercentageLink } from './quality_stat_percentage_link';
|
||||
|
||||
const nameColumnName = i18n.translate('xpack.datasetQuality.nameColumnName', {
|
||||
defaultMessage: 'Data Set Name',
|
||||
defaultMessage: 'Data set name',
|
||||
});
|
||||
|
||||
const namespaceColumnName = i18n.translate('xpack.datasetQuality.namespaceColumnName', {
|
||||
|
|
@ -55,15 +57,19 @@ const sizeColumnName = i18n.translate('xpack.datasetQuality.sizeColumnName', {
|
|||
});
|
||||
|
||||
const degradedDocsColumnName = i18n.translate('xpack.datasetQuality.degradedDocsColumnName', {
|
||||
defaultMessage: 'Degraded Docs (%)',
|
||||
defaultMessage: 'Degraded docs (%)',
|
||||
});
|
||||
|
||||
const failedDocsColumnName = i18n.translate('xpack.datasetQuality.failedDocsColumnName', {
|
||||
defaultMessage: 'Failed docs (%)',
|
||||
});
|
||||
|
||||
const datasetQualityColumnName = i18n.translate('xpack.datasetQuality.datasetQualityColumnName', {
|
||||
defaultMessage: 'Data Set Quality',
|
||||
defaultMessage: 'Data set quality',
|
||||
});
|
||||
|
||||
const lastActivityColumnName = i18n.translate('xpack.datasetQuality.lastActivityColumnName', {
|
||||
defaultMessage: 'Last Activity',
|
||||
defaultMessage: 'Last activity',
|
||||
});
|
||||
|
||||
const actionsColumnName = i18n.translate('xpack.datasetQuality.actionsColumnName', {
|
||||
|
|
@ -112,10 +118,17 @@ const degradedDocsColumnTooltip = (
|
|||
/>
|
||||
);
|
||||
|
||||
const failedDocsColumnTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.failedDocsColumnTooltip"
|
||||
defaultMessage="The percentage of docs sent to failure store due to an issue during ingestion."
|
||||
/>
|
||||
);
|
||||
|
||||
const datasetQualityColumnTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.datasetQualityColumnTooltip"
|
||||
defaultMessage="Quality is based on the percentage of degraded docs in a data set. {visualQueue}"
|
||||
defaultMessage="Quality is based on the percentage of degraded and failed docs in a data set. {visualQueue}"
|
||||
values={{
|
||||
visualQueue: (
|
||||
<EuiFlexGroup direction="column" gutterSize="xs">
|
||||
|
|
@ -159,21 +172,27 @@ export const getDatasetQualityTableColumns = ({
|
|||
canUserMonitorDataset,
|
||||
canUserMonitorAnyDataStream,
|
||||
loadingDataStreamStats,
|
||||
loadingDocStats,
|
||||
loadingDegradedStats,
|
||||
loadingFailedStats,
|
||||
showFullDatasetNames,
|
||||
isActiveDataset,
|
||||
timeRange,
|
||||
urlService,
|
||||
failureStoreEnabled,
|
||||
}: {
|
||||
fieldFormats: FieldFormatsStart;
|
||||
canUserMonitorDataset: boolean;
|
||||
canUserMonitorAnyDataStream: boolean;
|
||||
loadingDataStreamStats: boolean;
|
||||
loadingDocStats: boolean;
|
||||
loadingDegradedStats: boolean;
|
||||
loadingFailedStats: boolean;
|
||||
showFullDatasetNames: boolean;
|
||||
isActiveDataset: (lastActivity: number) => boolean;
|
||||
timeRange: TimeRangeConfig;
|
||||
urlService: BrowserUrlService;
|
||||
failureStoreEnabled: boolean;
|
||||
}): Array<EuiBasicTableColumn<DataStreamStat>> => {
|
||||
return [
|
||||
{
|
||||
|
|
@ -248,7 +267,7 @@ export const getDatasetQualityTableColumns = ({
|
|||
width="60px"
|
||||
height="20px"
|
||||
borderRadius="m"
|
||||
isLoading={loadingDataStreamStats || loadingDegradedStats}
|
||||
isLoading={loadingDataStreamStats || loadingDocStats}
|
||||
>
|
||||
{formatNumber(
|
||||
DataStreamStat.calculateFilteredSize(dataStreamStat),
|
||||
|
|
@ -273,11 +292,11 @@ export const getDatasetQualityTableColumns = ({
|
|||
</EuiToolTip>
|
||||
</EuiTableHeader>
|
||||
),
|
||||
field: 'degradedDocs.percentage',
|
||||
field: 'quality',
|
||||
sortable: true,
|
||||
render: (_, dataStreamStat: DataStreamStat) => (
|
||||
<DatasetQualityIndicator
|
||||
isLoading={loadingDegradedStats}
|
||||
isLoading={loadingDegradedStats || loadingFailedStats || loadingDocStats}
|
||||
quality={dataStreamStat.quality}
|
||||
/>
|
||||
),
|
||||
|
|
@ -297,45 +316,102 @@ export const getDatasetQualityTableColumns = ({
|
|||
field: 'degradedDocs.percentage',
|
||||
sortable: true,
|
||||
render: (_, dataStreamStat: DataStreamStat) => (
|
||||
<DegradedDocsPercentageLink
|
||||
<QualityStatPercentageLink
|
||||
isLoading={loadingDegradedStats}
|
||||
dataStreamStat={dataStreamStat}
|
||||
timeRange={timeRange}
|
||||
accessor="degradedDocs"
|
||||
query={{ language: 'kuery', query: DEGRADED_DOCS_QUERY }}
|
||||
fewDocStatsTooltip={(degradedDocsCount: number) =>
|
||||
i18n.translate('xpack.datasetQuality.fewDegradedDocsTooltip', {
|
||||
defaultMessage: '{degradedDocsCount} degraded docs in this data set.',
|
||||
values: {
|
||||
degradedDocsCount,
|
||||
},
|
||||
})
|
||||
}
|
||||
dataTestSubj="datasetQualityDegradedDocsPercentageLink"
|
||||
/>
|
||||
),
|
||||
width: '140px',
|
||||
},
|
||||
{
|
||||
name: (
|
||||
<EuiTableHeader data-test-subj="datasetQualityLastActivityColumn">
|
||||
{lastActivityColumnName}
|
||||
</EuiTableHeader>
|
||||
),
|
||||
field: 'lastActivity',
|
||||
render: (timestamp: number) => (
|
||||
<EuiSkeletonRectangle
|
||||
width="200px"
|
||||
height="20px"
|
||||
borderRadius="m"
|
||||
isLoading={loadingDataStreamStats}
|
||||
>
|
||||
{!isActiveDataset(timestamp) ? (
|
||||
<EuiFlexGroup gutterSize="xs" alignItems="center">
|
||||
<EuiText size="s">{inactiveDatasetActivityColumnDescription}</EuiText>
|
||||
<EuiToolTip position="top" content={inactiveDatasetActivityColumnTooltip}>
|
||||
<EuiIcon tabIndex={0} type="iInCircle" size="s" />
|
||||
</EuiToolTip>
|
||||
</EuiFlexGroup>
|
||||
) : (
|
||||
fieldFormats
|
||||
.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
|
||||
.convert(timestamp)
|
||||
)}
|
||||
</EuiSkeletonRectangle>
|
||||
),
|
||||
width: '300px',
|
||||
sortable: true,
|
||||
},
|
||||
...(failureStoreEnabled
|
||||
? [
|
||||
{
|
||||
name: (
|
||||
<EuiTableHeader data-test-subj="datasetQualityFailedPercentageColumn">
|
||||
<EuiToolTip content={failedDocsColumnTooltip}>
|
||||
<span>
|
||||
{`${failedDocsColumnName} `}
|
||||
<EuiIcon
|
||||
size="s"
|
||||
color="subdued"
|
||||
type="questionInCircle"
|
||||
className="eui-alignTop"
|
||||
/>
|
||||
</span>
|
||||
</EuiToolTip>
|
||||
</EuiTableHeader>
|
||||
),
|
||||
field: 'failedDocs.percentage',
|
||||
sortable: true,
|
||||
render: (_: any, dataStreamStat: DataStreamStat) => (
|
||||
<QualityStatPercentageLink
|
||||
isLoading={loadingFailedStats}
|
||||
dataStreamStat={dataStreamStat}
|
||||
timeRange={timeRange}
|
||||
accessor="failedDocs"
|
||||
selector={FAILURE_STORE_SELECTOR}
|
||||
fewDocStatsTooltip={(failedDocsCount: number) =>
|
||||
i18n.translate('xpack.datasetQuality.fewFailedDocsTooltip', {
|
||||
defaultMessage: '{failedDocsCount} failed docs in this data set.',
|
||||
values: {
|
||||
failedDocsCount,
|
||||
},
|
||||
})
|
||||
}
|
||||
dataTestSubj="datasetQualityFailedDocsPercentageLink"
|
||||
/>
|
||||
),
|
||||
width: '140px',
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(canUserMonitorDataset && canUserMonitorAnyDataStream
|
||||
? [
|
||||
{
|
||||
name: (
|
||||
<EuiTableHeader data-test-subj="datasetQualityLastActivityColumn">
|
||||
{lastActivityColumnName}
|
||||
</EuiTableHeader>
|
||||
),
|
||||
field: 'lastActivity',
|
||||
render: (timestamp: number) => (
|
||||
<EuiSkeletonRectangle
|
||||
width="200px"
|
||||
height="20px"
|
||||
borderRadius="m"
|
||||
isLoading={loadingDataStreamStats}
|
||||
>
|
||||
{!isActiveDataset(timestamp) ? (
|
||||
<EuiFlexGroup gutterSize="xs" alignItems="center">
|
||||
<EuiText size="s">{inactiveDatasetActivityColumnDescription}</EuiText>
|
||||
<EuiToolTip position="top" content={inactiveDatasetActivityColumnTooltip}>
|
||||
<EuiIcon tabIndex={0} type="iInCircle" size="s" />
|
||||
</EuiToolTip>
|
||||
</EuiFlexGroup>
|
||||
) : (
|
||||
fieldFormats
|
||||
.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ES_FIELD_TYPES.DATE])
|
||||
.convert(timestamp)
|
||||
)}
|
||||
</EuiSkeletonRectangle>
|
||||
),
|
||||
width: '300px',
|
||||
sortable: true,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
name: actionsColumnName,
|
||||
render: (dataStreamStat: DataStreamStat) => (
|
||||
|
|
|
|||
|
|
@ -5,51 +5,65 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { EuiSkeletonRectangle, EuiFlexGroup, EuiLink } from '@elastic/eui';
|
||||
import { EuiFlexGroup, EuiLink, EuiSkeletonRectangle } from '@elastic/eui';
|
||||
import React from 'react';
|
||||
import { _IGNORED } from '../../../../common/es_fields';
|
||||
import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat';
|
||||
import { DataStreamSelector, TimeRangeConfig } from '../../../../common/types';
|
||||
import { useDatasetRedirectLinkTelemetry, useRedirectLink } from '../../../hooks';
|
||||
import { QualityPercentageIndicator } from '../../quality_indicator';
|
||||
import { DataStreamStat } from '../../../../common/data_streams_stats/data_stream_stat';
|
||||
import { TimeRangeConfig } from '../../../../common/types';
|
||||
|
||||
export const DegradedDocsPercentageLink = ({
|
||||
export const QualityStatPercentageLink = ({
|
||||
isLoading,
|
||||
dataStreamStat,
|
||||
timeRange,
|
||||
dataTestSubj,
|
||||
query = { language: 'kuery', query: '' },
|
||||
accessor,
|
||||
selector,
|
||||
fewDocStatsTooltip,
|
||||
}: {
|
||||
isLoading: boolean;
|
||||
dataStreamStat: DataStreamStat;
|
||||
timeRange: TimeRangeConfig;
|
||||
dataTestSubj: string;
|
||||
query?: { language: string; query: string };
|
||||
accessor: 'degradedDocs' | 'failedDocs';
|
||||
selector?: DataStreamSelector;
|
||||
fewDocStatsTooltip: (docsCount: number) => string;
|
||||
}) => {
|
||||
const {
|
||||
degradedDocs: { percentage, count },
|
||||
[accessor]: { percentage, count },
|
||||
} = dataStreamStat;
|
||||
|
||||
const { sendTelemetry } = useDatasetRedirectLinkTelemetry({
|
||||
rawName: dataStreamStat.rawName,
|
||||
query: { language: 'kuery', query: `${_IGNORED}: *` },
|
||||
rawName: `${dataStreamStat.rawName}${selector ?? ''}`,
|
||||
query,
|
||||
});
|
||||
|
||||
const redirectLinkProps = useRedirectLink({
|
||||
dataStreamStat,
|
||||
query: { language: 'kuery', query: `${_IGNORED}: *` },
|
||||
query,
|
||||
sendTelemetry,
|
||||
timeRangeConfig: timeRange,
|
||||
selector,
|
||||
});
|
||||
|
||||
return (
|
||||
<EuiSkeletonRectangle width="50px" height="20px" borderRadius="m" isLoading={isLoading}>
|
||||
<EuiFlexGroup alignItems="center" gutterSize="s">
|
||||
{percentage ? (
|
||||
<EuiLink
|
||||
data-test-subj="datasetQualityDegradedDocsPercentageLink"
|
||||
{...redirectLinkProps.linkProps}
|
||||
>
|
||||
<QualityPercentageIndicator percentage={percentage} degradedDocsCount={count} />
|
||||
<EuiLink data-test-subj={dataTestSubj} {...redirectLinkProps.linkProps}>
|
||||
<QualityPercentageIndicator
|
||||
percentage={percentage}
|
||||
docsCount={count}
|
||||
fewDocsTooltipContent={fewDocStatsTooltip}
|
||||
/>
|
||||
</EuiLink>
|
||||
) : (
|
||||
<QualityPercentageIndicator percentage={percentage} />
|
||||
<QualityPercentageIndicator
|
||||
percentage={percentage}
|
||||
fewDocsTooltipContent={fewDocStatsTooltip}
|
||||
/>
|
||||
)}
|
||||
</EuiFlexGroup>
|
||||
</EuiSkeletonRectangle>
|
||||
|
|
@ -11,6 +11,7 @@ import { ITelemetryClient } from '../../services/telemetry';
|
|||
export interface DatasetQualityDetailsContextValue {
|
||||
service: DatasetQualityDetailsControllerStateService;
|
||||
telemetryClient: ITelemetryClient;
|
||||
isServerless: boolean;
|
||||
}
|
||||
|
||||
export const DatasetQualityDetailsContext = createContext({} as DatasetQualityDetailsContextValue);
|
||||
|
|
|
|||
|
|
@ -13,12 +13,12 @@ import { Header } from './header';
|
|||
import { Overview } from './overview';
|
||||
import { Details } from './details';
|
||||
|
||||
const DegradedFieldFlyout = dynamic(() => import('./degraded_field_flyout'));
|
||||
const QualityIssueFlyout = dynamic(() => import('./quality_issue_flyout'));
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DatasetQualityDetails() {
|
||||
const { isIndexNotFoundError, dataStream, isDegradedFieldFlyoutOpen } =
|
||||
const { isIndexNotFoundError, dataStream, isQualityIssueFlyoutOpen } =
|
||||
useDatasetQualityDetailsState();
|
||||
const { startTracking } = useDatasetDetailsTelemetry();
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ export default function DatasetQualityDetails() {
|
|||
<Details />
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
{isDegradedFieldFlyoutOpen && <DegradedFieldFlyout />}
|
||||
{isQualityIssueFlyoutOpen && <QualityIssueFlyout />}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,12 +26,14 @@ export interface CreateDatasetQualityArgs {
|
|||
core: CoreStart;
|
||||
plugins: DatasetQualityStartDeps;
|
||||
telemetryClient: ITelemetryClient;
|
||||
isServerless: boolean;
|
||||
}
|
||||
|
||||
export const createDatasetQualityDetails = ({
|
||||
core,
|
||||
plugins,
|
||||
telemetryClient,
|
||||
isServerless,
|
||||
}: CreateDatasetQualityArgs) => {
|
||||
return ({ controller }: DatasetQualityDetailsProps) => {
|
||||
const KibanaContextProviderForPlugin = useKibanaContextForPluginProvider(core, plugins);
|
||||
|
|
@ -40,6 +42,7 @@ export const createDatasetQualityDetails = ({
|
|||
() => ({
|
||||
service: controller.service,
|
||||
telemetryClient,
|
||||
isServerless,
|
||||
}),
|
||||
[controller.service]
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,157 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import {
|
||||
EuiAccordion,
|
||||
EuiButtonIcon,
|
||||
EuiCode,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiIcon,
|
||||
EuiPanel,
|
||||
EuiSkeletonRectangle,
|
||||
EuiSpacer,
|
||||
EuiTitle,
|
||||
EuiToolTip,
|
||||
OnTimeChangeProps,
|
||||
useGeneratedHtmlId,
|
||||
} from '@elastic/eui';
|
||||
import type { DataViewField } from '@kbn/data-views-plugin/common';
|
||||
import { css } from '@emotion/react';
|
||||
import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public';
|
||||
import {
|
||||
discoverAriaText,
|
||||
openInDiscoverText,
|
||||
overviewDegradedDocsText,
|
||||
} from '../../../../../../common/translations';
|
||||
import { DegradedDocsChart } from './degraded_docs_chart';
|
||||
import {
|
||||
useDatasetDetailsRedirectLinkTelemetry,
|
||||
useDatasetQualityDetailsState,
|
||||
useDegradedDocsChart,
|
||||
useRedirectLink,
|
||||
} from '../../../../../hooks';
|
||||
import { _IGNORED } from '../../../../../../common/es_fields';
|
||||
import { NavigationSource } from '../../../../../services/telemetry';
|
||||
|
||||
const degradedDocsTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.details.degradedDocsTooltip"
|
||||
defaultMessage="The percentage of degraded documents —documents with the {ignoredProperty} property— in your data set."
|
||||
values={{
|
||||
ignoredProperty: (
|
||||
<EuiCode language="json" transparentBackground>
|
||||
_ignored
|
||||
</EuiCode>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: number }) {
|
||||
const { timeRange, updateTimeRange, datasetDetails } = useDatasetQualityDetailsState();
|
||||
const { dataView, breakdown, ...chartProps } = useDegradedDocsChart();
|
||||
|
||||
const accordionId = useGeneratedHtmlId({
|
||||
prefix: overviewDegradedDocsText,
|
||||
});
|
||||
|
||||
const [breakdownDataViewField, setBreakdownDataViewField] = useState<DataViewField | undefined>(
|
||||
undefined
|
||||
);
|
||||
|
||||
const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({
|
||||
query: { language: 'kuery', query: `${_IGNORED}: *` },
|
||||
navigationSource: NavigationSource.Trend,
|
||||
});
|
||||
|
||||
const degradedDocLinkLogsExplorer = useRedirectLink({
|
||||
dataStreamStat: datasetDetails,
|
||||
timeRangeConfig: timeRange,
|
||||
query: { language: 'kuery', query: `${_IGNORED}: *` },
|
||||
breakdownField: breakdownDataViewField?.name,
|
||||
sendTelemetry,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (breakdown.dataViewField && breakdown.fieldSupportsBreakdown) {
|
||||
setBreakdownDataViewField(breakdown.dataViewField);
|
||||
} else {
|
||||
setBreakdownDataViewField(undefined);
|
||||
}
|
||||
}, [setBreakdownDataViewField, breakdown]);
|
||||
|
||||
const onTimeRangeChange = useCallback(
|
||||
({ start, end }: Pick<OnTimeChangeProps, 'start' | 'end'>) => {
|
||||
updateTimeRange({ start, end, refreshInterval: timeRange.refresh.value });
|
||||
},
|
||||
[updateTimeRange, timeRange.refresh]
|
||||
);
|
||||
|
||||
const accordionTitle = (
|
||||
<EuiFlexItem
|
||||
css={css`
|
||||
flex-direction: row;
|
||||
justify-content: flex-start;
|
||||
align-items: flex-start;
|
||||
gap: 4px;
|
||||
`}
|
||||
>
|
||||
<EuiTitle size={'xxs'}>
|
||||
<h5>{overviewDegradedDocsText}</h5>
|
||||
</EuiTitle>
|
||||
<EuiToolTip content={degradedDocsTooltip}>
|
||||
<EuiIcon size="m" color="subdued" type="questionInCircle" className="eui-alignTop" />
|
||||
</EuiToolTip>
|
||||
</EuiFlexItem>
|
||||
);
|
||||
|
||||
return (
|
||||
<EuiPanel hasBorder grow={false}>
|
||||
<EuiAccordion
|
||||
id={accordionId}
|
||||
buttonContent={accordionTitle}
|
||||
paddingSize="none"
|
||||
initialIsOpen={true}
|
||||
data-test-subj="datasetQualityDetailsOverviewDocumentTrends"
|
||||
>
|
||||
<EuiFlexGroup justifyContent="flexEnd" gutterSize="s">
|
||||
<EuiSkeletonRectangle width={160} height={32} isLoading={!dataView}>
|
||||
<UnifiedBreakdownFieldSelector
|
||||
dataView={dataView!}
|
||||
breakdown={{ field: breakdownDataViewField }}
|
||||
onBreakdownFieldChange={breakdown.onChange}
|
||||
/>
|
||||
</EuiSkeletonRectangle>
|
||||
<EuiToolTip content={openInDiscoverText}>
|
||||
<EuiButtonIcon
|
||||
display="base"
|
||||
iconType="discoverApp"
|
||||
aria-label={discoverAriaText}
|
||||
size="s"
|
||||
data-test-subj="datasetQualityDetailsLinkToDiscover"
|
||||
{...degradedDocLinkLogsExplorer.linkProps}
|
||||
/>
|
||||
</EuiToolTip>
|
||||
</EuiFlexGroup>
|
||||
|
||||
<EuiSpacer size="m" />
|
||||
|
||||
<DegradedDocsChart
|
||||
{...chartProps}
|
||||
timeRange={timeRange}
|
||||
lastReloadTime={lastReloadTime}
|
||||
onTimeRangeChange={onTimeRangeChange}
|
||||
/>
|
||||
</EuiAccordion>
|
||||
</EuiPanel>
|
||||
);
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ import { i18n } from '@kbn/i18n';
|
|||
import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
import { DEGRADED_DOCS_QUERY } from '../../../../../../common/constants';
|
||||
import {
|
||||
flyoutDegradedDocsPercentageText,
|
||||
flyoutDegradedDocsTrendText,
|
||||
|
|
@ -180,7 +181,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
|
|||
scale: 'ratio',
|
||||
sourceField: '___records___',
|
||||
filter: {
|
||||
query: '_ignored: *',
|
||||
query: DEGRADED_DOCS_QUERY,
|
||||
language: 'kuery',
|
||||
},
|
||||
params: {
|
||||
|
|
@ -215,7 +216,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
|
|||
min: 0,
|
||||
max: 34,
|
||||
},
|
||||
text: "count(kql='_ignored: *') / count()",
|
||||
text: `count(kql='${DEGRADED_DOCS_QUERY}') / count()`,
|
||||
},
|
||||
},
|
||||
references: ['count_ignored', 'count_total'],
|
||||
|
|
@ -229,7 +230,7 @@ function getChartColumns(breakdownField?: string): Record<string, GenericIndexPa
|
|||
references: [DatasetQualityLensColumn.Math],
|
||||
isBucketed: false,
|
||||
params: {
|
||||
formula: "count(kql='_ignored: *') / count()",
|
||||
formula: `count(kql='${DEGRADED_DOCS_QUERY}') / count()`,
|
||||
format: {
|
||||
id: 'percent',
|
||||
params: {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,278 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import type { GenericIndexPatternColumn, TypedLensByValueInput } from '@kbn/lens-plugin/public';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
import { ALL_PATTERNS_SELECTOR, FAILURE_STORE_SELECTOR } from '../../../../../../common/constants';
|
||||
import {
|
||||
flyoutFailedDocsTrendText,
|
||||
flyoutFailedDocsPercentageText,
|
||||
} from '../../../../../../common/translations';
|
||||
|
||||
enum DatasetQualityLensColumn {
|
||||
Date = 'date_column',
|
||||
FailedDocs = 'failed_docs_column',
|
||||
CountFailed = 'count_failed',
|
||||
CountTotal = 'count_total',
|
||||
Math = 'math_column',
|
||||
Breakdown = 'breakdown_column',
|
||||
}
|
||||
|
||||
const MAX_BREAKDOWN_SERIES = 5;
|
||||
const FAILED_DOCS_QUERY = `_index: "*${FAILURE_STORE_SELECTOR}"`;
|
||||
|
||||
interface GetLensAttributesParams {
|
||||
color: string;
|
||||
dataStream: string;
|
||||
datasetTitle: string;
|
||||
breakdownFieldName?: string;
|
||||
}
|
||||
|
||||
export function getLensAttributes({
|
||||
color,
|
||||
dataStream,
|
||||
datasetTitle,
|
||||
breakdownFieldName,
|
||||
}: GetLensAttributesParams) {
|
||||
const dataViewId = uuidv4();
|
||||
|
||||
const columnOrder = [
|
||||
DatasetQualityLensColumn.Date,
|
||||
DatasetQualityLensColumn.CountFailed,
|
||||
DatasetQualityLensColumn.CountTotal,
|
||||
DatasetQualityLensColumn.Math,
|
||||
DatasetQualityLensColumn.FailedDocs,
|
||||
];
|
||||
|
||||
if (breakdownFieldName) {
|
||||
columnOrder.unshift(DatasetQualityLensColumn.Breakdown);
|
||||
}
|
||||
|
||||
const columns = getChartColumns(breakdownFieldName);
|
||||
return {
|
||||
visualizationType: 'lnsXY',
|
||||
title: flyoutFailedDocsTrendText,
|
||||
references: [],
|
||||
state: {
|
||||
...getAdHocDataViewState(dataViewId, dataStream, datasetTitle),
|
||||
datasourceStates: {
|
||||
formBased: {
|
||||
layers: {
|
||||
layer1: {
|
||||
columnOrder,
|
||||
columns,
|
||||
indexPatternId: dataViewId,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
filters: [],
|
||||
query: {
|
||||
language: 'kuery',
|
||||
query: '',
|
||||
},
|
||||
visualization: {
|
||||
axisTitlesVisibilitySettings: {
|
||||
x: false,
|
||||
yLeft: false,
|
||||
yRight: false,
|
||||
},
|
||||
fittingFunction: 'None',
|
||||
gridlinesVisibilitySettings: {
|
||||
x: true,
|
||||
yLeft: true,
|
||||
yRight: true,
|
||||
},
|
||||
layers: [
|
||||
{
|
||||
accessors: [DatasetQualityLensColumn.FailedDocs],
|
||||
layerId: 'layer1',
|
||||
layerType: 'data',
|
||||
seriesType: 'line',
|
||||
xAccessor: DatasetQualityLensColumn.Date,
|
||||
...(breakdownFieldName
|
||||
? { splitAccessor: DatasetQualityLensColumn.Breakdown }
|
||||
: {
|
||||
yConfig: [
|
||||
{
|
||||
forAccessor: DatasetQualityLensColumn.FailedDocs,
|
||||
color,
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
],
|
||||
legend: {
|
||||
isVisible: true,
|
||||
position: 'right',
|
||||
legendSize: 'large',
|
||||
shouldTruncate: true,
|
||||
},
|
||||
preferredSeriesType: 'line',
|
||||
tickLabelsVisibilitySettings: {
|
||||
x: true,
|
||||
yLeft: true,
|
||||
yRight: true,
|
||||
},
|
||||
valueLabels: 'hide',
|
||||
yLeftExtent: {
|
||||
mode: 'custom',
|
||||
lowerBound: 0,
|
||||
upperBound: undefined,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as TypedLensByValueInput['attributes'];
|
||||
}
|
||||
|
||||
function getAdHocDataViewState(id: string, dataStream: string, title: string) {
|
||||
return {
|
||||
internalReferences: [
|
||||
{
|
||||
id,
|
||||
name: 'indexpattern-datasource-current-indexpattern',
|
||||
type: 'index-pattern',
|
||||
},
|
||||
{
|
||||
id,
|
||||
name: 'indexpattern-datasource-layer-layer1',
|
||||
type: 'index-pattern',
|
||||
},
|
||||
],
|
||||
adHocDataViews: {
|
||||
[id]: {
|
||||
id,
|
||||
title: `${dataStream}${ALL_PATTERNS_SELECTOR}`,
|
||||
timeFieldName: '@timestamp',
|
||||
sourceFilters: [],
|
||||
fieldFormats: {},
|
||||
runtimeFieldMap: {},
|
||||
fieldAttrs: {},
|
||||
allowNoIndex: false,
|
||||
name: `${dataStream}${ALL_PATTERNS_SELECTOR}`,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function getChartColumns(breakdownField?: string): Record<string, GenericIndexPatternColumn> {
|
||||
return {
|
||||
[DatasetQualityLensColumn.Date]: {
|
||||
dataType: 'date',
|
||||
isBucketed: true,
|
||||
label: '@timestamp',
|
||||
operationType: 'date_histogram',
|
||||
params: {
|
||||
interval: 'auto',
|
||||
},
|
||||
scale: 'interval',
|
||||
sourceField: '@timestamp',
|
||||
} as GenericIndexPatternColumn,
|
||||
[DatasetQualityLensColumn.CountFailed]: {
|
||||
label: '',
|
||||
dataType: 'number',
|
||||
operationType: 'count',
|
||||
isBucketed: false,
|
||||
scale: 'ratio',
|
||||
sourceField: '___records___',
|
||||
filter: {
|
||||
query: FAILED_DOCS_QUERY,
|
||||
language: 'kuery',
|
||||
},
|
||||
params: {
|
||||
emptyAsNull: false,
|
||||
},
|
||||
customLabel: true,
|
||||
} as GenericIndexPatternColumn,
|
||||
[DatasetQualityLensColumn.CountTotal]: {
|
||||
label: '',
|
||||
dataType: 'number',
|
||||
operationType: 'count',
|
||||
isBucketed: false,
|
||||
scale: 'ratio',
|
||||
sourceField: '___records___',
|
||||
params: {
|
||||
emptyAsNull: false,
|
||||
},
|
||||
customLabel: true,
|
||||
} as GenericIndexPatternColumn,
|
||||
[DatasetQualityLensColumn.Math]: {
|
||||
label: '',
|
||||
dataType: 'number',
|
||||
operationType: 'math',
|
||||
isBucketed: false,
|
||||
scale: 'ratio',
|
||||
params: {
|
||||
tinymathAst: {
|
||||
type: 'function',
|
||||
name: 'divide',
|
||||
args: ['count_failed', 'count_total'],
|
||||
location: {
|
||||
min: 0,
|
||||
max: 34,
|
||||
},
|
||||
text: `count(kql='${FAILED_DOCS_QUERY}') / count()`,
|
||||
},
|
||||
},
|
||||
references: ['count_failed', 'count_total'],
|
||||
customLabel: true,
|
||||
} as GenericIndexPatternColumn,
|
||||
[DatasetQualityLensColumn.FailedDocs]: {
|
||||
label: flyoutFailedDocsPercentageText,
|
||||
customLabel: true,
|
||||
operationType: 'formula',
|
||||
dataType: 'number',
|
||||
references: [DatasetQualityLensColumn.Math],
|
||||
isBucketed: false,
|
||||
params: {
|
||||
formula: `count(kql='${FAILED_DOCS_QUERY}') / count()`,
|
||||
format: {
|
||||
id: 'percent',
|
||||
params: {
|
||||
decimals: 3,
|
||||
},
|
||||
},
|
||||
isFormulaBroken: false,
|
||||
},
|
||||
} as GenericIndexPatternColumn,
|
||||
...(breakdownField
|
||||
? {
|
||||
[DatasetQualityLensColumn.Breakdown]: {
|
||||
dataType: 'number',
|
||||
isBucketed: true,
|
||||
label: getFlyoutFailedDocsTopNText(MAX_BREAKDOWN_SERIES, breakdownField),
|
||||
operationType: 'terms',
|
||||
scale: 'ordinal',
|
||||
sourceField: breakdownField,
|
||||
params: {
|
||||
size: MAX_BREAKDOWN_SERIES,
|
||||
orderBy: {
|
||||
type: 'alphabetical',
|
||||
fallback: true,
|
||||
},
|
||||
orderDirection: 'desc',
|
||||
otherBucket: true,
|
||||
missingBucket: false,
|
||||
parentFormat: {
|
||||
id: 'terms',
|
||||
},
|
||||
},
|
||||
} as GenericIndexPatternColumn,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
const getFlyoutFailedDocsTopNText = (count: number, fieldName: string) =>
|
||||
i18n.translate('xpack.datasetQuality.details.failedDocsTopNValues', {
|
||||
defaultMessage: 'Top {count} values of {fieldName}',
|
||||
values: { count, fieldName },
|
||||
description:
|
||||
'Tooltip label for the top N values of a field in the failed documents trend chart.',
|
||||
});
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
EuiAccordion,
|
||||
EuiButtonGroup,
|
||||
EuiButtonIcon,
|
||||
EuiCode,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiIcon,
|
||||
EuiPanel,
|
||||
EuiSkeletonRectangle,
|
||||
EuiSpacer,
|
||||
EuiTitle,
|
||||
EuiToolTip,
|
||||
OnTimeChangeProps,
|
||||
useGeneratedHtmlId,
|
||||
} from '@elastic/eui';
|
||||
import { css } from '@emotion/react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public';
|
||||
import React, { useCallback } from 'react';
|
||||
import {
|
||||
discoverAriaText,
|
||||
openInDiscoverText,
|
||||
overviewPanelDatasetQualityIndicatorDegradedDocs,
|
||||
overviewTrendsDocsText,
|
||||
} from '../../../../../common/translations';
|
||||
import { useDatasetQualityDetailsState, useQualityIssuesDocsChart } from '../../../../hooks';
|
||||
import { QualityIssueType } from '../../../../state_machines/dataset_quality_details_controller';
|
||||
import { useDatasetQualityDetailsContext } from '../../context';
|
||||
import { TrendDocsChart } from './trend_docs_chart';
|
||||
|
||||
const trendDocsTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.details.trendDocsTooltip"
|
||||
defaultMessage="The percentage of ignored fields or failed docs over the selected timeframe."
|
||||
/>
|
||||
);
|
||||
|
||||
const degradedDocsTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.details.degradedDocsTooltip"
|
||||
defaultMessage="The number of degraded documents —documents with the {ignoredProperty} property— in your data set."
|
||||
values={{
|
||||
ignoredProperty: (
|
||||
<EuiCode language="json" transparentBackground>
|
||||
_ignored
|
||||
</EuiCode>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DocumentTrends({ lastReloadTime }: { lastReloadTime: number }) {
|
||||
const { isServerless } = useDatasetQualityDetailsContext();
|
||||
const { timeRange, updateTimeRange, docsTrendChart } = useDatasetQualityDetailsState();
|
||||
const {
|
||||
dataView,
|
||||
breakdown,
|
||||
redirectLinkProps,
|
||||
handleDocsTrendChartChange,
|
||||
...qualityIssuesChartProps
|
||||
} = useQualityIssuesDocsChart();
|
||||
|
||||
const accordionId = useGeneratedHtmlId({
|
||||
prefix: overviewTrendsDocsText,
|
||||
});
|
||||
|
||||
const onTimeRangeChange = useCallback(
|
||||
({ start, end }: Pick<OnTimeChangeProps, 'start' | 'end'>) => {
|
||||
updateTimeRange({ start, end, refreshInterval: timeRange.refresh.value });
|
||||
},
|
||||
[updateTimeRange, timeRange.refresh]
|
||||
);
|
||||
|
||||
const accordionTitle = isServerless ? (
|
||||
<EuiFlexItem
|
||||
css={css`
|
||||
flex-direction: row;
|
||||
justify-content: flex-start;
|
||||
align-items: flex-start;
|
||||
gap: 4px;
|
||||
`}
|
||||
>
|
||||
<EuiTitle size={'xxs'}>
|
||||
<h5>{overviewPanelDatasetQualityIndicatorDegradedDocs}</h5>
|
||||
</EuiTitle>
|
||||
<EuiToolTip content={degradedDocsTooltip}>
|
||||
<EuiIcon size="m" color="subdued" type="questionInCircle" className="eui-alignTop" />
|
||||
</EuiToolTip>
|
||||
</EuiFlexItem>
|
||||
) : (
|
||||
<EuiFlexItem
|
||||
css={css`
|
||||
flex-direction: row;
|
||||
justify-content: flex-start;
|
||||
align-items: flex-start;
|
||||
gap: 4px;
|
||||
`}
|
||||
>
|
||||
<EuiTitle size={'xxs'}>
|
||||
<h5>{overviewTrendsDocsText}</h5>
|
||||
</EuiTitle>
|
||||
<EuiToolTip content={trendDocsTooltip}>
|
||||
<EuiIcon size="m" color="subdued" type="questionInCircle" className="eui-alignTop" />
|
||||
</EuiToolTip>
|
||||
</EuiFlexItem>
|
||||
);
|
||||
|
||||
return (
|
||||
<EuiPanel hasBorder grow={false}>
|
||||
<EuiAccordion
|
||||
id={accordionId}
|
||||
buttonContent={accordionTitle}
|
||||
paddingSize="none"
|
||||
initialIsOpen={true}
|
||||
data-test-subj="datasetQualityDetailsOverviewDocumentTrends"
|
||||
>
|
||||
<EuiSpacer size="m" />
|
||||
<EuiFlexGroup justifyContent="flexEnd" gutterSize="s">
|
||||
<EuiFlexItem>
|
||||
{!isServerless && (
|
||||
<EuiButtonGroup
|
||||
data-test-subj="datasetQualityDetailsChartTypeButtonGroup"
|
||||
legend={i18n.translate('xpack.datasetQuality.details.chartTypeLegend', {
|
||||
defaultMessage: 'Quality chart type',
|
||||
})}
|
||||
onChange={(id) => handleDocsTrendChartChange(id as QualityIssueType)}
|
||||
options={[
|
||||
{
|
||||
id: 'degraded',
|
||||
label: i18n.translate('xpack.datasetQuality.details.chartType.degradedDocs', {
|
||||
defaultMessage: 'Ignored fields',
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: 'failed',
|
||||
label: i18n.translate('xpack.datasetQuality.details.chartType.failedDocs', {
|
||||
defaultMessage: 'Failed docs',
|
||||
}),
|
||||
},
|
||||
]}
|
||||
idSelected={docsTrendChart}
|
||||
/>
|
||||
)}
|
||||
</EuiFlexItem>
|
||||
<EuiSkeletonRectangle width={160} height={32} isLoading={!dataView}>
|
||||
<UnifiedBreakdownFieldSelector
|
||||
dataView={dataView!}
|
||||
breakdown={{
|
||||
field:
|
||||
breakdown.dataViewField && breakdown.fieldSupportsBreakdown
|
||||
? breakdown.dataViewField
|
||||
: undefined,
|
||||
}}
|
||||
onBreakdownFieldChange={breakdown.onChange}
|
||||
/>
|
||||
</EuiSkeletonRectangle>
|
||||
<EuiToolTip content={openInDiscoverText}>
|
||||
<EuiButtonIcon
|
||||
display="base"
|
||||
iconType="discoverApp"
|
||||
aria-label={discoverAriaText}
|
||||
size="s"
|
||||
data-test-subj="datasetQualityDetailsLinkToDiscover"
|
||||
{...redirectLinkProps.linkProps}
|
||||
/>
|
||||
</EuiToolTip>
|
||||
</EuiFlexGroup>
|
||||
<EuiSpacer size="m" />
|
||||
<TrendDocsChart
|
||||
{...qualityIssuesChartProps}
|
||||
timeRange={timeRange}
|
||||
lastReloadTime={lastReloadTime}
|
||||
onTimeRangeChange={onTimeRangeChange}
|
||||
/>
|
||||
</EuiAccordion>
|
||||
</EuiPanel>
|
||||
);
|
||||
}
|
||||
|
|
@ -11,10 +11,10 @@ import { EuiFlexGroup, EuiLoadingChart, OnTimeChangeProps } from '@elastic/eui';
|
|||
import { ViewMode } from '@kbn/embeddable-plugin/common';
|
||||
import { KibanaErrorBoundary } from '@kbn/shared-ux-error-boundary';
|
||||
|
||||
import { flyoutDegradedDocsTrendText } from '../../../../../../common/translations';
|
||||
import { useKibanaContextForPlugin } from '../../../../../utils';
|
||||
import { TimeRangeConfig } from '../../../../../../common/types';
|
||||
import { useDegradedDocsChart } from '../../../../../hooks';
|
||||
import { flyoutDegradedDocsTrendText } from '../../../../../common/translations';
|
||||
import { useKibanaContextForPlugin } from '../../../../utils';
|
||||
import { TimeRangeConfig } from '../../../../../common/types';
|
||||
import { useQualityIssuesDocsChart } from '../../../../hooks';
|
||||
|
||||
const CHART_HEIGHT = 180;
|
||||
const DISABLED_ACTIONS = [
|
||||
|
|
@ -24,9 +24,9 @@ const DISABLED_ACTIONS = [
|
|||
'create-ml-ad-job-action',
|
||||
];
|
||||
|
||||
interface DegradedDocsChartProps
|
||||
interface TrendDocsChartProps
|
||||
extends Pick<
|
||||
ReturnType<typeof useDegradedDocsChart>,
|
||||
ReturnType<typeof useQualityIssuesDocsChart>,
|
||||
'attributes' | 'isChartLoading' | 'onChartLoading' | 'extraActions'
|
||||
> {
|
||||
timeRange: TimeRangeConfig;
|
||||
|
|
@ -34,7 +34,7 @@ interface DegradedDocsChartProps
|
|||
onTimeRangeChange: (props: Pick<OnTimeChangeProps, 'start' | 'end'>) => void;
|
||||
}
|
||||
|
||||
export function DegradedDocsChart({
|
||||
export function TrendDocsChart({
|
||||
attributes,
|
||||
isChartLoading,
|
||||
onChartLoading,
|
||||
|
|
@ -42,7 +42,7 @@ export function DegradedDocsChart({
|
|||
timeRange,
|
||||
lastReloadTime,
|
||||
onTimeRangeChange,
|
||||
}: DegradedDocsChartProps) {
|
||||
}: TrendDocsChartProps) {
|
||||
const {
|
||||
services: { lens },
|
||||
} = useKibanaContextForPlugin();
|
||||
|
|
@ -10,11 +10,11 @@ import { dynamic } from '@kbn/shared-ux-utility';
|
|||
import { EuiSpacer, OnRefreshProps } from '@elastic/eui';
|
||||
import { useDatasetQualityDetailsState } from '../../../hooks';
|
||||
import { AggregationNotSupported } from './aggregation_not_supported';
|
||||
import { DegradedFields } from './degraded_fields';
|
||||
import { QualityIssues } from './quality_issues';
|
||||
|
||||
const OverviewHeader = dynamic(() => import('./header'));
|
||||
const Summary = dynamic(() => import('./summary'));
|
||||
const DegradedDocs = dynamic(() => import('./document_trends/degraded_docs'));
|
||||
const DocumentTrends = dynamic(() => import('./document_trends'));
|
||||
|
||||
export function Overview() {
|
||||
const { dataStream, isNonAggregatable, updateTimeRange } = useDatasetQualityDetailsState();
|
||||
|
|
@ -34,9 +34,9 @@ export function Overview() {
|
|||
<EuiSpacer size="m" />
|
||||
<Summary />
|
||||
<EuiSpacer size="m" />
|
||||
<DegradedDocs lastReloadTime={lastReloadTime} />
|
||||
<DocumentTrends lastReloadTime={lastReloadTime} />
|
||||
<EuiSpacer size="m" />
|
||||
<DegradedFields />
|
||||
<QualityIssues />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,59 +5,63 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { EuiBasicTableColumn, EuiButtonIcon, EuiText, formatNumber } from '@elastic/eui';
|
||||
import { css } from '@emotion/react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { EuiBasicTableColumn, EuiButtonIcon } from '@elastic/eui';
|
||||
import type { FieldFormat } from '@kbn/field-formats-plugin/common';
|
||||
import { formatNumber } from '@elastic/eui';
|
||||
|
||||
import { DegradedField } from '../../../../../common/api_types';
|
||||
import { SparkPlot } from '../../../common/spark_plot';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import React from 'react';
|
||||
import { QualityIssue } from '../../../../../common/api_types';
|
||||
import { NUMBER_FORMAT } from '../../../../../common/constants';
|
||||
import {
|
||||
countColumnName,
|
||||
fieldColumnName,
|
||||
documentIndexFailed,
|
||||
issueColumnName,
|
||||
lastOccurrenceColumnName,
|
||||
} from '../../../../../common/translations';
|
||||
import { QualityIssueType } from '../../../../state_machines/dataset_quality_details_controller';
|
||||
import { SparkPlot } from '../../../common/spark_plot';
|
||||
|
||||
const expandDatasetAriaLabel = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldTable.expandLabel',
|
||||
'xpack.datasetQuality.details.qualityIssuesTable.expandLabel',
|
||||
{
|
||||
defaultMessage: 'Expand',
|
||||
}
|
||||
);
|
||||
const collapseDatasetAriaLabel = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldTable.collapseLabel',
|
||||
'xpack.datasetQuality.details.qualityIssuesTable.collapseLabel',
|
||||
{
|
||||
defaultMessage: 'Collapse',
|
||||
}
|
||||
);
|
||||
|
||||
export const getDegradedFieldsColumns = ({
|
||||
export const getQualityIssuesColumns = ({
|
||||
dateFormatter,
|
||||
isLoading,
|
||||
expandedDegradedField,
|
||||
openDegradedFieldFlyout,
|
||||
expandedQualityIssue,
|
||||
openQualityIssueFlyout,
|
||||
}: {
|
||||
dateFormatter: FieldFormat;
|
||||
isLoading: boolean;
|
||||
expandedDegradedField?: string;
|
||||
openDegradedFieldFlyout: (name: string) => void;
|
||||
}): Array<EuiBasicTableColumn<DegradedField>> => [
|
||||
expandedQualityIssue?: {
|
||||
name: string;
|
||||
type: QualityIssueType;
|
||||
};
|
||||
openQualityIssueFlyout: (name: string, type: QualityIssueType) => void;
|
||||
}): Array<EuiBasicTableColumn<QualityIssue>> => [
|
||||
{
|
||||
name: '',
|
||||
field: 'name',
|
||||
render: (_, { name }) => {
|
||||
const isExpanded = name === expandedDegradedField;
|
||||
render: (_, { name, type }) => {
|
||||
const isExpanded = name === expandedQualityIssue?.name && type === expandedQualityIssue?.type;
|
||||
|
||||
const onExpandClick = () => {
|
||||
openDegradedFieldFlyout(name);
|
||||
openQualityIssueFlyout(name, type);
|
||||
};
|
||||
|
||||
return (
|
||||
<EuiButtonIcon
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldsExpandButton"
|
||||
data-test-subj="datasetQualityDetailsQualityIssuesExpandButton"
|
||||
size="xs"
|
||||
color="text"
|
||||
onClick={onExpandClick}
|
||||
|
|
@ -75,8 +79,27 @@ export const getDegradedFieldsColumns = ({
|
|||
`,
|
||||
},
|
||||
{
|
||||
name: fieldColumnName,
|
||||
name: issueColumnName,
|
||||
field: 'name',
|
||||
render: (_, { name, type }) => {
|
||||
return type === 'degraded' ? (
|
||||
<EuiText size="s">
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.details.qualityIssues.degradedField"
|
||||
defaultMessage="{name} field ignored"
|
||||
values={{
|
||||
name: (
|
||||
<>
|
||||
<strong>{name}</strong>{' '}
|
||||
</>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
</EuiText>
|
||||
) : (
|
||||
<>{documentIndexFailed}</>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: countColumnName,
|
||||
|
|
@ -5,4 +5,4 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
export * from './degraded_fields';
|
||||
export * from './quality_issues';
|
||||
|
|
@ -18,28 +18,28 @@ import {
|
|||
EuiSwitch,
|
||||
} from '@elastic/eui';
|
||||
import {
|
||||
overviewDegradedFieldsSectionTitle,
|
||||
overviewDegradedFieldsSectionTitleTooltip,
|
||||
overviewDegradedFieldToggleSwitch,
|
||||
overviewDegradedFieldToggleSwitchTooltip,
|
||||
overviewQualityIssuesSectionTitle,
|
||||
overviewQualityIssueSectionTitleTooltip,
|
||||
currentIssuesToggleSwitch,
|
||||
currentIssuesToggleSwitchTooltip,
|
||||
overviewQualityIssuesAccordionTechPreviewBadge,
|
||||
} from '../../../../../common/translations';
|
||||
import { DegradedFieldTable } from './table';
|
||||
import { useDegradedFields } from '../../../../hooks';
|
||||
import { QualityIssuesTable } from './table';
|
||||
import { useQualityIssues } from '../../../../hooks';
|
||||
|
||||
export function DegradedFields() {
|
||||
export function QualityIssues() {
|
||||
const accordionId = useGeneratedHtmlId({
|
||||
prefix: overviewDegradedFieldsSectionTitle,
|
||||
prefix: overviewQualityIssuesSectionTitle,
|
||||
});
|
||||
const toggleTextSwitchId = useGeneratedHtmlId({ prefix: 'toggleTextSwitch' });
|
||||
|
||||
const { totalItemCount, toggleCurrentQualityIssues, showCurrentQualityIssues } =
|
||||
useDegradedFields();
|
||||
useQualityIssues();
|
||||
|
||||
const latestBackingIndexToggle = (
|
||||
<>
|
||||
<EuiSwitch
|
||||
label={overviewDegradedFieldToggleSwitch}
|
||||
label={currentIssuesToggleSwitch}
|
||||
checked={showCurrentQualityIssues}
|
||||
onChange={toggleCurrentQualityIssues}
|
||||
aria-describedby={toggleTextSwitchId}
|
||||
|
|
@ -47,16 +47,16 @@ export function DegradedFields() {
|
|||
data-test-subj="datasetQualityDetailsOverviewDegradedFieldToggleSwitch"
|
||||
css={{ marginRight: '5px' }}
|
||||
/>
|
||||
<EuiIconTip content={overviewDegradedFieldToggleSwitchTooltip} position="top" />
|
||||
<EuiIconTip content={currentIssuesToggleSwitchTooltip} position="top" />
|
||||
</>
|
||||
);
|
||||
|
||||
const accordionTitle = (
|
||||
<EuiFlexGroup alignItems="center" gutterSize="s" direction="row">
|
||||
<EuiTitle size="xxs">
|
||||
<h6>{overviewDegradedFieldsSectionTitle}</h6>
|
||||
<h6>{overviewQualityIssuesSectionTitle}</h6>
|
||||
</EuiTitle>
|
||||
<EuiIconTip content={overviewDegradedFieldsSectionTitleTooltip} color="subdued" size="m" />
|
||||
<EuiIconTip content={overviewQualityIssueSectionTitleTooltip} color="subdued" size="m" />
|
||||
<EuiBadge
|
||||
color="default"
|
||||
data-test-subj="datasetQualityDetailsOverviewDegradedFieldTitleCount"
|
||||
|
|
@ -81,7 +81,7 @@ export function DegradedFields() {
|
|||
extraAction={latestBackingIndexToggle}
|
||||
data-test-subj="datasetQualityDetailsOverviewDocumentTrends"
|
||||
>
|
||||
<DegradedFieldTable />
|
||||
<QualityIssuesTable />
|
||||
</EuiAccordion>
|
||||
</EuiPanel>
|
||||
);
|
||||
|
|
@ -5,17 +5,17 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { EuiBasicTable, EuiEmptyPrompt } from '@elastic/eui';
|
||||
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
|
||||
import { getDegradedFieldsColumns } from './columns';
|
||||
import React from 'react';
|
||||
import {
|
||||
overviewDegradedFieldsTableLoadingText,
|
||||
overviewDegradedFieldsTableNoData,
|
||||
qualityIssuesTableNoData,
|
||||
} from '../../../../../common/translations';
|
||||
import { useDegradedFields } from '../../../../hooks/use_degraded_fields';
|
||||
import { useQualityIssues } from '../../../../hooks/use_quality_issues';
|
||||
import { getQualityIssuesColumns } from './columns';
|
||||
|
||||
export const DegradedFieldTable = () => {
|
||||
export const QualityIssuesTable = () => {
|
||||
const {
|
||||
isDegradedFieldsLoading,
|
||||
pagination,
|
||||
|
|
@ -25,15 +25,15 @@ export const DegradedFieldTable = () => {
|
|||
fieldFormats,
|
||||
expandedDegradedField,
|
||||
openDegradedFieldFlyout,
|
||||
} = useDegradedFields();
|
||||
} = useQualityIssues();
|
||||
const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
|
||||
ES_FIELD_TYPES.DATE,
|
||||
]);
|
||||
const columns = getDegradedFieldsColumns({
|
||||
const columns = getQualityIssuesColumns({
|
||||
dateFormatter,
|
||||
isLoading: isDegradedFieldsLoading,
|
||||
expandedDegradedField,
|
||||
openDegradedFieldFlyout,
|
||||
expandedQualityIssue: expandedDegradedField,
|
||||
openQualityIssueFlyout: openDegradedFieldFlyout,
|
||||
});
|
||||
|
||||
return (
|
||||
|
|
@ -56,7 +56,7 @@ export const DegradedFieldTable = () => {
|
|||
<EuiEmptyPrompt
|
||||
data-test-subj="datasetQualityDetailsDegradedTableNoData"
|
||||
layout="vertical"
|
||||
title={<h2>{overviewDegradedFieldsTableNoData}</h2>}
|
||||
title={<h2>{qualityIssuesTableNoData}</h2>}
|
||||
hasBorder={false}
|
||||
titleSize="m"
|
||||
/>
|
||||
|
|
@ -5,11 +5,12 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { EuiCode, EuiFlexGroup } from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import React from 'react';
|
||||
import { EuiFlexGroup } from '@elastic/eui';
|
||||
import { Panel, PanelIndicator } from './panel';
|
||||
import {
|
||||
overviewPanelDatasetQualityIndicatorDegradedDocs,
|
||||
overviewPanelDatasetQualityIndicatorFailedDocs,
|
||||
overviewPanelDocumentsIndicatorSize,
|
||||
overviewPanelDocumentsIndicatorTotalCount,
|
||||
overviewPanelResourcesIndicatorServices,
|
||||
|
|
@ -20,10 +21,34 @@ import {
|
|||
} from '../../../../../common/translations';
|
||||
import { useOverviewSummaryPanel } from '../../../../hooks/use_overview_summary_panel';
|
||||
import { DatasetQualityIndicator } from '../../../quality_indicator';
|
||||
import { Panel, PanelIndicator } from './panel';
|
||||
import { useDatasetQualityDetailsContext } from '../../context';
|
||||
|
||||
const degradedDocsTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.details.degradedDocsTooltip"
|
||||
defaultMessage="The number of degraded documents —documents with the {ignoredProperty} property— in your data set."
|
||||
values={{
|
||||
ignoredProperty: (
|
||||
<EuiCode language="json" transparentBackground>
|
||||
_ignored
|
||||
</EuiCode>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
const failedDocsColumnTooltip = (
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.failedDocsSummaryTooltip"
|
||||
defaultMessage="The number of documents sent to failure store due to an issue during ingestion."
|
||||
/>
|
||||
);
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function Summary() {
|
||||
const { isServerless } = useDatasetQualityDetailsContext();
|
||||
const {
|
||||
isSummaryPanelLoading,
|
||||
totalDocsCount,
|
||||
|
|
@ -32,6 +57,7 @@ export default function Summary() {
|
|||
totalServicesCount,
|
||||
totalHostsCount,
|
||||
totalDegradedDocsCount,
|
||||
totalFailedDocsCount,
|
||||
quality,
|
||||
} = useOverviewSummaryPanel();
|
||||
return (
|
||||
|
|
@ -75,7 +101,16 @@ export default function Summary() {
|
|||
label={overviewPanelDatasetQualityIndicatorDegradedDocs}
|
||||
value={totalDegradedDocsCount}
|
||||
isLoading={isSummaryPanelLoading}
|
||||
tooltip={degradedDocsTooltip}
|
||||
/>
|
||||
{!isServerless && (
|
||||
<PanelIndicator
|
||||
label={overviewPanelDatasetQualityIndicatorFailedDocs}
|
||||
value={totalFailedDocsCount}
|
||||
isLoading={isSummaryPanelLoading}
|
||||
tooltip={failedDocsColumnTooltip}
|
||||
/>
|
||||
)}
|
||||
</Panel>
|
||||
</EuiFlexGroup>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -6,10 +6,31 @@
|
|||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSkeletonTitle, EuiText } from '@elastic/eui';
|
||||
import {
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiIcon,
|
||||
EuiPanel,
|
||||
EuiSkeletonTitle,
|
||||
EuiText,
|
||||
EuiToolTip,
|
||||
} from '@elastic/eui';
|
||||
import { css } from '@emotion/react';
|
||||
import { euiThemeVars } from '@kbn/ui-theme';
|
||||
import { PrivilegesWarningIconWrapper } from '../../../common';
|
||||
import { notAvailableLabel } from '../../../../../common/translations';
|
||||
import { VerticalRule } from '../../../common/vertical_rule';
|
||||
|
||||
const verticalRule = css`
|
||||
width: 1px;
|
||||
height: 65px;
|
||||
background-color: ${euiThemeVars.euiColorLightShade};
|
||||
`;
|
||||
|
||||
const verticalRuleHidden = css`
|
||||
width: 1px;
|
||||
height: 65px;
|
||||
visibility: hidden;
|
||||
`;
|
||||
|
||||
export function Panel({
|
||||
title,
|
||||
|
|
@ -25,14 +46,14 @@ export function Panel({
|
|||
return panelChildren.map((panelChild, index) => (
|
||||
<React.Fragment key={index}>
|
||||
{panelChild}
|
||||
{index < panelChildren.length - 1 && <VerticalRule />}
|
||||
{index < panelChildren.length - 1 && <span css={verticalRule} />}
|
||||
</React.Fragment>
|
||||
));
|
||||
}
|
||||
return (
|
||||
<>
|
||||
{panelChildren}
|
||||
<VerticalRule style={{ visibility: 'hidden' }} />
|
||||
<span css={verticalRuleHidden} />
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -58,11 +79,13 @@ export function Panel({
|
|||
export function PanelIndicator({
|
||||
label,
|
||||
value,
|
||||
tooltip,
|
||||
isLoading,
|
||||
userHasPrivilege = true,
|
||||
}: {
|
||||
label: string;
|
||||
value: string | number;
|
||||
tooltip?: React.ReactElement;
|
||||
isLoading: boolean;
|
||||
userHasPrivilege?: boolean;
|
||||
}) {
|
||||
|
|
@ -72,9 +95,23 @@ export function PanelIndicator({
|
|||
<EuiSkeletonTitle size="m" />
|
||||
) : (
|
||||
<>
|
||||
<EuiText size="xs" color="subdued">
|
||||
{label}
|
||||
</EuiText>
|
||||
{tooltip ? (
|
||||
<EuiToolTip content={tooltip}>
|
||||
<EuiText size="xs" color="subdued">
|
||||
{`${label} `}
|
||||
<EuiIcon
|
||||
size="s"
|
||||
color="subdued"
|
||||
type="questionInCircle"
|
||||
className="eui-alignTop"
|
||||
/>
|
||||
</EuiText>
|
||||
</EuiToolTip>
|
||||
) : (
|
||||
<EuiText size="xs" color="subdued">
|
||||
{label}
|
||||
</EuiText>
|
||||
)}
|
||||
<PrivilegesWarningIconWrapper
|
||||
hasPrivileges={userHasPrivilege}
|
||||
title={label}
|
||||
|
|
|
|||
|
|
@ -15,77 +15,27 @@ import {
|
|||
EuiTextColor,
|
||||
EuiTitle,
|
||||
EuiToolTip,
|
||||
formatNumber,
|
||||
useEuiTheme,
|
||||
} from '@elastic/eui';
|
||||
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
|
||||
|
||||
import { NUMBER_FORMAT } from '../../../../common/constants';
|
||||
import { useQualityIssues } from '../../../../hooks';
|
||||
import {
|
||||
countColumnName,
|
||||
degradedFieldCurrentFieldLimitColumnName,
|
||||
degradedFieldMaximumCharacterLimitColumnName,
|
||||
degradedFieldPotentialCauseColumnName,
|
||||
degradedFieldValuesColumnName,
|
||||
lastOccurrenceColumnName,
|
||||
} from '../../../../common/translations';
|
||||
import { useDegradedFields } from '../../../hooks';
|
||||
import { SparkPlot } from '../../common/spark_plot';
|
||||
import { DegradedField } from '../../../../common/api_types';
|
||||
} from '../../../../../common/translations';
|
||||
|
||||
export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) => {
|
||||
export const DegradedFieldInfo = () => {
|
||||
const { euiTheme } = useEuiTheme();
|
||||
|
||||
const {
|
||||
fieldFormats,
|
||||
degradedFieldValues,
|
||||
isDegradedFieldsLoading,
|
||||
isAnalysisInProgress,
|
||||
degradedFieldAnalysisFormattedResult,
|
||||
degradedFieldAnalysis,
|
||||
} = useDegradedFields();
|
||||
|
||||
const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
|
||||
ES_FIELD_TYPES.DATE,
|
||||
]);
|
||||
} = useQualityIssues();
|
||||
|
||||
return (
|
||||
<EuiFlexGroup direction="column" gutterSize="none">
|
||||
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount`}>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{countColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-docCount"
|
||||
grow={2}
|
||||
>
|
||||
<SparkPlot
|
||||
series={fieldList?.timeSeries}
|
||||
valueLabel={formatNumber(fieldList?.count, NUMBER_FORMAT)}
|
||||
isLoading={isDegradedFieldsLoading}
|
||||
/>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
<EuiFlexGroup
|
||||
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-lastOccurrence`}
|
||||
>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{lastOccurrenceColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-lastOccurrence"
|
||||
grow={2}
|
||||
>
|
||||
<span>{dateFormatter.convert(fieldList?.lastOccurrence)}</span>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
|
||||
<>
|
||||
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-cause`}>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
|
|
@ -178,6 +128,6 @@ export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField })
|
|||
<EuiHorizontalRule margin="s" />
|
||||
</>
|
||||
)}
|
||||
</EuiFlexGroup>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useMemo } from 'react';
|
||||
import { EuiSpacer } from '@elastic/eui';
|
||||
import { useDatasetQualityDetailsState, useQualityIssues } from '../../../../hooks';
|
||||
import { QualityIssueFieldInfo } from '../field_info';
|
||||
import { PossibleDegradedFieldMitigations } from './possible_mitigations';
|
||||
import { DegradedFieldInfo } from './field_info';
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DegradedFieldFlyout() {
|
||||
const { expandedDegradedField, renderedItems } = useQualityIssues();
|
||||
const { dataStreamSettings } = useDatasetQualityDetailsState();
|
||||
|
||||
const fieldList = useMemo(() => {
|
||||
return renderedItems.find((item) => {
|
||||
return item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type;
|
||||
});
|
||||
}, [renderedItems, expandedDegradedField]);
|
||||
|
||||
const isUserViewingTheIssueOnLatestBackingIndex =
|
||||
dataStreamSettings?.lastBackingIndexName === fieldList?.indexFieldWasLastPresentIn;
|
||||
|
||||
return (
|
||||
<>
|
||||
<QualityIssueFieldInfo fieldList={fieldList}>
|
||||
<DegradedFieldInfo />
|
||||
</QualityIssueFieldInfo>
|
||||
{isUserViewingTheIssueOnLatestBackingIndex && (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<PossibleDegradedFieldMitigations />
|
||||
</>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -7,8 +7,8 @@
|
|||
|
||||
import React from 'react';
|
||||
import { EuiLink } from '@elastic/eui';
|
||||
import { useKibanaContextForPlugin } from '../../../../../utils';
|
||||
import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../common/translations';
|
||||
import { useKibanaContextForPlugin } from '../../../../../../utils';
|
||||
import { fieldLimitMitigationOfficialDocumentation } from '../../../../../../../common/translations';
|
||||
|
||||
export function FieldLimitDocLink() {
|
||||
const {
|
||||
|
|
@ -23,8 +23,8 @@ import {
|
|||
fieldLimitMitigationConsiderationText4,
|
||||
fieldLimitMitigationDescriptionText,
|
||||
increaseFieldMappingLimitTitle,
|
||||
} from '../../../../../../common/translations';
|
||||
import { useDegradedFields } from '../../../../../hooks';
|
||||
} from '../../../../../../../common/translations';
|
||||
import { useQualityIssues } from '../../../../../../hooks';
|
||||
import { IncreaseFieldMappingLimit } from './increase_field_mapping_limit';
|
||||
import { FieldLimitDocLink } from './field_limit_documentation_link';
|
||||
import { MessageCallout } from './message_callout';
|
||||
|
|
@ -38,7 +38,7 @@ export function FieldMappingLimit({
|
|||
prefix: increaseFieldMappingLimitTitle,
|
||||
});
|
||||
|
||||
const { degradedFieldAnalysis } = useDegradedFields();
|
||||
const { degradedFieldAnalysis } = useQualityIssues();
|
||||
|
||||
const accordionTitle = (
|
||||
<EuiTitle size="xxs">
|
||||
|
|
@ -19,15 +19,15 @@ import {
|
|||
fieldLimitMitigationCurrentLimitLabelText,
|
||||
fieldLimitMitigationNewLimitButtonText,
|
||||
fieldLimitMitigationNewLimitPlaceholderText,
|
||||
} from '../../../../../../common/translations';
|
||||
import { useDegradedFields } from '../../../../../hooks';
|
||||
} from '../../../../../../../common/translations';
|
||||
import { useQualityIssues } from '../../../../../../hooks';
|
||||
|
||||
export function IncreaseFieldMappingLimit({ totalFieldLimit }: { totalFieldLimit: number }) {
|
||||
// Propose the user a 30% increase over the current limit
|
||||
const proposedNewLimit = Math.round(totalFieldLimit * 1.3);
|
||||
const [newFieldLimit, setNewFieldLimit] = useState<number>(proposedNewLimit);
|
||||
const [isInvalid, setIsInvalid] = useState(false);
|
||||
const { updateNewFieldLimit, isMitigationInProgress } = useDegradedFields();
|
||||
const { updateNewFieldLimit, isMitigationInProgress } = useQualityIssues();
|
||||
|
||||
const validateNewLimit = (newLimit: string) => {
|
||||
const parsedLimit = parseInt(newLimit, 10);
|
||||
|
|
@ -15,10 +15,10 @@ import {
|
|||
fieldLimitMitigationRolloverButton,
|
||||
fieldLimitMitigationSuccessComponentTemplateLinkText,
|
||||
fieldLimitMitigationSuccessMessage,
|
||||
} from '../../../../../../common/translations';
|
||||
import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../../hooks';
|
||||
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name';
|
||||
import { useKibanaContextForPlugin } from '../../../../../utils';
|
||||
} from '../../../../../../../common/translations';
|
||||
import { useDatasetQualityDetailsState, useQualityIssues } from '../../../../../../hooks';
|
||||
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../../common/utils/component_template_name';
|
||||
import { useKibanaContextForPlugin } from '../../../../../../utils';
|
||||
|
||||
export function MessageCallout() {
|
||||
const {
|
||||
|
|
@ -26,7 +26,7 @@ export function MessageCallout() {
|
|||
newFieldLimitData,
|
||||
isRolloverRequired,
|
||||
isMitigationAppliedSuccessfully,
|
||||
} = useDegradedFields();
|
||||
} = useQualityIssues();
|
||||
const { error: serverError } = newFieldLimitData ?? {};
|
||||
|
||||
if (serverError) {
|
||||
|
|
@ -82,7 +82,7 @@ export function SuccessCallout() {
|
|||
}
|
||||
|
||||
export function ManualRolloverCallout() {
|
||||
const { triggerRollover, isRolloverInProgress } = useDegradedFields();
|
||||
const { triggerRollover, isRolloverInProgress } = useQualityIssues();
|
||||
return (
|
||||
<EuiCallOut
|
||||
title={fieldLimitMitigationPartiallyFailedMessage}
|
||||
|
|
@ -7,19 +7,22 @@
|
|||
|
||||
import React from 'react';
|
||||
import { EuiSpacer } from '@elastic/eui';
|
||||
import { ManualMitigations } from './manual';
|
||||
import { FieldMappingLimit } from './field_limit/field_mapping_limit';
|
||||
import { useDatasetQualityDetailsState, useDegradedFields } from '../../../../hooks';
|
||||
import { useDatasetQualityDetailsState, useQualityIssues } from '../../../../../hooks';
|
||||
import { ManualMitigations } from './manual';
|
||||
import { PossibleMitigationTitle } from './title';
|
||||
|
||||
export function PossibleMitigations() {
|
||||
const { degradedFieldAnalysis, isAnalysisInProgress } = useDegradedFields();
|
||||
export function PossibleDegradedFieldMitigations() {
|
||||
const { degradedFieldAnalysis, isAnalysisInProgress } = useQualityIssues();
|
||||
const { integrationDetails } = useDatasetQualityDetailsState();
|
||||
const areIntegrationAssetsAvailable = !!integrationDetails?.integration?.areAssetsAvailable;
|
||||
const areIntegrationAssetsAvailable = Boolean(
|
||||
integrationDetails?.integration?.areAssetsAvailable
|
||||
);
|
||||
|
||||
return (
|
||||
!isAnalysisInProgress && (
|
||||
<div>
|
||||
<EuiSpacer size="s" />
|
||||
<PossibleMitigationTitle />
|
||||
<EuiSpacer size="m" />
|
||||
{degradedFieldAnalysis?.isFieldLimitIssue && (
|
||||
|
|
@ -8,10 +8,10 @@
|
|||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import { MANAGEMENT_APP_ID } from '@kbn/deeplinks-management/constants';
|
||||
import { EuiFlexGroup, EuiIcon, EuiLink, EuiPanel, EuiTitle } from '@elastic/eui';
|
||||
import { useKibanaContextForPlugin } from '../../../../../utils';
|
||||
import { useDatasetQualityDetailsState } from '../../../../../hooks';
|
||||
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../common/utils/component_template_name';
|
||||
import { otherMitigationsCustomComponentTemplate } from '../../../../../../common/translations';
|
||||
import { useKibanaContextForPlugin } from '../../../../../../utils';
|
||||
import { useDatasetQualityDetailsState } from '../../../../../../hooks';
|
||||
import { getComponentTemplatePrefixFromIndexTemplate } from '../../../../../../../common/utils/component_template_name';
|
||||
import { otherMitigationsCustomComponentTemplate } from '../../../../../../../common/translations';
|
||||
|
||||
export function CreateEditComponentTemplateLink({
|
||||
areIntegrationAssetsAvailable,
|
||||
|
|
@ -7,10 +7,10 @@
|
|||
|
||||
import React from 'react';
|
||||
import { EuiSkeletonRectangle, EuiSpacer } from '@elastic/eui';
|
||||
import { useDatasetQualityDetailsState } from '../../../../../hooks';
|
||||
import { useDatasetQualityDetailsState } from '../../../../../../hooks';
|
||||
import { CreateEditComponentTemplateLink } from './component_template_link';
|
||||
import { CreateEditPipelineLink } from './pipeline_link';
|
||||
import { otherMitigationsLoadingAriaText } from '../../../../../../common/translations';
|
||||
import { otherMitigationsLoadingAriaText } from '../../../../../../../common/translations';
|
||||
|
||||
export function ManualMitigations() {
|
||||
const {
|
||||
|
|
@ -24,9 +24,9 @@ import {
|
|||
manualMitigationCustomPipelineCopyPipelineNameAriaText,
|
||||
manualMitigationCustomPipelineCreateEditPipelineLink,
|
||||
otherMitigationsCustomIngestPipeline,
|
||||
} from '../../../../../../common/translations';
|
||||
import { useKibanaContextForPlugin } from '../../../../../utils';
|
||||
import { useDatasetQualityDetailsState } from '../../../../../hooks';
|
||||
} from '../../../../../../../common/translations';
|
||||
import { useKibanaContextForPlugin } from '../../../../../../utils';
|
||||
import { useDatasetQualityDetailsState } from '../../../../../../hooks';
|
||||
|
||||
const AccordionTitle = () => (
|
||||
<EuiTitle size="xxs">
|
||||
|
|
@ -11,7 +11,7 @@ import { EuiBetaBadge, EuiFlexGroup, EuiIcon, EuiTitle } from '@elastic/eui';
|
|||
import {
|
||||
overviewQualityIssuesAccordionTechPreviewBadge,
|
||||
possibleMitigationTitle,
|
||||
} from '../../../../../common/translations';
|
||||
} from '../../../../../../common/translations';
|
||||
|
||||
export function PossibleMitigationTitle() {
|
||||
return (
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { EuiBadge, EuiBasicTableColumn, EuiCode, EuiIcon, EuiToolTip } from '@elastic/eui';
|
||||
import { FailedDocsError } from '../../../../../common/api_types';
|
||||
|
||||
const contentColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.contentLabel',
|
||||
{
|
||||
defaultMessage: 'Content',
|
||||
}
|
||||
);
|
||||
|
||||
const typeColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.typeLabel',
|
||||
{
|
||||
defaultMessage: 'Type',
|
||||
}
|
||||
);
|
||||
|
||||
const typeColumnTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.typeTooltip',
|
||||
{
|
||||
defaultMessage: 'Error message category',
|
||||
}
|
||||
);
|
||||
|
||||
export const getFailedDocsErrorsColumns = (): Array<EuiBasicTableColumn<FailedDocsError>> => [
|
||||
{
|
||||
name: contentColumnName,
|
||||
field: 'message',
|
||||
render: (_, { message }) => {
|
||||
return <EuiCode language="js">{message}</EuiCode>;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: (
|
||||
<EuiToolTip content={typeColumnTooltip}>
|
||||
<span>
|
||||
{`${typeColumnName} `}
|
||||
<EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
|
||||
</span>
|
||||
</EuiToolTip>
|
||||
),
|
||||
field: 'type',
|
||||
render: (_, { type }) => {
|
||||
return (
|
||||
<EuiBadge color="hollow">
|
||||
<strong>{type}</strong>
|
||||
</EuiBadge>
|
||||
);
|
||||
},
|
||||
sortable: true,
|
||||
},
|
||||
];
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
EuiBasicTable,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiHorizontalRule,
|
||||
EuiSpacer,
|
||||
EuiText,
|
||||
EuiTitle,
|
||||
} from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import React from 'react';
|
||||
import {
|
||||
failedDocsErrorsColumnName,
|
||||
overviewDegradedFieldsTableLoadingText,
|
||||
} from '../../../../../common/translations';
|
||||
import { useQualityIssues } from '../../../../hooks';
|
||||
|
||||
const failedDocsErrorsTableNoData = i18n.translate(
|
||||
'xpack.datasetQuality.details.qualityIssue.failedDocs.erros.noData',
|
||||
{
|
||||
defaultMessage: 'No errors found',
|
||||
}
|
||||
);
|
||||
|
||||
export const FailedFieldInfo = () => {
|
||||
const {
|
||||
isDegradedFieldsLoading,
|
||||
failedDocsErrorsColumns,
|
||||
renderedFailedDocsErrorsItems,
|
||||
failedDocsErrorsSort,
|
||||
isFailedDocsErrorsLoading,
|
||||
resultsCount,
|
||||
failedDocsErrorsPagination,
|
||||
onFailedDocsErrorsTableChange,
|
||||
} = useQualityIssues();
|
||||
|
||||
return (
|
||||
<>
|
||||
<EuiFlexGroup
|
||||
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-cause`}
|
||||
direction="column"
|
||||
gutterSize="xs"
|
||||
>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{failedDocsErrorsColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause"
|
||||
grow={2}
|
||||
>
|
||||
<EuiSpacer size="m" />
|
||||
<EuiText size="xs">
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.qualityIssueFlyout.table"
|
||||
defaultMessage="Showing {items}"
|
||||
values={{
|
||||
items: resultsCount,
|
||||
}}
|
||||
/>
|
||||
</EuiText>
|
||||
<EuiHorizontalRule margin="xs" />
|
||||
<EuiBasicTable
|
||||
tableLayout="fixed"
|
||||
columns={failedDocsErrorsColumns}
|
||||
items={renderedFailedDocsErrorsItems ?? []}
|
||||
loading={isFailedDocsErrorsLoading}
|
||||
sorting={failedDocsErrorsSort}
|
||||
onChange={onFailedDocsErrorsTableChange}
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldTable"
|
||||
rowProps={{
|
||||
'data-test-subj': 'datasetQualityDetailsDegradedTableRow',
|
||||
}}
|
||||
noItemsMessage={
|
||||
isDegradedFieldsLoading
|
||||
? overviewDegradedFieldsTableLoadingText
|
||||
: failedDocsErrorsTableNoData
|
||||
}
|
||||
pagination={failedDocsErrorsPagination}
|
||||
/>
|
||||
</EuiFlexItem>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
</EuiFlexGroup>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useMemo } from 'react';
|
||||
import { EuiSpacer } from '@elastic/eui';
|
||||
import { useQualityIssues } from '../../../../hooks';
|
||||
import { QualityIssueFieldInfo } from '../field_info';
|
||||
import { FailedFieldInfo } from './filed_info';
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function FailedDocsFlyout() {
|
||||
const { expandedDegradedField, renderedItems } = useQualityIssues();
|
||||
|
||||
const fieldList = useMemo(() => {
|
||||
return renderedItems.find((item) => {
|
||||
return item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type;
|
||||
});
|
||||
}, [renderedItems, expandedDegradedField]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<QualityIssueFieldInfo fieldList={fieldList}>
|
||||
<FailedFieldInfo />
|
||||
</QualityIssueFieldInfo>
|
||||
<EuiSpacer size="s" />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule, EuiTitle, formatNumber } from '@elastic/eui';
|
||||
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
|
||||
|
||||
import { NUMBER_FORMAT } from '../../../../common/constants';
|
||||
import { countColumnName, lastOccurrenceColumnName } from '../../../../common/translations';
|
||||
import { useQualityIssues } from '../../../hooks';
|
||||
import { SparkPlot } from '../../common/spark_plot';
|
||||
import { QualityIssue } from '../../../../common/api_types';
|
||||
|
||||
export const QualityIssueFieldInfo = ({
|
||||
fieldList,
|
||||
children,
|
||||
}: {
|
||||
fieldList?: QualityIssue;
|
||||
children?: React.ReactNode;
|
||||
}) => {
|
||||
const { fieldFormats, isDegradedFieldsLoading } = useQualityIssues();
|
||||
|
||||
const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
|
||||
ES_FIELD_TYPES.DATE,
|
||||
]);
|
||||
|
||||
return (
|
||||
<EuiFlexGroup direction="column" gutterSize="none">
|
||||
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount`}>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{countColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-docCount"
|
||||
grow={2}
|
||||
>
|
||||
<SparkPlot
|
||||
series={fieldList?.timeSeries}
|
||||
valueLabel={formatNumber(fieldList?.count, NUMBER_FORMAT)}
|
||||
isLoading={isDegradedFieldsLoading}
|
||||
/>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
<EuiFlexGroup
|
||||
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-lastOccurrence`}
|
||||
>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{lastOccurrenceColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-lastOccurrence"
|
||||
grow={2}
|
||||
>
|
||||
<span>{dateFormatter.convert(fieldList?.lastOccurrence)}</span>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
{children}
|
||||
</EuiFlexGroup>
|
||||
);
|
||||
};
|
||||
|
|
@ -5,51 +5,52 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useMemo } from 'react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import {
|
||||
EuiBadge,
|
||||
EuiButtonIcon,
|
||||
EuiFlexGroup,
|
||||
EuiFlyout,
|
||||
EuiFlyoutHeader,
|
||||
EuiFlyoutBody,
|
||||
EuiFlyoutHeader,
|
||||
EuiSpacer,
|
||||
EuiText,
|
||||
EuiTitle,
|
||||
useGeneratedHtmlId,
|
||||
EuiTextColor,
|
||||
EuiFlexGroup,
|
||||
EuiButtonIcon,
|
||||
EuiTitle,
|
||||
EuiToolTip,
|
||||
useGeneratedHtmlId,
|
||||
} from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import { NavigationSource } from '../../../services/telemetry';
|
||||
import {
|
||||
useDatasetDetailsRedirectLinkTelemetry,
|
||||
useDatasetQualityDetailsState,
|
||||
useDegradedFields,
|
||||
useRedirectLink,
|
||||
} from '../../../hooks';
|
||||
import React, { useMemo } from 'react';
|
||||
import { DEGRADED_DOCS_QUERY, FAILURE_STORE_SELECTOR } from '../../../../common/constants';
|
||||
import { _IGNORED } from '../../../../common/es_fields';
|
||||
import {
|
||||
degradedFieldMessageIssueDoesNotExistInLatestIndex,
|
||||
discoverAriaText,
|
||||
fieldIgnoredText,
|
||||
openInDiscoverText,
|
||||
overviewDegradedFieldsSectionTitle,
|
||||
overviewQualityIssuesSectionTitle,
|
||||
} from '../../../../common/translations';
|
||||
import { DegradedFieldInfo } from './field_info';
|
||||
import { _IGNORED } from '../../../../common/es_fields';
|
||||
import { PossibleMitigations } from './possible_mitigations';
|
||||
import {
|
||||
useDatasetDetailsRedirectLinkTelemetry,
|
||||
useDatasetQualityDetailsState,
|
||||
useQualityIssues,
|
||||
useRedirectLink,
|
||||
} from '../../../hooks';
|
||||
import { NavigationSource } from '../../../services/telemetry';
|
||||
import DegradedFieldFlyout from './degraded_field';
|
||||
import FailedDocsFlyout from './failed_docs';
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DegradedFieldFlyout() {
|
||||
export default function QualityIssueFlyout() {
|
||||
const {
|
||||
closeDegradedFieldFlyout,
|
||||
expandedDegradedField,
|
||||
renderedItems,
|
||||
isAnalysisInProgress,
|
||||
degradedFieldAnalysisFormattedResult,
|
||||
} = useDegradedFields();
|
||||
} = useQualityIssues();
|
||||
const { dataStreamSettings, datasetDetails, timeRange } = useDatasetQualityDetailsState();
|
||||
const pushedFlyoutTitleId = useGeneratedHtmlId({
|
||||
prefix: 'pushedFlyoutTitle',
|
||||
|
|
@ -57,7 +58,7 @@ export default function DegradedFieldFlyout() {
|
|||
|
||||
const fieldList = useMemo(() => {
|
||||
return renderedItems.find((item) => {
|
||||
return item.name === expandedDegradedField;
|
||||
return item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type;
|
||||
});
|
||||
}, [renderedItems, expandedDegradedField]);
|
||||
|
||||
|
|
@ -72,7 +73,17 @@ export default function DegradedFieldFlyout() {
|
|||
const redirectLinkProps = useRedirectLink({
|
||||
dataStreamStat: datasetDetails,
|
||||
timeRangeConfig: timeRange,
|
||||
query: { language: 'kuery', query: `${_IGNORED}: ${expandedDegradedField}` },
|
||||
query: {
|
||||
language: 'kuery',
|
||||
query:
|
||||
expandedDegradedField && expandedDegradedField.type === 'degraded'
|
||||
? DEGRADED_DOCS_QUERY
|
||||
: '',
|
||||
},
|
||||
selector:
|
||||
expandedDegradedField && expandedDegradedField.type === 'failed'
|
||||
? FAILURE_STORE_SELECTOR
|
||||
: undefined,
|
||||
sendTelemetry,
|
||||
});
|
||||
|
||||
|
|
@ -85,12 +96,26 @@ export default function DegradedFieldFlyout() {
|
|||
data-test-subj={'datasetQualityDetailsDegradedFieldFlyout'}
|
||||
>
|
||||
<EuiFlyoutHeader hasBorder>
|
||||
<EuiBadge color="warning">{overviewDegradedFieldsSectionTitle}</EuiBadge>
|
||||
<EuiBadge color="warning">{overviewQualityIssuesSectionTitle}</EuiBadge>
|
||||
<EuiSpacer size="s" />
|
||||
<EuiFlexGroup justifyContent="spaceBetween" gutterSize="s">
|
||||
<EuiTitle size="m">
|
||||
<EuiText>
|
||||
{expandedDegradedField} <span style={{ fontWeight: 400 }}>{fieldIgnoredText}</span>
|
||||
{expandedDegradedField?.type === 'degraded' ? (
|
||||
<>
|
||||
{expandedDegradedField?.name}{' '}
|
||||
<span style={{ fontWeight: 400 }}>{fieldIgnoredText}</span>
|
||||
</>
|
||||
) : (
|
||||
<span style={{ fontWeight: 400 }}>
|
||||
{i18n.translate(
|
||||
'xpack.datasetQuality.datasetQualityDetails.qualityIssueFlyout.failedDocsTitle',
|
||||
{
|
||||
defaultMessage: 'Documents indexing failed',
|
||||
}
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
</EuiText>
|
||||
</EuiTitle>
|
||||
<EuiToolTip content={openInDiscoverText}>
|
||||
|
|
@ -104,18 +129,20 @@ export default function DegradedFieldFlyout() {
|
|||
/>
|
||||
</EuiToolTip>
|
||||
</EuiFlexGroup>
|
||||
{!isUserViewingTheIssueOnLatestBackingIndex && (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<EuiTextColor
|
||||
color="danger"
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist"
|
||||
>
|
||||
{degradedFieldMessageIssueDoesNotExistInLatestIndex}
|
||||
</EuiTextColor>
|
||||
</>
|
||||
)}
|
||||
{isUserViewingTheIssueOnLatestBackingIndex &&
|
||||
{expandedDegradedField?.type === 'degraded' &&
|
||||
!isUserViewingTheIssueOnLatestBackingIndex && (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<EuiTextColor
|
||||
color="danger"
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist"
|
||||
>
|
||||
{degradedFieldMessageIssueDoesNotExistInLatestIndex}
|
||||
</EuiTextColor>
|
||||
</>
|
||||
)}
|
||||
{expandedDegradedField?.type === 'degraded' &&
|
||||
isUserViewingTheIssueOnLatestBackingIndex &&
|
||||
!isAnalysisInProgress &&
|
||||
degradedFieldAnalysisFormattedResult &&
|
||||
!degradedFieldAnalysisFormattedResult.identifiedUsingHeuristics && (
|
||||
|
|
@ -144,13 +171,8 @@ export default function DegradedFieldFlyout() {
|
|||
)}
|
||||
</EuiFlyoutHeader>
|
||||
<EuiFlyoutBody>
|
||||
<DegradedFieldInfo fieldList={fieldList} />
|
||||
{isUserViewingTheIssueOnLatestBackingIndex && (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<PossibleMitigations />
|
||||
</>
|
||||
)}
|
||||
{expandedDegradedField?.type === 'degraded' && <DegradedFieldFlyout />}
|
||||
{expandedDegradedField?.type === 'failed' && <FailedDocsFlyout />}
|
||||
</EuiFlyoutBody>
|
||||
</EuiFlyout>
|
||||
);
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { countBy } from 'lodash';
|
||||
import { QualityIndicators } from '../../../common/types';
|
||||
import { mapPercentageToQuality } from '../../../common/utils';
|
||||
|
||||
export const mapPercentagesToQualityCounts = (
|
||||
percentages: number[]
|
||||
): Record<QualityIndicators, number> =>
|
||||
countBy(percentages.map(mapPercentageToQuality)) as Record<QualityIndicators, number>;
|
||||
|
|
@ -7,5 +7,4 @@
|
|||
|
||||
export * from './indicator';
|
||||
export * from './percentage_indicator';
|
||||
export * from './helpers';
|
||||
export * from './dataset_quality_indicator';
|
||||
|
|
|
|||
|
|
@ -6,50 +6,53 @@
|
|||
*/
|
||||
|
||||
import { EuiIcon, EuiText, EuiToolTip } from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedNumber } from '@kbn/i18n-react';
|
||||
import React from 'react';
|
||||
|
||||
const FEW_DEGRADED_DOCS_THRESHOLD = 0.0005;
|
||||
const FEW_QUALITY_STATS_DOCS_THRESHOLD = 0.0005;
|
||||
|
||||
export function QualityPercentageIndicator({
|
||||
percentage,
|
||||
degradedDocsCount,
|
||||
docsCount = 0,
|
||||
fewDocsTooltipContent,
|
||||
}: {
|
||||
percentage: number;
|
||||
degradedDocsCount?: number;
|
||||
docsCount?: number;
|
||||
fewDocsTooltipContent: (docsCount: number) => string;
|
||||
}) {
|
||||
const isFewDegradedDocsAvailable = percentage && percentage < FEW_DEGRADED_DOCS_THRESHOLD;
|
||||
const isFewDocsAvailable = percentage && percentage < FEW_QUALITY_STATS_DOCS_THRESHOLD;
|
||||
|
||||
return isFewDegradedDocsAvailable ? (
|
||||
<DatasetWithFewDegradedDocs degradedDocsCount={degradedDocsCount} />
|
||||
return isFewDocsAvailable ? (
|
||||
<DatasetWithFewQualityStatsDocs
|
||||
docsCount={docsCount}
|
||||
fewDocsTooltipContent={fewDocsTooltipContent}
|
||||
/>
|
||||
) : (
|
||||
<DatasetWithManyDegradedDocs percentage={percentage} />
|
||||
<DatasetWithManyQualityStatsDocs percentage={percentage} />
|
||||
);
|
||||
}
|
||||
|
||||
const DatasetWithFewDegradedDocs = ({ degradedDocsCount }: { degradedDocsCount?: number }) => {
|
||||
const DatasetWithFewQualityStatsDocs = ({
|
||||
docsCount,
|
||||
fewDocsTooltipContent,
|
||||
}: {
|
||||
docsCount: number;
|
||||
fewDocsTooltipContent: (docsCount: number) => string;
|
||||
}) => {
|
||||
return (
|
||||
<EuiText size="s">
|
||||
~0%{' '}
|
||||
<EuiToolTip
|
||||
content={i18n.translate('xpack.datasetQuality.fewDegradedDocsTooltip', {
|
||||
defaultMessage: '{degradedDocsCount} degraded docs in this data set.',
|
||||
values: {
|
||||
degradedDocsCount,
|
||||
},
|
||||
})}
|
||||
>
|
||||
<EuiToolTip content={fewDocsTooltipContent(docsCount)}>
|
||||
<EuiIcon type="warning" color="warning" size="s" />
|
||||
</EuiToolTip>
|
||||
</EuiText>
|
||||
);
|
||||
};
|
||||
|
||||
const DatasetWithManyDegradedDocs = ({ percentage }: { percentage: number }) => {
|
||||
const DatasetWithManyQualityStatsDocs = ({ percentage }: { percentage: number }) => {
|
||||
return (
|
||||
<EuiText size="s">
|
||||
<FormattedNumber value={percentage} />%
|
||||
<FormattedNumber value={Number(percentage.toFixed(2))} />%
|
||||
</EuiText>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DegradedFieldSortField } from '../../hooks';
|
||||
import { QualityIssueSortField } from '../../hooks';
|
||||
import {
|
||||
DatasetQualityDetailsControllerContext,
|
||||
DEFAULT_CONTEXT,
|
||||
|
|
@ -17,11 +17,13 @@ export const getPublicStateFromContext = (
|
|||
): DatasetQualityDetailsPublicState => {
|
||||
return {
|
||||
dataStream: context.dataStream,
|
||||
degradedFields: context.degradedFields,
|
||||
qualityIssues: context.qualityIssues,
|
||||
failedDocsErrors: context.failedDocsErrors,
|
||||
timeRange: context.timeRange,
|
||||
breakdownField: context.breakdownField,
|
||||
qualityIssuesChart: context.qualityIssuesChart,
|
||||
integration: context.integration,
|
||||
expandedDegradedField: context.expandedDegradedField,
|
||||
expandedQualityIssue: context.expandedQualityIssue,
|
||||
showCurrentQualityIssues: context.showCurrentQualityIssues,
|
||||
};
|
||||
};
|
||||
|
|
@ -30,16 +32,16 @@ export const getContextFromPublicState = (
|
|||
publicState: DatasetQualityDetailsPublicStateUpdate
|
||||
): DatasetQualityDetailsControllerContext => ({
|
||||
...DEFAULT_CONTEXT,
|
||||
degradedFields: {
|
||||
qualityIssues: {
|
||||
table: {
|
||||
...DEFAULT_CONTEXT.degradedFields.table,
|
||||
...publicState.degradedFields?.table,
|
||||
sort: publicState.degradedFields?.table?.sort
|
||||
...DEFAULT_CONTEXT.qualityIssues.table,
|
||||
...publicState.qualityIssues?.table,
|
||||
sort: publicState.qualityIssues?.table?.sort
|
||||
? {
|
||||
...publicState.degradedFields.table.sort,
|
||||
field: publicState.degradedFields.table.sort.field as DegradedFieldSortField,
|
||||
...publicState.qualityIssues.table.sort,
|
||||
field: publicState.qualityIssues.table.sort.field as QualityIssueSortField,
|
||||
}
|
||||
: DEFAULT_CONTEXT.degradedFields.table.sort,
|
||||
: DEFAULT_CONTEXT.qualityIssues.table.sort,
|
||||
},
|
||||
},
|
||||
timeRange: {
|
||||
|
|
@ -52,7 +54,8 @@ export const getContextFromPublicState = (
|
|||
},
|
||||
dataStream: publicState.dataStream,
|
||||
breakdownField: publicState.breakdownField,
|
||||
expandedDegradedField: publicState.expandedDegradedField,
|
||||
qualityIssuesChart: publicState.qualityIssuesChart ?? DEFAULT_CONTEXT.qualityIssuesChart,
|
||||
expandedQualityIssue: publicState.expandedQualityIssue,
|
||||
showCurrentQualityIssues:
|
||||
publicState.showCurrentQualityIssues ?? DEFAULT_CONTEXT.showCurrentQualityIssues,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,17 +8,17 @@
|
|||
import { Observable } from 'rxjs';
|
||||
import {
|
||||
DatasetQualityDetailsControllerStateService,
|
||||
DegradedFieldsTableConfig,
|
||||
QualityIssuesTableConfig,
|
||||
WithDefaultControllerState,
|
||||
} from '../../state_machines/dataset_quality_details_controller';
|
||||
|
||||
type DegradedFieldTableSortOptions = Omit<DegradedFieldsTableConfig['table']['sort'], 'field'> & {
|
||||
type QuaityIssuesTableSortOptions = Omit<QualityIssuesTableConfig['table']['sort'], 'field'> & {
|
||||
field: string;
|
||||
};
|
||||
|
||||
export type DatasetQualityDegradedFieldTableOptions = Partial<
|
||||
Omit<DegradedFieldsTableConfig['table'], 'sort'> & {
|
||||
sort?: DegradedFieldTableSortOptions;
|
||||
export type DatasetQualityIssuesTableOptions = Partial<
|
||||
Omit<QualityIssuesTableConfig['table'], 'sort'> & {
|
||||
sort?: QuaityIssuesTableSortOptions;
|
||||
}
|
||||
>;
|
||||
|
||||
|
|
@ -30,13 +30,17 @@ export type DatasetQualityDetailsPublicState = WithDefaultControllerState;
|
|||
export type DatasetQualityDetailsPublicStateUpdate = Partial<
|
||||
Pick<
|
||||
WithDefaultControllerState,
|
||||
'timeRange' | 'breakdownField' | 'expandedDegradedField' | 'showCurrentQualityIssues'
|
||||
| 'timeRange'
|
||||
| 'breakdownField'
|
||||
| 'showCurrentQualityIssues'
|
||||
| 'expandedQualityIssue'
|
||||
| 'qualityIssuesChart'
|
||||
>
|
||||
> & {
|
||||
dataStream: string;
|
||||
} & {
|
||||
degradedFields?: {
|
||||
table?: DatasetQualityDegradedFieldTableOptions;
|
||||
qualityIssues?: {
|
||||
table?: DatasetQualityIssuesTableOptions;
|
||||
};
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@
|
|||
*/
|
||||
|
||||
export * from './use_dataset_quality_table';
|
||||
export * from './use_degraded_docs_chart';
|
||||
export * from './use_quality_issues_docs_chart';
|
||||
export * from './use_redirect_link';
|
||||
export * from './use_summary_panel';
|
||||
export * from './use_create_dataview';
|
||||
export * from './use_redirect_link_telemetry';
|
||||
export * from './use_dataset_quality_details_state';
|
||||
export * from './use_degraded_fields';
|
||||
export * from './use_quality_issues';
|
||||
export * from './use_integration_actions';
|
||||
export * from './use_dataset_telemetry';
|
||||
export * from './use_dataset_details_telemetry';
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import { DataStreamDetails } from '../../common/api_types';
|
|||
import { Integration } from '../../common/data_streams_stats/integration';
|
||||
import { mapPercentageToQuality } from '../../common/utils';
|
||||
import { MASKED_FIELD_PLACEHOLDER, UNKOWN_FIELD_PLACEHOLDER } from '../../common/constants';
|
||||
import { calculatePercentage } from '../utils';
|
||||
|
||||
export function useDatasetDetailsTelemetry() {
|
||||
const {
|
||||
|
|
@ -167,10 +168,11 @@ function getDatasetDetailsEbtProps({
|
|||
type: datasetDetails.type,
|
||||
};
|
||||
const degradedDocs = dataStreamDetails?.degradedDocsCount ?? 0;
|
||||
const totalDocs = dataStreamDetails?.docsCount ?? 0;
|
||||
const degradedPercentage =
|
||||
totalDocs > 0 ? Number(((degradedDocs / totalDocs) * 100).toFixed(2)) : 0;
|
||||
const health = mapPercentageToQuality(degradedPercentage);
|
||||
const failedDocs = dataStreamDetails?.failedDocsCount ?? 0;
|
||||
const totalDocs = (dataStreamDetails?.docsCount ?? 0) + failedDocs;
|
||||
const degradedPercentage = calculatePercentage({ totalDocs, count: degradedDocs });
|
||||
const failedPercentage = calculatePercentage({ totalDocs, count: failedDocs });
|
||||
const health = mapPercentageToQuality([degradedPercentage, failedPercentage]);
|
||||
const { startDate: from, endDate: to } = getDateISORange(timeRange);
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import { BasicDataStream } from '../../common/types';
|
|||
import { useKibanaContextForPlugin } from '../utils';
|
||||
|
||||
export const useDatasetQualityDetailsState = () => {
|
||||
const { service, telemetryClient } = useDatasetQualityDetailsContext();
|
||||
const { service, telemetryClient, isServerless } = useDatasetQualityDetailsContext();
|
||||
|
||||
const {
|
||||
services: { fieldFormats },
|
||||
|
|
@ -23,11 +23,11 @@ export const useDatasetQualityDetailsState = () => {
|
|||
|
||||
const {
|
||||
dataStream,
|
||||
degradedFields,
|
||||
qualityIssues,
|
||||
timeRange,
|
||||
breakdownField,
|
||||
isIndexNotFoundError,
|
||||
expandedDegradedField,
|
||||
expandedQualityIssue,
|
||||
} = useSelector(service, (state) => state.context) ?? {};
|
||||
|
||||
const isNonAggregatable = useSelector(service, (state) =>
|
||||
|
|
@ -51,9 +51,19 @@ export const useDatasetQualityDetailsState = () => {
|
|||
);
|
||||
|
||||
const dataStreamSettings = useSelector(service, (state) =>
|
||||
state.matches('initializing.dataStreamSettings.fetchingDataStreamDegradedFields') ||
|
||||
state.matches('initializing.dataStreamSettings.doneFetchingDegradedFields') ||
|
||||
state.matches('initializing.dataStreamSettings.errorFetchingDegradedFields')
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.fetchingDataStreamDegradedFields'
|
||||
) ||
|
||||
state.matches('initializing.dataStreamSettings.doneFetchingQualityIssues') ||
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.errorFetchingDegradedFields'
|
||||
) ||
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.qualityIssues.dataStreamFailedDocs.fetchingFailedDocs'
|
||||
) ||
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.qualityIssues.dataStreamFailedDocs.errorFetchingFailedDocs'
|
||||
)
|
||||
? state.context.dataStreamSettings
|
||||
: undefined
|
||||
);
|
||||
|
|
@ -104,6 +114,8 @@ export const useDatasetQualityDetailsState = () => {
|
|||
rawName: dataStream,
|
||||
};
|
||||
|
||||
const docsTrendChart = useSelector(service, (state) => state.context.qualityIssuesChart);
|
||||
|
||||
const loadingState = useSelector(service, (state) => ({
|
||||
nonAggregatableDatasetLoading: state.matches('initializing.nonAggregatableDataset.fetching'),
|
||||
dataStreamDetailsLoading: state.matches('initializing.dataStreamDetails.fetching'),
|
||||
|
|
@ -126,8 +138,8 @@ export const useDatasetQualityDetailsState = () => {
|
|||
),
|
||||
}));
|
||||
|
||||
const isDegradedFieldFlyoutOpen = useSelector(service, (state) =>
|
||||
state.matches('initializing.degradedFieldFlyout.open')
|
||||
const isQualityIssueFlyoutOpen = useSelector(service, (state) =>
|
||||
state.matches('initializing.qualityIssueFlyout.open')
|
||||
);
|
||||
|
||||
const updateTimeRange = useCallback(
|
||||
|
|
@ -147,12 +159,14 @@ export const useDatasetQualityDetailsState = () => {
|
|||
return {
|
||||
service,
|
||||
telemetryClient,
|
||||
isServerless,
|
||||
fieldFormats,
|
||||
isIndexNotFoundError,
|
||||
dataStream,
|
||||
datasetDetails,
|
||||
degradedFields,
|
||||
qualityIssues,
|
||||
dataStreamDetails,
|
||||
docsTrendChart,
|
||||
breakdownField,
|
||||
isBreakdownFieldEcs,
|
||||
isBreakdownFieldAsserted,
|
||||
|
|
@ -164,7 +178,7 @@ export const useDatasetQualityDetailsState = () => {
|
|||
integrationDetails,
|
||||
canUserAccessDashboards,
|
||||
canUserViewIntegrations,
|
||||
expandedDegradedField,
|
||||
isDegradedFieldFlyoutOpen,
|
||||
expandedQualityIssue,
|
||||
isQualityIssueFlyoutOpen,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ const sortingOverrides: Partial<{
|
|||
}> = {
|
||||
['title']: 'name',
|
||||
['size']: DataStreamStat.calculateFilteredSize,
|
||||
['quality']: (item) => Math.max(item.degradedDocs.percentage, item.failedDocs.percentage),
|
||||
};
|
||||
|
||||
export const useDatasetQualityTable = () => {
|
||||
|
|
@ -35,7 +36,7 @@ export const useDatasetQualityTable = () => {
|
|||
},
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const { service } = useDatasetQualityContext();
|
||||
const { service, isServerless } = useDatasetQualityContext();
|
||||
|
||||
const { page, rowsPerPage, sort } = useSelector(service, (state) => state.context.table);
|
||||
|
||||
|
|
@ -65,15 +66,23 @@ export const useDatasetQualityTable = () => {
|
|||
service,
|
||||
(state) =>
|
||||
state.matches('stats.datasets.fetching') ||
|
||||
state.matches('stats.docsStats.fetching') ||
|
||||
state.matches('integrations.fetching') ||
|
||||
state.matches('stats.degradedDocs.fetching')
|
||||
state.matches('stats.degradedDocs.fetching') ||
|
||||
state.matches('stats.failedDocs.fetching')
|
||||
);
|
||||
const loadingDataStreamStats = useSelector(service, (state) =>
|
||||
state.matches('stats.datasets.fetching')
|
||||
);
|
||||
const loadingDocStats = useSelector(service, (state) =>
|
||||
state.matches('stats.docsStats.fetching')
|
||||
);
|
||||
const loadingDegradedStats = useSelector(service, (state) =>
|
||||
state.matches('stats.degradedDocs.fetching')
|
||||
);
|
||||
const loadingFailedStats = useSelector(service, (state) =>
|
||||
state.matches('stats.failedDocs.fetching')
|
||||
);
|
||||
|
||||
const datasets = useSelector(service, (state) => state.context.datasets);
|
||||
|
||||
|
|
@ -99,22 +108,28 @@ export const useDatasetQualityTable = () => {
|
|||
canUserMonitorDataset,
|
||||
canUserMonitorAnyDataStream,
|
||||
loadingDataStreamStats,
|
||||
loadingDocStats,
|
||||
loadingDegradedStats,
|
||||
loadingFailedStats,
|
||||
showFullDatasetNames,
|
||||
isActiveDataset: isActive,
|
||||
timeRange,
|
||||
urlService: url,
|
||||
failureStoreEnabled: !isServerless,
|
||||
}),
|
||||
[
|
||||
fieldFormats,
|
||||
canUserMonitorDataset,
|
||||
canUserMonitorAnyDataStream,
|
||||
loadingDataStreamStats,
|
||||
loadingDocStats,
|
||||
loadingDegradedStats,
|
||||
loadingFailedStats,
|
||||
showFullDatasetNames,
|
||||
isActive,
|
||||
timeRange,
|
||||
url,
|
||||
isServerless,
|
||||
]
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,242 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { useSelector } from '@xstate/react';
|
||||
import { useCallback, useMemo } from 'react';
|
||||
import { orderBy } from 'lodash';
|
||||
import { DegradedField } from '../../common/data_streams_stats';
|
||||
import { SortDirection } from '../../common/types';
|
||||
import {
|
||||
DEFAULT_DEGRADED_FIELD_SORT_DIRECTION,
|
||||
DEFAULT_DEGRADED_FIELD_SORT_FIELD,
|
||||
} from '../../common/constants';
|
||||
import { useKibanaContextForPlugin } from '../utils';
|
||||
import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state';
|
||||
import {
|
||||
degradedFieldCauseFieldIgnored,
|
||||
degradedFieldCauseFieldIgnoredTooltip,
|
||||
degradedFieldCauseFieldLimitExceeded,
|
||||
degradedFieldCauseFieldLimitExceededTooltip,
|
||||
degradedFieldCauseFieldMalformed,
|
||||
degradedFieldCauseFieldMalformedTooltip,
|
||||
} from '../../common/translations';
|
||||
|
||||
export type DegradedFieldSortField = keyof DegradedField;
|
||||
|
||||
export function useDegradedFields() {
|
||||
const { service } = useDatasetQualityDetailsState();
|
||||
const {
|
||||
services: { fieldFormats },
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const { degradedFields, expandedDegradedField, showCurrentQualityIssues } = useSelector(
|
||||
service,
|
||||
(state) => state.context
|
||||
);
|
||||
const { data, table } = degradedFields ?? {};
|
||||
const { page, rowsPerPage, sort } = table;
|
||||
|
||||
const totalItemCount = data?.length ?? 0;
|
||||
|
||||
const pagination = {
|
||||
pageIndex: page,
|
||||
pageSize: rowsPerPage,
|
||||
totalItemCount,
|
||||
hidePerPageOptions: true,
|
||||
};
|
||||
|
||||
const onTableChange = useCallback(
|
||||
(options: {
|
||||
page: { index: number; size: number };
|
||||
sort?: { field: DegradedFieldSortField; direction: SortDirection };
|
||||
}) => {
|
||||
service.send({
|
||||
type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA',
|
||||
degraded_field_criteria: {
|
||||
page: options.page.index,
|
||||
rowsPerPage: options.page.size,
|
||||
sort: {
|
||||
field: options.sort?.field || DEFAULT_DEGRADED_FIELD_SORT_FIELD,
|
||||
direction: options.sort?.direction || DEFAULT_DEGRADED_FIELD_SORT_DIRECTION,
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
[service]
|
||||
);
|
||||
|
||||
const renderedItems = useMemo(() => {
|
||||
const sortedItems = orderBy(data, sort.field, sort.direction);
|
||||
return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage);
|
||||
}, [data, sort.field, sort.direction, page, rowsPerPage]);
|
||||
|
||||
const expandedRenderedItem = useMemo(() => {
|
||||
return renderedItems.find((item) => item.name === expandedDegradedField);
|
||||
}, [expandedDegradedField, renderedItems]);
|
||||
|
||||
const isDegradedFieldsLoading = useSelector(
|
||||
service,
|
||||
(state) =>
|
||||
state.matches('initializing.dataStreamSettings.fetchingDataStreamSettings') ||
|
||||
state.matches('initializing.dataStreamSettings.fetchingDataStreamDegradedFields')
|
||||
);
|
||||
|
||||
const closeDegradedFieldFlyout = useCallback(
|
||||
() => service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' }),
|
||||
[service]
|
||||
);
|
||||
|
||||
const openDegradedFieldFlyout = useCallback(
|
||||
(fieldName: string) => {
|
||||
if (expandedDegradedField === fieldName) {
|
||||
service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' });
|
||||
} else {
|
||||
service.send({ type: 'OPEN_DEGRADED_FIELD_FLYOUT', fieldName });
|
||||
}
|
||||
},
|
||||
[expandedDegradedField, service]
|
||||
);
|
||||
|
||||
const toggleCurrentQualityIssues = useCallback(() => {
|
||||
service.send('TOGGLE_CURRENT_QUALITY_ISSUES');
|
||||
}, [service]);
|
||||
|
||||
const degradedFieldValues = useSelector(service, (state) =>
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.ignoredValues.done')
|
||||
? state.context.degradedFieldValues
|
||||
: undefined
|
||||
);
|
||||
|
||||
const degradedFieldAnalysis = useSelector(service, (state) =>
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed') ||
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating') ||
|
||||
state.matches(
|
||||
'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover'
|
||||
) ||
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver') ||
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') ||
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error')
|
||||
? state.context.degradedFieldAnalysis
|
||||
: undefined
|
||||
);
|
||||
|
||||
const degradedFieldAnalysisFormattedResult = useMemo(() => {
|
||||
if (!degradedFieldAnalysis) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// 1st check if it's a field limit issue
|
||||
if (degradedFieldAnalysis.isFieldLimitIssue) {
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldLimitExceeded,
|
||||
tooltipContent: degradedFieldCauseFieldLimitExceededTooltip,
|
||||
shouldDisplayIgnoredValuesAndLimit: false,
|
||||
identifiedUsingHeuristics: true,
|
||||
};
|
||||
}
|
||||
|
||||
// 2nd check if it's a ignored above issue
|
||||
const fieldMapping = degradedFieldAnalysis.fieldMapping;
|
||||
|
||||
if (fieldMapping && fieldMapping?.type === 'keyword' && fieldMapping?.ignore_above) {
|
||||
const isAnyValueExceedingIgnoreAbove = degradedFieldValues?.values.some(
|
||||
(value) => value.length > fieldMapping.ignore_above!
|
||||
);
|
||||
if (isAnyValueExceedingIgnoreAbove) {
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldIgnored,
|
||||
tooltipContent: degradedFieldCauseFieldIgnoredTooltip,
|
||||
shouldDisplayIgnoredValuesAndLimit: true,
|
||||
identifiedUsingHeuristics: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 3rd check if its a ignore_malformed issue. There is no check, at the moment.
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldMalformed,
|
||||
tooltipContent: degradedFieldCauseFieldMalformedTooltip,
|
||||
shouldDisplayIgnoredValuesAndLimit: false,
|
||||
identifiedUsingHeuristics: false, // TODO: Add heuristics to identify ignore_malformed issues
|
||||
};
|
||||
}, [degradedFieldAnalysis, degradedFieldValues]);
|
||||
|
||||
const isDegradedFieldsValueLoading = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching'
|
||||
);
|
||||
});
|
||||
|
||||
const isRolloverRequired = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover'
|
||||
);
|
||||
});
|
||||
|
||||
const isMitigationAppliedSuccessfully = useSelector(service, (state) => {
|
||||
return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success');
|
||||
});
|
||||
|
||||
const isAnalysisInProgress = useSelector(service, (state) => {
|
||||
return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing');
|
||||
});
|
||||
|
||||
const isRolloverInProgress = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver'
|
||||
);
|
||||
});
|
||||
|
||||
const updateNewFieldLimit = useCallback(
|
||||
(newFieldLimit: number) => {
|
||||
service.send({ type: 'SET_NEW_FIELD_LIMIT', newFieldLimit });
|
||||
},
|
||||
[service]
|
||||
);
|
||||
|
||||
const isMitigationInProgress = useSelector(service, (state) => {
|
||||
return state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating');
|
||||
});
|
||||
|
||||
const newFieldLimitData = useSelector(service, (state) =>
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.success') ||
|
||||
state.matches('initializing.degradedFieldFlyout.open.initialized.mitigation.error')
|
||||
? state.context.fieldLimit
|
||||
: undefined
|
||||
);
|
||||
|
||||
const triggerRollover = useCallback(() => {
|
||||
service.send('ROLLOVER_DATA_STREAM');
|
||||
}, [service]);
|
||||
|
||||
return {
|
||||
isDegradedFieldsLoading,
|
||||
pagination,
|
||||
onTableChange,
|
||||
renderedItems,
|
||||
sort: { sort },
|
||||
fieldFormats,
|
||||
totalItemCount,
|
||||
expandedDegradedField,
|
||||
openDegradedFieldFlyout,
|
||||
closeDegradedFieldFlyout,
|
||||
degradedFieldValues,
|
||||
isDegradedFieldsValueLoading,
|
||||
isAnalysisInProgress,
|
||||
degradedFieldAnalysis,
|
||||
degradedFieldAnalysisFormattedResult,
|
||||
toggleCurrentQualityIssues,
|
||||
showCurrentQualityIssues,
|
||||
expandedRenderedItem,
|
||||
updateNewFieldLimit,
|
||||
isMitigationInProgress,
|
||||
isRolloverInProgress,
|
||||
newFieldLimitData,
|
||||
isRolloverRequired,
|
||||
isMitigationAppliedSuccessfully,
|
||||
triggerRollover,
|
||||
};
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import { formatNumber } from '@elastic/eui';
|
|||
import { mapPercentageToQuality } from '../../common/utils';
|
||||
import { BYTE_NUMBER_FORMAT, MAX_HOSTS_METRIC_VALUE, NUMBER_FORMAT } from '../../common/constants';
|
||||
import { useDatasetQualityDetailsContext } from '../components/dataset_quality_details/context';
|
||||
import { calculatePercentage } from '../utils';
|
||||
|
||||
export const useOverviewSummaryPanel = () => {
|
||||
const { service } = useDatasetQualityDetailsContext();
|
||||
|
|
@ -55,12 +56,19 @@ export const useOverviewSummaryPanel = () => {
|
|||
NUMBER_FORMAT
|
||||
);
|
||||
|
||||
const degradedPercentage =
|
||||
Number(totalDocsCount) > 0
|
||||
? (Number(totalDegradedDocsCount) / Number(totalDocsCount)) * 100
|
||||
: 0;
|
||||
const totalFailedDocsCount = formatNumber(dataStreamDetails?.failedDocsCount ?? 0, NUMBER_FORMAT);
|
||||
|
||||
const quality = mapPercentageToQuality(degradedPercentage);
|
||||
const degradedPercentage = calculatePercentage({
|
||||
totalDocs: dataStreamDetails.docsCount,
|
||||
count: dataStreamDetails?.degradedDocsCount,
|
||||
});
|
||||
|
||||
const failedPercentage = calculatePercentage({
|
||||
totalDocs: dataStreamDetails.docsCount,
|
||||
count: dataStreamDetails?.failedDocsCount,
|
||||
});
|
||||
|
||||
const quality = mapPercentageToQuality([degradedPercentage, failedPercentage]);
|
||||
|
||||
return {
|
||||
totalDocsCount,
|
||||
|
|
@ -70,6 +78,7 @@ export const useOverviewSummaryPanel = () => {
|
|||
totalHostsCount,
|
||||
isSummaryPanelLoading,
|
||||
totalDegradedDocsCount,
|
||||
totalFailedDocsCount,
|
||||
quality,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,367 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { useSelector } from '@xstate/react';
|
||||
import { orderBy } from 'lodash';
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
import { FailedDocsError, QualityIssue } from '../../common/api_types';
|
||||
import {
|
||||
DEFAULT_FAILED_DOCS_ERROR_SORT_DIRECTION,
|
||||
DEFAULT_FAILED_DOCS_ERROR_SORT_FIELD,
|
||||
DEFAULT_QUALITY_ISSUE_SORT_DIRECTION,
|
||||
DEFAULT_QUALITY_ISSUE_SORT_FIELD,
|
||||
} from '../../common/constants';
|
||||
import {
|
||||
degradedFieldCauseFieldIgnored,
|
||||
degradedFieldCauseFieldIgnoredTooltip,
|
||||
degradedFieldCauseFieldLimitExceeded,
|
||||
degradedFieldCauseFieldLimitExceededTooltip,
|
||||
degradedFieldCauseFieldMalformed,
|
||||
degradedFieldCauseFieldMalformedTooltip,
|
||||
} from '../../common/translations';
|
||||
import { SortDirection } from '../../common/types';
|
||||
import { QualityIssueType } from '../state_machines/dataset_quality_details_controller';
|
||||
import { useKibanaContextForPlugin } from '../utils';
|
||||
import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state';
|
||||
import { getFailedDocsErrorsColumns } from '../components/dataset_quality_details/quality_issue_flyout/failed_docs/columns';
|
||||
|
||||
export type QualityIssueSortField = keyof QualityIssue;
|
||||
export type FailedDocsErrorSortField = keyof FailedDocsError;
|
||||
|
||||
export function useQualityIssues() {
|
||||
const { service } = useDatasetQualityDetailsState();
|
||||
const {
|
||||
services: { fieldFormats },
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const {
|
||||
qualityIssues,
|
||||
expandedQualityIssue: expandedDegradedField,
|
||||
showCurrentQualityIssues,
|
||||
failedDocsErrors,
|
||||
} = useSelector(service, (state) => state.context);
|
||||
const { data, table } = qualityIssues ?? {};
|
||||
const { page, rowsPerPage, sort } = table;
|
||||
|
||||
const { data: failedDocsErrorsData, table: failedDocsErrorsTable } = failedDocsErrors ?? {};
|
||||
const {
|
||||
page: failedDocsErrorsPage,
|
||||
rowsPerPage: failedDocsErrorsRowsPerPage,
|
||||
sort: failedDocsErrorsSort,
|
||||
} = failedDocsErrorsTable;
|
||||
|
||||
const totalItemCount = data?.length ?? 0;
|
||||
|
||||
const pagination = {
|
||||
pageIndex: page,
|
||||
pageSize: rowsPerPage,
|
||||
totalItemCount,
|
||||
hidePerPageOptions: true,
|
||||
};
|
||||
|
||||
const onTableChange = useCallback(
|
||||
(options: {
|
||||
page: { index: number; size: number };
|
||||
sort?: { field: QualityIssueSortField; direction: SortDirection };
|
||||
}) => {
|
||||
service.send({
|
||||
type: 'UPDATE_QUALITY_ISSUES_TABLE_CRITERIA',
|
||||
quality_issues_criteria: {
|
||||
page: options.page.index,
|
||||
rowsPerPage: options.page.size,
|
||||
sort: {
|
||||
field: options.sort?.field || DEFAULT_QUALITY_ISSUE_SORT_FIELD,
|
||||
direction: options.sort?.direction || DEFAULT_QUALITY_ISSUE_SORT_DIRECTION,
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
[service]
|
||||
);
|
||||
|
||||
const renderedItems = useMemo(() => {
|
||||
const sortedItems = orderBy(data, sort.field, sort.direction);
|
||||
return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage);
|
||||
}, [data, sort.field, sort.direction, page, rowsPerPage]);
|
||||
|
||||
const expandedRenderedItem = useMemo(() => {
|
||||
return renderedItems.find(
|
||||
(item) =>
|
||||
item.name === expandedDegradedField?.name && item.type === expandedDegradedField?.type
|
||||
);
|
||||
}, [expandedDegradedField, renderedItems]);
|
||||
|
||||
const isFailedDocsErrorsLoading = useSelector(service, (state) => {
|
||||
return state.matches('initializing.qualityIssueFlyout.open.failedDocsFlyout.fetching');
|
||||
});
|
||||
|
||||
const isDegradedFieldsLoading = useSelector(
|
||||
service,
|
||||
(state) =>
|
||||
state.matches('initializing.dataStreamSettings.fetchingDataStreamSettings') ||
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.fetchingDataStreamDegradedFields'
|
||||
)
|
||||
);
|
||||
|
||||
const closeDegradedFieldFlyout = useCallback(
|
||||
() => service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' }),
|
||||
[service]
|
||||
);
|
||||
|
||||
const openDegradedFieldFlyout = useCallback(
|
||||
(fieldName: string, qualityIssueType: QualityIssueType) => {
|
||||
if (
|
||||
expandedDegradedField?.name === fieldName &&
|
||||
expandedDegradedField?.type === qualityIssueType
|
||||
) {
|
||||
service.send({ type: 'CLOSE_DEGRADED_FIELD_FLYOUT' });
|
||||
} else {
|
||||
service.send({
|
||||
type: 'OPEN_QUALITY_ISSUE_FLYOUT',
|
||||
qualityIssue: {
|
||||
name: fieldName,
|
||||
type: qualityIssueType,
|
||||
},
|
||||
});
|
||||
}
|
||||
},
|
||||
[expandedDegradedField, service]
|
||||
);
|
||||
|
||||
const toggleCurrentQualityIssues = useCallback(() => {
|
||||
service.send('TOGGLE_CURRENT_QUALITY_ISSUES');
|
||||
}, [service]);
|
||||
|
||||
const degradedFieldValues = useSelector(service, (state) =>
|
||||
state.matches('initializing.qualityIssueFlyout.open.degradedFieldFlyout.ignoredValues.done')
|
||||
? state.context.degradedFieldValues
|
||||
: undefined
|
||||
);
|
||||
|
||||
const degradedFieldAnalysis = useSelector(service, (state) =>
|
||||
state.matches('initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.analyzed') ||
|
||||
state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.mitigating'
|
||||
) ||
|
||||
state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.askingForRollover'
|
||||
) ||
|
||||
state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.rollingOver'
|
||||
) ||
|
||||
state.matches('initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.success') ||
|
||||
state.matches('initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.error')
|
||||
? state.context.degradedFieldAnalysis
|
||||
: undefined
|
||||
);
|
||||
|
||||
const degradedFieldAnalysisFormattedResult = useMemo(() => {
|
||||
if (!degradedFieldAnalysis) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// 1st check if it's a field limit issue
|
||||
if (degradedFieldAnalysis.isFieldLimitIssue) {
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldLimitExceeded,
|
||||
tooltipContent: degradedFieldCauseFieldLimitExceededTooltip,
|
||||
shouldDisplayIgnoredValuesAndLimit: false,
|
||||
identifiedUsingHeuristics: true,
|
||||
};
|
||||
}
|
||||
|
||||
// 2nd check if it's a ignored above issue
|
||||
const fieldMapping = degradedFieldAnalysis.fieldMapping;
|
||||
|
||||
if (fieldMapping && fieldMapping?.type === 'keyword' && fieldMapping?.ignore_above) {
|
||||
const isAnyValueExceedingIgnoreAbove = degradedFieldValues?.values.some(
|
||||
(value) => value.length > fieldMapping.ignore_above!
|
||||
);
|
||||
if (isAnyValueExceedingIgnoreAbove) {
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldIgnored,
|
||||
tooltipContent: degradedFieldCauseFieldIgnoredTooltip,
|
||||
shouldDisplayIgnoredValuesAndLimit: true,
|
||||
identifiedUsingHeuristics: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 3rd check if its a ignore_malformed issue. There is no check, at the moment.
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldMalformed,
|
||||
tooltipContent: degradedFieldCauseFieldMalformedTooltip,
|
||||
shouldDisplayIgnoredValuesAndLimit: false,
|
||||
identifiedUsingHeuristics: false, // TODO: Add heuristics to identify ignore_malformed issues
|
||||
};
|
||||
}, [degradedFieldAnalysis, degradedFieldValues]);
|
||||
|
||||
const isDegradedFieldsValueLoading = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.ignoredValues.fetching'
|
||||
);
|
||||
});
|
||||
|
||||
const isRolloverRequired = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.askingForRollover'
|
||||
);
|
||||
});
|
||||
|
||||
const isMitigationAppliedSuccessfully = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.success'
|
||||
);
|
||||
});
|
||||
|
||||
const isAnalysisInProgress = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.analyzing'
|
||||
);
|
||||
});
|
||||
|
||||
const isRolloverInProgress = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.rollingOver'
|
||||
);
|
||||
});
|
||||
|
||||
const updateNewFieldLimit = useCallback(
|
||||
(newFieldLimit: number) => {
|
||||
service.send({ type: 'SET_NEW_FIELD_LIMIT', newFieldLimit });
|
||||
},
|
||||
[service]
|
||||
);
|
||||
|
||||
const isMitigationInProgress = useSelector(service, (state) => {
|
||||
return state.matches(
|
||||
'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.mitigating'
|
||||
);
|
||||
});
|
||||
|
||||
const newFieldLimitData = useSelector(service, (state) =>
|
||||
state.matches('initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.success') ||
|
||||
state.matches('initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.error')
|
||||
? state.context.fieldLimit
|
||||
: undefined
|
||||
);
|
||||
|
||||
const triggerRollover = useCallback(() => {
|
||||
service.send('ROLLOVER_DATA_STREAM');
|
||||
}, [service]);
|
||||
|
||||
const failedDocsErrorsColumns = useMemo(() => getFailedDocsErrorsColumns(), []);
|
||||
|
||||
const renderedFailedDocsErrorsItems = useMemo(() => {
|
||||
const sortedItems = orderBy(
|
||||
failedDocsErrorsData,
|
||||
failedDocsErrorsSort.field,
|
||||
failedDocsErrorsSort.direction
|
||||
);
|
||||
return sortedItems.slice(
|
||||
failedDocsErrorsPage * failedDocsErrorsRowsPerPage,
|
||||
(failedDocsErrorsPage + 1) * failedDocsErrorsRowsPerPage
|
||||
);
|
||||
}, [
|
||||
failedDocsErrorsData,
|
||||
failedDocsErrorsSort.field,
|
||||
failedDocsErrorsSort.direction,
|
||||
failedDocsErrorsPage,
|
||||
failedDocsErrorsRowsPerPage,
|
||||
]);
|
||||
|
||||
const onFailedDocsErrorsTableChange = useCallback(
|
||||
(options: {
|
||||
page: { index: number; size: number };
|
||||
sort?: { field: FailedDocsErrorSortField; direction: SortDirection };
|
||||
}) => {
|
||||
service.send({
|
||||
type: 'UPDATE_FAILED_DOCS_ERRORS_TABLE_CRITERIA',
|
||||
failed_docs_errors_criteria: {
|
||||
page: options.page.index,
|
||||
rowsPerPage: options.page.size,
|
||||
sort: {
|
||||
field: options.sort?.field || DEFAULT_FAILED_DOCS_ERROR_SORT_FIELD,
|
||||
direction: options.sort?.direction || DEFAULT_FAILED_DOCS_ERROR_SORT_DIRECTION,
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
[service]
|
||||
);
|
||||
|
||||
const failedDocsErrorsPagination = {
|
||||
pageIndex: failedDocsErrorsPage,
|
||||
pageSize: failedDocsErrorsRowsPerPage,
|
||||
totalItemCount: failedDocsErrorsData?.length ?? 0,
|
||||
hidePerPageOptions: true,
|
||||
};
|
||||
|
||||
const resultsCount = useMemo(() => {
|
||||
const startNumberItemsOnPage =
|
||||
failedDocsErrorsRowsPerPage * failedDocsErrorsPage +
|
||||
(renderedFailedDocsErrorsItems.length ? 1 : 0);
|
||||
const endNumberItemsOnPage =
|
||||
failedDocsErrorsRowsPerPage * failedDocsErrorsPage + renderedFailedDocsErrorsItems.length;
|
||||
|
||||
return failedDocsErrorsRowsPerPage === 0 ? (
|
||||
<strong>
|
||||
{i18n.translate('xpack.datasetQuality.resultsCount.strong.lllLabel', {
|
||||
defaultMessage: 'lll',
|
||||
})}
|
||||
</strong>
|
||||
) : (
|
||||
<>
|
||||
<strong>
|
||||
{startNumberItemsOnPage}-{endNumberItemsOnPage}
|
||||
</strong>{' '}
|
||||
{' of '} {failedDocsErrorsData?.length}
|
||||
</>
|
||||
);
|
||||
}, [
|
||||
failedDocsErrorsRowsPerPage,
|
||||
failedDocsErrorsPage,
|
||||
renderedFailedDocsErrorsItems.length,
|
||||
failedDocsErrorsData?.length,
|
||||
]);
|
||||
|
||||
return {
|
||||
isDegradedFieldsLoading,
|
||||
pagination,
|
||||
onTableChange,
|
||||
renderedItems,
|
||||
sort: { sort },
|
||||
fieldFormats,
|
||||
totalItemCount,
|
||||
expandedDegradedField,
|
||||
openDegradedFieldFlyout,
|
||||
closeDegradedFieldFlyout,
|
||||
degradedFieldValues,
|
||||
isDegradedFieldsValueLoading,
|
||||
isAnalysisInProgress,
|
||||
degradedFieldAnalysis,
|
||||
degradedFieldAnalysisFormattedResult,
|
||||
toggleCurrentQualityIssues,
|
||||
showCurrentQualityIssues,
|
||||
expandedRenderedItem,
|
||||
updateNewFieldLimit,
|
||||
isMitigationInProgress,
|
||||
isRolloverInProgress,
|
||||
newFieldLimitData,
|
||||
isRolloverRequired,
|
||||
isMitigationAppliedSuccessfully,
|
||||
triggerRollover,
|
||||
renderedFailedDocsErrorsItems,
|
||||
failedDocsErrorsSort: { sort: failedDocsErrorsSort },
|
||||
resultsCount,
|
||||
failedDocsErrorsColumns,
|
||||
isFailedDocsErrorsLoading,
|
||||
failedDocsErrorsPagination,
|
||||
onFailedDocsErrorsTableChange,
|
||||
};
|
||||
}
|
||||
|
|
@ -5,18 +5,26 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import type { Action } from '@kbn/ui-actions-plugin/public';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { useEuiTheme } from '@elastic/eui';
|
||||
import type { DataView, DataViewField } from '@kbn/data-views-plugin/common';
|
||||
import { fieldSupportsBreakdown } from '@kbn/field-utils';
|
||||
import { DEFAULT_LOGS_DATA_VIEW } from '../../common/constants';
|
||||
import { useCreateDataView } from './use_create_dataview';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import type { Action } from '@kbn/ui-actions-plugin/public';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import {
|
||||
DEFAULT_LOGS_DATA_VIEW,
|
||||
DEGRADED_DOCS_QUERY,
|
||||
FAILURE_STORE_SELECTOR,
|
||||
} from '../../common/constants';
|
||||
import { getLensAttributes as getDegradedLensAttributes } from '../components/dataset_quality_details/overview/document_trends/degraded_docs/lens_attributes';
|
||||
import { getLensAttributes as getFailedLensAttributes } from '../components/dataset_quality_details/overview/document_trends/failed_docs/lens_attributes';
|
||||
import { QualityIssueType } from '../state_machines/dataset_quality_details_controller';
|
||||
import { useKibanaContextForPlugin } from '../utils';
|
||||
import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state';
|
||||
import { getLensAttributes } from '../components/dataset_quality_details/overview/document_trends/degraded_docs/lens_attributes';
|
||||
import { useCreateDataView } from './use_create_dataview';
|
||||
import { useDatasetDetailsTelemetry } from './use_dataset_details_telemetry';
|
||||
import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state';
|
||||
import { useRedirectLink } from './use_redirect_link';
|
||||
import { useDatasetDetailsRedirectLinkTelemetry } from './use_redirect_link_telemetry';
|
||||
|
||||
const openInLensText = i18n.translate('xpack.datasetQuality.details.chartOpenInLensText', {
|
||||
defaultMessage: 'Open in Lens',
|
||||
|
|
@ -24,7 +32,7 @@ const openInLensText = i18n.translate('xpack.datasetQuality.details.chartOpenInL
|
|||
|
||||
const ACTION_OPEN_IN_LENS = 'ACTION_OPEN_IN_LENS';
|
||||
|
||||
export const useDegradedDocsChart = () => {
|
||||
export const useQualityIssuesDocsChart = () => {
|
||||
const { euiTheme } = useEuiTheme();
|
||||
const {
|
||||
services: { lens },
|
||||
|
|
@ -34,6 +42,7 @@ export const useDegradedDocsChart = () => {
|
|||
dataStream,
|
||||
datasetDetails,
|
||||
timeRange,
|
||||
docsTrendChart,
|
||||
breakdownField,
|
||||
integrationDetails,
|
||||
isBreakdownFieldAsserted,
|
||||
|
|
@ -47,12 +56,16 @@ export const useDegradedDocsChart = () => {
|
|||
} = useDatasetDetailsTelemetry();
|
||||
|
||||
const [isChartLoading, setIsChartLoading] = useState<boolean | undefined>(undefined);
|
||||
const [attributes, setAttributes] = useState<ReturnType<typeof getLensAttributes> | undefined>(
|
||||
undefined
|
||||
);
|
||||
const [attributes, setAttributes] = useState<
|
||||
ReturnType<typeof getDegradedLensAttributes | typeof getFailedLensAttributes> | undefined
|
||||
>(undefined);
|
||||
|
||||
const query = docsTrendChart === 'degraded' ? DEGRADED_DOCS_QUERY : '';
|
||||
|
||||
const { dataView } = useCreateDataView({
|
||||
indexPatternString: getDataViewIndexPattern(dataStream),
|
||||
indexPatternString: getDataViewIndexPattern(
|
||||
docsTrendChart === 'degraded' ? dataStream : `${dataStream}${FAILURE_STORE_SELECTOR}`
|
||||
),
|
||||
});
|
||||
|
||||
const breakdownDataViewField = useMemo(
|
||||
|
|
@ -74,6 +87,16 @@ export const useDegradedDocsChart = () => {
|
|||
[service]
|
||||
);
|
||||
|
||||
const handleDocsTrendChartChange = useCallback(
|
||||
(qualityIssuesChart: QualityIssueType) => {
|
||||
service.send({
|
||||
type: 'QUALITY_ISSUES_CHART_CHANGE',
|
||||
qualityIssuesChart,
|
||||
});
|
||||
},
|
||||
[service]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (isBreakdownFieldAsserted) trackDatasetDetailsBreakdownFieldChanged();
|
||||
}, [trackDatasetDetailsBreakdownFieldChanged, isBreakdownFieldAsserted]);
|
||||
|
|
@ -84,18 +107,27 @@ export const useDegradedDocsChart = () => {
|
|||
integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ??
|
||||
datasetDetails.name;
|
||||
|
||||
const lensAttributes = getLensAttributes({
|
||||
color: euiTheme.colors.danger,
|
||||
dataStream: dataStreamName,
|
||||
datasetTitle,
|
||||
breakdownFieldName: breakdownDataViewField?.name,
|
||||
});
|
||||
const lensAttributes =
|
||||
docsTrendChart === 'degraded'
|
||||
? getDegradedLensAttributes({
|
||||
color: euiTheme.colors.danger,
|
||||
dataStream: dataStreamName,
|
||||
datasetTitle,
|
||||
breakdownFieldName: breakdownDataViewField?.name,
|
||||
})
|
||||
: getFailedLensAttributes({
|
||||
color: euiTheme.colors.danger,
|
||||
dataStream: dataStreamName,
|
||||
datasetTitle,
|
||||
breakdownFieldName: breakdownDataViewField?.name,
|
||||
});
|
||||
setAttributes(lensAttributes);
|
||||
}, [
|
||||
breakdownDataViewField?.name,
|
||||
euiTheme.colors.danger,
|
||||
setAttributes,
|
||||
dataStream,
|
||||
docsTrendChart,
|
||||
integrationDetails?.integration?.integration?.datasets,
|
||||
datasetDetails.name,
|
||||
]);
|
||||
|
|
@ -138,6 +170,20 @@ export const useDegradedDocsChart = () => {
|
|||
};
|
||||
}, [openInLensCallback]);
|
||||
|
||||
const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({
|
||||
query: { language: 'kuery', query },
|
||||
navigationSource: navigationSources.Chart,
|
||||
});
|
||||
|
||||
const redirectLinkProps = useRedirectLink({
|
||||
dataStreamStat: datasetDetails,
|
||||
query: { language: 'kuery', query },
|
||||
timeRangeConfig: timeRange,
|
||||
breakdownField: breakdownDataViewField?.name,
|
||||
sendTelemetry,
|
||||
selector: docsTrendChart === 'failed' ? FAILURE_STORE_SELECTOR : undefined,
|
||||
});
|
||||
|
||||
const extraActions: Action[] = [getOpenInLensAction];
|
||||
|
||||
const breakdown = useMemo(() => {
|
||||
|
|
@ -156,6 +202,8 @@ export const useDegradedDocsChart = () => {
|
|||
breakdown,
|
||||
extraActions,
|
||||
isChartLoading,
|
||||
redirectLinkProps,
|
||||
handleDocsTrendChartChange,
|
||||
onChartLoading: handleChartLoading,
|
||||
setAttributes,
|
||||
setIsChartLoading,
|
||||
|
|
@ -5,14 +5,14 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { useMemo } from 'react';
|
||||
import { DiscoverAppLocatorParams, DISCOVER_APP_LOCATOR } from '@kbn/discover-plugin/common';
|
||||
import { Query, AggregateQuery, buildPhraseFilter } from '@kbn/es-query';
|
||||
import { DISCOVER_APP_LOCATOR, DiscoverAppLocatorParams } from '@kbn/discover-plugin/common';
|
||||
import { AggregateQuery, Query, buildPhraseFilter } from '@kbn/es-query';
|
||||
import { getRouterLinkProps } from '@kbn/router-utils';
|
||||
import { RouterLinkProps } from '@kbn/router-utils/src/get_router_link_props';
|
||||
import { LocatorClient } from '@kbn/shared-ux-prompt-no-data-views-types';
|
||||
import { useMemo } from 'react';
|
||||
import { BasicDataStream, DataStreamSelector, TimeRangeConfig } from '../../common/types';
|
||||
import { useKibanaContextForPlugin } from '../utils';
|
||||
import { BasicDataStream, TimeRangeConfig } from '../../common/types';
|
||||
import { SendTelemetryFn } from './use_redirect_link_telemetry';
|
||||
|
||||
export const useRedirectLink = <T extends BasicDataStream>({
|
||||
|
|
@ -21,12 +21,14 @@ export const useRedirectLink = <T extends BasicDataStream>({
|
|||
timeRangeConfig,
|
||||
breakdownField,
|
||||
sendTelemetry,
|
||||
selector,
|
||||
}: {
|
||||
dataStreamStat: T;
|
||||
query?: Query | AggregateQuery;
|
||||
timeRangeConfig: TimeRangeConfig;
|
||||
breakdownField?: string;
|
||||
sendTelemetry: SendTelemetryFn;
|
||||
selector?: DataStreamSelector;
|
||||
}) => {
|
||||
const {
|
||||
services: { share },
|
||||
|
|
@ -46,6 +48,7 @@ export const useRedirectLink = <T extends BasicDataStream>({
|
|||
from,
|
||||
to,
|
||||
breakdownField,
|
||||
selector,
|
||||
});
|
||||
|
||||
const onClickWithTelemetry = (event: Parameters<RouterLinkProps['onClick']>[0]) => {
|
||||
|
|
@ -68,7 +71,16 @@ export const useRedirectLink = <T extends BasicDataStream>({
|
|||
navigate: navigateWithTelemetry,
|
||||
isLogsExplorerAvailable: false,
|
||||
};
|
||||
}, [breakdownField, dataStreamStat, from, to, query, sendTelemetry, share.url.locators]);
|
||||
}, [
|
||||
share.url.locators,
|
||||
dataStreamStat,
|
||||
query,
|
||||
from,
|
||||
to,
|
||||
breakdownField,
|
||||
selector,
|
||||
sendTelemetry,
|
||||
]);
|
||||
};
|
||||
|
||||
const buildDiscoverConfig = <T extends BasicDataStream>({
|
||||
|
|
@ -78,6 +90,7 @@ const buildDiscoverConfig = <T extends BasicDataStream>({
|
|||
from,
|
||||
to,
|
||||
breakdownField,
|
||||
selector,
|
||||
}: {
|
||||
locatorClient: LocatorClient;
|
||||
dataStreamStat: T;
|
||||
|
|
@ -85,15 +98,34 @@ const buildDiscoverConfig = <T extends BasicDataStream>({
|
|||
from: string;
|
||||
to: string;
|
||||
breakdownField?: string;
|
||||
selector?: DataStreamSelector;
|
||||
}): {
|
||||
navigate: () => void;
|
||||
routerLinkProps: RouterLinkProps;
|
||||
} => {
|
||||
const dataViewId = `${dataStreamStat.type}-${dataStreamStat.name}-*`;
|
||||
const dataViewNamespace = `${selector ? dataStreamStat.namespace : '*'}`;
|
||||
const dataViewSelector = selector ? `${selector}` : '';
|
||||
const dataViewId = `${dataStreamStat.type}-${dataStreamStat.name}-${dataViewNamespace}${dataViewSelector}`;
|
||||
const dataViewTitle = dataStreamStat.integration
|
||||
? `[${dataStreamStat.integration.title}] ${dataStreamStat.name}`
|
||||
? `[${dataStreamStat.integration.title}] ${dataStreamStat.name}-${dataViewNamespace}${dataViewSelector}`
|
||||
: `${dataViewId}`;
|
||||
|
||||
const filters = selector
|
||||
? []
|
||||
: [
|
||||
buildPhraseFilter(
|
||||
{
|
||||
name: 'data_stream.namespace',
|
||||
type: 'string',
|
||||
},
|
||||
dataStreamStat.namespace,
|
||||
{
|
||||
id: dataViewId,
|
||||
title: dataViewTitle,
|
||||
}
|
||||
),
|
||||
];
|
||||
|
||||
const params: DiscoverAppLocatorParams = {
|
||||
timeRange: {
|
||||
from,
|
||||
|
|
@ -112,19 +144,7 @@ const buildDiscoverConfig = <T extends BasicDataStream>({
|
|||
query,
|
||||
breakdownField,
|
||||
columns: [],
|
||||
filters: [
|
||||
buildPhraseFilter(
|
||||
{
|
||||
name: 'data_stream.namespace',
|
||||
type: 'string',
|
||||
},
|
||||
dataStreamStat.namespace,
|
||||
{
|
||||
id: dataViewId,
|
||||
title: dataViewTitle,
|
||||
}
|
||||
),
|
||||
],
|
||||
filters,
|
||||
interval: 'auto',
|
||||
sort: [['@timestamp', 'desc']],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import createContainer from 'constate';
|
||||
import { useSelector } from '@xstate/react';
|
||||
import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
|
||||
import createContainer from 'constate';
|
||||
import { countBy } from 'lodash';
|
||||
import { useDatasetQualityTable } from '.';
|
||||
import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
|
||||
import { QualityIndicators } from '../../common/types';
|
||||
import { useDatasetQualityContext } from '../components/dataset_quality/context';
|
||||
import { filterInactiveDatasets } from '../utils';
|
||||
|
||||
|
|
@ -27,9 +29,10 @@ const useSummaryPanel = () => {
|
|||
Datasets Quality
|
||||
*/
|
||||
|
||||
const datasetsQuality = {
|
||||
percentages: filteredItems.map((item) => item.degradedDocs.percentage),
|
||||
};
|
||||
const datasetsQuality = countBy(filteredItems.map((item) => item.quality)) as Record<
|
||||
QualityIndicators,
|
||||
number
|
||||
>;
|
||||
|
||||
const isDegradedDocsLoading = useSelector(service, (state) =>
|
||||
state.matches('stats.degradedDocs.fetching')
|
||||
|
|
|
|||
|
|
@ -24,11 +24,13 @@ export class DatasetQualityPlugin
|
|||
implements Plugin<DatasetQualityPluginSetup, DatasetQualityPluginStart>
|
||||
{
|
||||
private telemetry = new TelemetryService();
|
||||
private isServerless = false;
|
||||
|
||||
constructor(context: PluginInitializerContext) {}
|
||||
constructor(private context: PluginInitializerContext) {}
|
||||
|
||||
public setup(core: CoreSetup, plugins: DatasetQualitySetupDeps) {
|
||||
this.telemetry.setup({ analytics: core.analytics });
|
||||
this.isServerless = this.context.env.packageInfo.buildFlavor === 'serverless';
|
||||
|
||||
return {};
|
||||
}
|
||||
|
|
@ -48,6 +50,7 @@ export class DatasetQualityPlugin
|
|||
core,
|
||||
plugins,
|
||||
telemetryClient,
|
||||
isServerless: this.isServerless,
|
||||
});
|
||||
|
||||
const createDatasetQualityController = createDatasetQualityControllerLazyFactory({
|
||||
|
|
@ -59,6 +62,7 @@ export class DatasetQualityPlugin
|
|||
core,
|
||||
plugins,
|
||||
telemetryClient,
|
||||
isServerless: this.isServerless,
|
||||
});
|
||||
|
||||
const createDatasetQualityDetailsController = createDatasetQualityDetailsControllerLazyFactory({
|
||||
|
|
|
|||
|
|
@ -16,11 +16,15 @@ import {
|
|||
degradedFieldAnalysisRt,
|
||||
DegradedFieldValues,
|
||||
degradedFieldValuesRt,
|
||||
FailedDocsDetails,
|
||||
FailedDocsErrorsResponse,
|
||||
failedDocsErrorsRt,
|
||||
getDataStreamDegradedFieldsResponseRt,
|
||||
getDataStreamsDetailsResponseRt,
|
||||
getDataStreamsSettingsResponseRt,
|
||||
IntegrationDashboardsResponse,
|
||||
integrationDashboardsRT,
|
||||
qualityIssueBaseRT,
|
||||
UpdateFieldLimitResponse,
|
||||
updateFieldLimitResponseRt,
|
||||
} from '../../../common/api_types';
|
||||
|
|
@ -32,6 +36,8 @@ import {
|
|||
GetDataStreamDegradedFieldValuesPathParams,
|
||||
GetDataStreamDetailsParams,
|
||||
GetDataStreamDetailsResponse,
|
||||
GetDataStreamFailedDocsDetailsParams,
|
||||
GetDataStreamFailedDocsErrorsParams,
|
||||
GetDataStreamSettingsParams,
|
||||
GetDataStreamSettingsResponse,
|
||||
GetIntegrationDashboardsParams,
|
||||
|
|
@ -87,6 +93,59 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient {
|
|||
return dataStreamDetails as DataStreamDetails;
|
||||
}
|
||||
|
||||
public async getFailedDocsDetails({
|
||||
dataStream,
|
||||
start,
|
||||
end,
|
||||
}: GetDataStreamFailedDocsDetailsParams) {
|
||||
const response = await this.http
|
||||
.get<FailedDocsDetails>(`/internal/dataset_quality/data_streams/${dataStream}/failed_docs`, {
|
||||
query: { start, end },
|
||||
})
|
||||
.catch((error) => {
|
||||
throw new DatasetQualityError(
|
||||
`Failed to fetch data stream failed docs details": ${error}`,
|
||||
error
|
||||
);
|
||||
});
|
||||
|
||||
return decodeOrThrow(
|
||||
qualityIssueBaseRT,
|
||||
(message: string) =>
|
||||
new DatasetQualityError(
|
||||
`Failed to decode data stream failed docs details response: ${message}"`
|
||||
)
|
||||
)(response);
|
||||
}
|
||||
|
||||
public async getFailedDocsErrors({
|
||||
dataStream,
|
||||
start,
|
||||
end,
|
||||
}: GetDataStreamFailedDocsErrorsParams): Promise<FailedDocsErrorsResponse> {
|
||||
const response = await this.http
|
||||
.get<FailedDocsDetails>(
|
||||
`/internal/dataset_quality/data_streams/${dataStream}/failed_docs/errors`,
|
||||
{
|
||||
query: { start, end },
|
||||
}
|
||||
)
|
||||
.catch((error) => {
|
||||
throw new DatasetQualityError(
|
||||
`Failed to fetch data stream failed docs details": ${error}`,
|
||||
error
|
||||
);
|
||||
});
|
||||
|
||||
return decodeOrThrow(
|
||||
failedDocsErrorsRt,
|
||||
(message: string) =>
|
||||
new DatasetQualityError(
|
||||
`Failed to decode data stream failed docs details response: ${message}"`
|
||||
)
|
||||
)(response);
|
||||
}
|
||||
|
||||
public async getDataStreamDegradedFields({
|
||||
dataStream,
|
||||
start,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import {
|
|||
GetDataStreamDegradedFieldsParams,
|
||||
DegradedFieldResponse,
|
||||
GetDataStreamDegradedFieldValuesPathParams,
|
||||
GetDataStreamFailedDocsDetailsParams,
|
||||
GetDataStreamFailedDocsErrorsParams,
|
||||
} from '../../../common/data_streams_stats';
|
||||
import {
|
||||
AnalyzeDegradedFieldsParams,
|
||||
|
|
@ -27,6 +29,8 @@ import {
|
|||
DataStreamRolloverResponse,
|
||||
DegradedFieldAnalysis,
|
||||
DegradedFieldValues,
|
||||
FailedDocsDetails,
|
||||
FailedDocsErrorsResponse,
|
||||
UpdateFieldLimitResponse,
|
||||
} from '../../../common/api_types';
|
||||
|
||||
|
|
@ -43,6 +47,10 @@ export interface DataStreamDetailsServiceStartDeps {
|
|||
export interface IDataStreamDetailsClient {
|
||||
getDataStreamSettings(params: GetDataStreamSettingsParams): Promise<DataStreamSettings>;
|
||||
getDataStreamDetails(params: GetDataStreamDetailsParams): Promise<DataStreamDetails>;
|
||||
getFailedDocsDetails(params: GetDataStreamFailedDocsDetailsParams): Promise<FailedDocsDetails>;
|
||||
getFailedDocsErrors(
|
||||
params: GetDataStreamFailedDocsErrorsParams
|
||||
): Promise<FailedDocsErrorsResponse>;
|
||||
getDataStreamDegradedFields(
|
||||
params: GetDataStreamDegradedFieldsParams
|
||||
): Promise<DegradedFieldResponse>;
|
||||
|
|
|
|||
|
|
@ -8,11 +8,12 @@
|
|||
import { HttpStart } from '@kbn/core/public';
|
||||
import { decodeOrThrow } from '@kbn/io-ts-utils';
|
||||
import rison from '@kbn/rison';
|
||||
import { KNOWN_TYPES } from '../../../common/constants';
|
||||
import {
|
||||
DataStreamDegradedDocsResponse,
|
||||
DataStreamFailedDocsResponse,
|
||||
DataStreamTotalDocsResponse,
|
||||
getDataStreamDegradedDocsResponseRt,
|
||||
getDataStreamFailedDocsResponseRt,
|
||||
getDataStreamsStatsResponseRt,
|
||||
getDataStreamTotalDocsResponseRt,
|
||||
getIntegrationsResponseRt,
|
||||
|
|
@ -20,17 +21,19 @@ import {
|
|||
IntegrationsResponse,
|
||||
NonAggregatableDatasets,
|
||||
} from '../../../common/api_types';
|
||||
import { KNOWN_TYPES } from '../../../common/constants';
|
||||
import {
|
||||
DataStreamStatServiceResponse,
|
||||
GetDataStreamsDegradedDocsStatsQuery,
|
||||
GetDataStreamsFailedDocsStatsQuery,
|
||||
GetDataStreamsStatsQuery,
|
||||
GetDataStreamsStatsResponse,
|
||||
GetDataStreamsTotalDocsQuery,
|
||||
GetNonAggregatableDataStreamsParams,
|
||||
} from '../../../common/data_streams_stats';
|
||||
import { Integration } from '../../../common/data_streams_stats/integration';
|
||||
import { IDataStreamsStatsClient } from './types';
|
||||
import { DatasetQualityError } from '../../../common/errors';
|
||||
import { IDataStreamsStatsClient } from './types';
|
||||
|
||||
export class DataStreamsStatsClient implements IDataStreamsStatsClient {
|
||||
constructor(private readonly http: HttpStart) {}
|
||||
|
|
@ -108,6 +111,30 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
|
|||
return degradedDocs;
|
||||
}
|
||||
|
||||
public async getDataStreamsFailedStats(params: GetDataStreamsFailedDocsStatsQuery) {
|
||||
const types = params.types.length === 0 ? KNOWN_TYPES : params.types;
|
||||
const response = await this.http
|
||||
.get<DataStreamFailedDocsResponse>('/internal/dataset_quality/data_streams/failed_docs', {
|
||||
query: {
|
||||
...params,
|
||||
types: rison.encodeArray(types),
|
||||
},
|
||||
})
|
||||
.catch((error) => {
|
||||
throw new DatasetQualityError(`Failed to fetch data streams failed stats: ${error}`, error);
|
||||
});
|
||||
|
||||
const { failedDocs } = decodeOrThrow(
|
||||
getDataStreamFailedDocsResponseRt,
|
||||
(message: string) =>
|
||||
new DatasetQualityError(
|
||||
`Failed to decode data streams failed docs stats response: ${message}`
|
||||
)
|
||||
)(response);
|
||||
|
||||
return failedDocs;
|
||||
}
|
||||
|
||||
public async getNonAggregatableDatasets(params: GetNonAggregatableDataStreamsParams) {
|
||||
const types = params.types.length === 0 ? KNOWN_TYPES : params.types;
|
||||
const response = await this.http
|
||||
|
|
|
|||
|
|
@ -6,15 +6,16 @@
|
|||
*/
|
||||
|
||||
import { HttpStart } from '@kbn/core/public';
|
||||
import { DataStreamDocsStat, NonAggregatableDatasets } from '../../../common/api_types';
|
||||
import {
|
||||
DataStreamStatServiceResponse,
|
||||
GetDataStreamsDegradedDocsStatsQuery,
|
||||
GetDataStreamsFailedDocsStatsQuery,
|
||||
GetDataStreamsStatsQuery,
|
||||
GetDataStreamsTotalDocsQuery,
|
||||
GetNonAggregatableDataStreamsParams,
|
||||
} from '../../../common/data_streams_stats';
|
||||
import { Integration } from '../../../common/data_streams_stats/integration';
|
||||
import { DataStreamDocsStat, NonAggregatableDatasets } from '../../../common/api_types';
|
||||
|
||||
export type DataStreamsStatsServiceSetup = void;
|
||||
|
||||
|
|
@ -31,6 +32,9 @@ export interface IDataStreamsStatsClient {
|
|||
getDataStreamsDegradedStats(
|
||||
params?: GetDataStreamsDegradedDocsStatsQuery
|
||||
): Promise<DataStreamDocsStat[]>;
|
||||
getDataStreamsFailedStats(
|
||||
params?: GetDataStreamsFailedDocsStatsQuery
|
||||
): Promise<DataStreamDocsStat[]>;
|
||||
getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery): Promise<DataStreamDocsStat[]>;
|
||||
getIntegrations(): Promise<Integration[]>;
|
||||
getNonAggregatableDatasets(
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
|
|||
},
|
||||
dataStreamStats: [],
|
||||
degradedDocStats: [],
|
||||
failedDocStats: [],
|
||||
totalDocsStats: DEFAULT_DICTIONARY_TYPE,
|
||||
filters: {
|
||||
inactive: true,
|
||||
|
|
|
|||
|
|
@ -47,3 +47,12 @@ export const fetchIntegrationsFailedNotifier = (toasts: IToasts, error: Error) =
|
|||
text: error.message,
|
||||
});
|
||||
};
|
||||
|
||||
export const fetchFailedStatsFailedNotifier = (toasts: IToasts, error: Error) => {
|
||||
toasts.addDanger({
|
||||
title: i18n.translate('xpack.datasetQuality.fetchFailedStatsFailed', {
|
||||
defaultMessage: "We couldn't get your failed docs information.",
|
||||
}),
|
||||
text: error.message,
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import { DEFAULT_CONTEXT } from './defaults';
|
|||
import {
|
||||
fetchDatasetStatsFailedNotifier,
|
||||
fetchDegradedStatsFailedNotifier,
|
||||
fetchFailedStatsFailedNotifier,
|
||||
fetchIntegrationsFailedNotifier,
|
||||
fetchTotalDocsFailedNotifier,
|
||||
} from './notifications';
|
||||
|
|
@ -125,6 +126,46 @@ export const createPureDatasetQualityControllerStateMachine = (
|
|||
},
|
||||
},
|
||||
},
|
||||
failedDocs: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'loadFailedDocs',
|
||||
onDone: {
|
||||
target: 'loaded',
|
||||
actions: ['storeFailedDocStats', 'storeDatasets'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: 'notImplemented',
|
||||
cond: 'checkIfNotImplemented',
|
||||
},
|
||||
{
|
||||
target: 'unauthorized',
|
||||
cond: 'checkIfActionForbidden',
|
||||
},
|
||||
{
|
||||
target: 'loaded',
|
||||
actions: ['notifyFetchFailedStatsFailed'],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
loaded: {},
|
||||
notImplemented: {},
|
||||
unauthorized: { type: 'final' },
|
||||
},
|
||||
on: {
|
||||
UPDATE_TIME_RANGE: {
|
||||
target: 'failedDocs.fetching',
|
||||
actions: ['storeTimeRange'],
|
||||
},
|
||||
REFRESH_DATA: {
|
||||
target: 'failedDocs.fetching',
|
||||
},
|
||||
},
|
||||
},
|
||||
docsStats: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
|
|
@ -381,6 +422,9 @@ export const createPureDatasetQualityControllerStateMachine = (
|
|||
storeDegradedDocStats: assign((_context, event: DoneInvokeEvent<DataStreamDocsStat[]>) => ({
|
||||
degradedDocStats: event.data,
|
||||
})),
|
||||
storeFailedDocStats: assign((_context, event: DoneInvokeEvent<DataStreamDocsStat[]>) => ({
|
||||
failedDocStats: event.data,
|
||||
})),
|
||||
storeNonAggregatableDatasets: assign(
|
||||
(_context, event: DoneInvokeEvent<NonAggregatableDatasets>) => ({
|
||||
nonAggregatableDatasets: event.data.datasets,
|
||||
|
|
@ -404,6 +448,7 @@ export const createPureDatasetQualityControllerStateMachine = (
|
|||
datasets: generateDatasets(
|
||||
context.dataStreamStats,
|
||||
context.degradedDocStats,
|
||||
context.failedDocStats,
|
||||
context.integrations,
|
||||
context.totalDocsStats
|
||||
),
|
||||
|
|
@ -412,7 +457,7 @@ export const createPureDatasetQualityControllerStateMachine = (
|
|||
}),
|
||||
},
|
||||
guards: {
|
||||
checkIfActionForbidden: (context, event) => {
|
||||
checkIfActionForbidden: (_context, event) => {
|
||||
return (
|
||||
'data' in event &&
|
||||
typeof event.data === 'object' &&
|
||||
|
|
@ -420,6 +465,14 @@ export const createPureDatasetQualityControllerStateMachine = (
|
|||
event.data.statusCode === 403
|
||||
);
|
||||
},
|
||||
checkIfNotImplemented: (_context, event) => {
|
||||
return (
|
||||
'data' in event &&
|
||||
typeof event.data === 'object' &&
|
||||
'statusCode' in event.data! &&
|
||||
event.data.statusCode === 501
|
||||
);
|
||||
},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
|
@ -447,6 +500,8 @@ export const createDatasetQualityControllerStateMachine = ({
|
|||
fetchIntegrationsFailedNotifier(toasts, event.data),
|
||||
notifyFetchTotalDocsFailed: (_context, event: DoneInvokeEvent<Error>, meta) =>
|
||||
fetchTotalDocsFailedNotifier(toasts, event.data, meta),
|
||||
notifyFetchFailedStatsFailed: (_context, event: DoneInvokeEvent<Error>) =>
|
||||
fetchFailedStatsFailedNotifier(toasts, event.data),
|
||||
},
|
||||
services: {
|
||||
loadDataStreamStats: (context, _event) =>
|
||||
|
|
@ -489,6 +544,16 @@ export const createDatasetQualityControllerStateMachine = ({
|
|||
end,
|
||||
});
|
||||
},
|
||||
loadFailedDocs: (context) => {
|
||||
const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange);
|
||||
|
||||
return dataStreamStatsClient.getDataStreamsFailedStats({
|
||||
types: context.filters.types as DataStreamType[],
|
||||
datasetQuery: context.filters.query,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
},
|
||||
loadNonAggregatableDatasets: (context) => {
|
||||
const { startDate: start, endDate: end } = getDateISORange(context.filters.timeRange);
|
||||
|
||||
|
|
|
|||
|
|
@ -60,6 +60,10 @@ export interface WithDegradedDocs {
|
|||
degradedDocStats: DataStreamDocsStat[];
|
||||
}
|
||||
|
||||
export interface WithFailedDocs {
|
||||
failedDocStats: DataStreamDocsStat[];
|
||||
}
|
||||
|
||||
export interface WithNonAggregatableDatasets {
|
||||
nonAggregatableDatasets: string[];
|
||||
}
|
||||
|
|
@ -76,6 +80,7 @@ export type DefaultDatasetQualityControllerState = WithTableOptions &
|
|||
WithDataStreamStats &
|
||||
WithTotalDocs &
|
||||
WithDegradedDocs &
|
||||
WithFailedDocs &
|
||||
WithDatasets &
|
||||
WithFilters &
|
||||
WithNonAggregatableDatasets &
|
||||
|
|
@ -92,10 +97,18 @@ export type DatasetQualityControllerTypeState =
|
|||
value: 'stats.datasets.loaded';
|
||||
context: DefaultDatasetQualityStateContext;
|
||||
}
|
||||
| {
|
||||
value: 'stats.docsStats.fetching';
|
||||
context: DefaultDatasetQualityStateContext;
|
||||
}
|
||||
| {
|
||||
value: 'stats.degradedDocs.fetching';
|
||||
context: DefaultDatasetQualityStateContext;
|
||||
}
|
||||
| {
|
||||
value: 'stats.failedDocs.fetching';
|
||||
context: DefaultDatasetQualityStateContext;
|
||||
}
|
||||
| {
|
||||
value: 'stats.nonAggregatableDatasets.fetching';
|
||||
context: DefaultDatasetQualityStateContext;
|
||||
|
|
|
|||
|
|
@ -7,20 +7,32 @@
|
|||
|
||||
import {
|
||||
DEFAULT_DATEPICKER_REFRESH,
|
||||
DEFAULT_DEGRADED_FIELD_SORT_DIRECTION,
|
||||
DEFAULT_DEGRADED_FIELD_SORT_FIELD,
|
||||
DEFAULT_FAILED_DOCS_ERROR_SORT_DIRECTION,
|
||||
DEFAULT_FAILED_DOCS_ERROR_SORT_FIELD,
|
||||
DEFAULT_QUALITY_ISSUE_SORT_DIRECTION,
|
||||
DEFAULT_QUALITY_ISSUE_SORT_FIELD,
|
||||
DEFAULT_TIME_RANGE,
|
||||
} from '../../../common/constants';
|
||||
import { DefaultDatasetQualityDetailsContext } from './types';
|
||||
import { DefaultDatasetQualityDetailsContext, QualityIssueType } from './types';
|
||||
|
||||
export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = {
|
||||
degradedFields: {
|
||||
qualityIssues: {
|
||||
table: {
|
||||
page: 0,
|
||||
rowsPerPage: 10,
|
||||
sort: {
|
||||
field: DEFAULT_DEGRADED_FIELD_SORT_FIELD,
|
||||
direction: DEFAULT_DEGRADED_FIELD_SORT_DIRECTION,
|
||||
field: DEFAULT_QUALITY_ISSUE_SORT_FIELD,
|
||||
direction: DEFAULT_QUALITY_ISSUE_SORT_DIRECTION,
|
||||
},
|
||||
},
|
||||
},
|
||||
failedDocsErrors: {
|
||||
table: {
|
||||
page: 0,
|
||||
rowsPerPage: 10,
|
||||
sort: {
|
||||
field: DEFAULT_FAILED_DOCS_ERROR_SORT_FIELD,
|
||||
direction: DEFAULT_FAILED_DOCS_ERROR_SORT_DIRECTION,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -30,4 +42,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = {
|
|||
refresh: DEFAULT_DATEPICKER_REFRESH,
|
||||
},
|
||||
showCurrentQualityIssues: false,
|
||||
qualityIssuesChart: 'degraded' as QualityIssueType,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,18 +5,9 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom, raise } from 'xstate';
|
||||
import { getDateISORange } from '@kbn/timerange';
|
||||
import type { IToasts } from '@kbn/core-notifications-browser';
|
||||
import {
|
||||
DatasetQualityDetailsControllerContext,
|
||||
DatasetQualityDetailsControllerEvent,
|
||||
DatasetQualityDetailsControllerTypeState,
|
||||
} from './types';
|
||||
import { DatasetQualityStartDeps } from '../../types';
|
||||
import { IDataStreamsStatsClient } from '../../services/data_streams_stats';
|
||||
import { IDataStreamDetailsClient } from '../../services/data_stream_details';
|
||||
import { indexNameToDataStreamParts } from '../../../common/utils';
|
||||
import { getDateISORange } from '@kbn/timerange';
|
||||
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom, raise } from 'xstate';
|
||||
import {
|
||||
Dashboard,
|
||||
DataStreamDetails,
|
||||
|
|
@ -24,20 +15,33 @@ import {
|
|||
DegradedFieldAnalysis,
|
||||
DegradedFieldResponse,
|
||||
DegradedFieldValues,
|
||||
FailedDocsDetails,
|
||||
FailedDocsErrorsResponse,
|
||||
NonAggregatableDatasets,
|
||||
QualityIssue,
|
||||
UpdateFieldLimitResponse,
|
||||
} from '../../../common/api_types';
|
||||
import { indexNameToDataStreamParts } from '../../../common/utils';
|
||||
import { IDataStreamDetailsClient } from '../../services/data_stream_details';
|
||||
import { IDataStreamsStatsClient } from '../../services/data_streams_stats';
|
||||
import { DatasetQualityStartDeps } from '../../types';
|
||||
import { fetchNonAggregatableDatasetsFailedNotifier } from '../common/notifications';
|
||||
import {
|
||||
DatasetQualityDetailsControllerContext,
|
||||
DatasetQualityDetailsControllerEvent,
|
||||
DatasetQualityDetailsControllerTypeState,
|
||||
QualityIssueType,
|
||||
} from './types';
|
||||
|
||||
import { IntegrationType } from '../../../common/data_stream_details';
|
||||
import {
|
||||
fetchDataStreamDetailsFailedNotifier,
|
||||
assertBreakdownFieldEcsFailedNotifier,
|
||||
fetchDataStreamSettingsFailedNotifier,
|
||||
fetchDataStreamDetailsFailedNotifier,
|
||||
fetchDataStreamIntegrationFailedNotifier,
|
||||
fetchDataStreamSettingsFailedNotifier,
|
||||
fetchIntegrationDashboardsFailedNotifier,
|
||||
updateFieldLimitFailedNotifier,
|
||||
rolloverDataStreamFailedNotifier,
|
||||
updateFieldLimitFailedNotifier,
|
||||
} from './notifications';
|
||||
|
||||
export const createPureDatasetQualityDetailsControllerStateMachine = (
|
||||
|
|
@ -122,6 +126,10 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
'#DatasetQualityDetailsController.initializing.checkBreakdownFieldIsEcs.fetching',
|
||||
actions: ['storeBreakDownField'],
|
||||
},
|
||||
QUALITY_ISSUES_CHART_CHANGE: {
|
||||
target: 'done',
|
||||
actions: ['storeQualityIssuesChart'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -152,7 +160,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
invoke: {
|
||||
src: 'loadDataStreamSettings',
|
||||
onDone: {
|
||||
target: 'fetchingDataStreamDegradedFields',
|
||||
target: 'qualityIssues',
|
||||
actions: ['storeDataStreamSettings'],
|
||||
},
|
||||
onError: [
|
||||
|
|
@ -168,42 +176,95 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
},
|
||||
},
|
||||
errorFetchingDataStreamSettings: {},
|
||||
fetchingDataStreamDegradedFields: {
|
||||
invoke: {
|
||||
src: 'loadDegradedFields',
|
||||
onDone: {
|
||||
target: 'doneFetchingDegradedFields',
|
||||
actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'],
|
||||
qualityIssues: {
|
||||
type: 'parallel',
|
||||
states: {
|
||||
dataStreamDegradedFields: {
|
||||
initial: 'fetchingDataStreamDegradedFields',
|
||||
states: {
|
||||
fetchingDataStreamDegradedFields: {
|
||||
invoke: {
|
||||
src: 'loadDegradedFields',
|
||||
onDone: {
|
||||
target: 'doneFetchingDegradedFields',
|
||||
actions: ['storeDegradedFields'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'errorFetchingDegradedFields',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
errorFetchingDegradedFields: {},
|
||||
doneFetchingDegradedFields: {
|
||||
type: 'final',
|
||||
},
|
||||
},
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
dataStreamFailedDocs: {
|
||||
initial: 'fetchingFailedDocs',
|
||||
states: {
|
||||
fetchingFailedDocs: {
|
||||
invoke: {
|
||||
src: 'loadFailedDocsDetails',
|
||||
onDone: {
|
||||
target: 'doneFetchingFailedDocs',
|
||||
actions: ['storeFailedDocsDetails'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: 'notImplemented',
|
||||
cond: 'checkIfNotImplemented',
|
||||
},
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'errorFetchingFailedDocs',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
notImplemented: {
|
||||
type: 'final',
|
||||
},
|
||||
errorFetchingFailedDocs: {},
|
||||
doneFetchingFailedDocs: {
|
||||
type: 'final',
|
||||
},
|
||||
},
|
||||
{
|
||||
target: 'errorFetchingDegradedFields',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
onDone: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.dataStreamSettings.doneFetchingQualityIssues',
|
||||
},
|
||||
},
|
||||
doneFetchingDegradedFields: {
|
||||
doneFetchingQualityIssues: {
|
||||
entry: ['raiseDegradedFieldsLoaded'],
|
||||
on: {
|
||||
UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: {
|
||||
target: 'doneFetchingDegradedFields',
|
||||
actions: ['storeDegradedFieldTableOptions'],
|
||||
UPDATE_QUALITY_ISSUES_TABLE_CRITERIA: {
|
||||
target: 'doneFetchingQualityIssues',
|
||||
actions: ['storeQualityIssuesTableOptions'],
|
||||
},
|
||||
OPEN_DEGRADED_FIELD_FLYOUT: {
|
||||
OPEN_QUALITY_ISSUE_FLYOUT: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
|
||||
actions: ['storeExpandedDegradedField', 'resetFieldLimitServerResponse'],
|
||||
'#DatasetQualityDetailsController.initializing.qualityIssueFlyout.open',
|
||||
actions: ['storeExpandedQualityIssue', 'resetFieldLimitServerResponse'],
|
||||
},
|
||||
TOGGLE_CURRENT_QUALITY_ISSUES: {
|
||||
target: 'fetchingDataStreamDegradedFields',
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.fetchingDataStreamDegradedFields',
|
||||
actions: ['toggleCurrentQualityIssues'],
|
||||
},
|
||||
},
|
||||
},
|
||||
errorFetchingDegradedFields: {},
|
||||
},
|
||||
on: {
|
||||
UPDATE_TIME_RANGE: {
|
||||
|
|
@ -259,21 +320,68 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
done: {},
|
||||
},
|
||||
},
|
||||
degradedFieldFlyout: {
|
||||
qualityIssueFlyout: {
|
||||
initial: 'pending',
|
||||
states: {
|
||||
pending: {
|
||||
always: [
|
||||
{
|
||||
target: 'closed',
|
||||
cond: 'hasNoDegradedFieldsSelected',
|
||||
cond: 'hasNoQualityIssueSelected',
|
||||
},
|
||||
],
|
||||
},
|
||||
open: {
|
||||
initial: 'initialized',
|
||||
initial: 'initializing',
|
||||
states: {
|
||||
initialized: {
|
||||
initializing: {
|
||||
always: [
|
||||
{
|
||||
target: 'degradedFieldFlyout',
|
||||
cond: 'isDegradedFieldFlyout',
|
||||
},
|
||||
{
|
||||
target: 'failedDocsFlyout',
|
||||
},
|
||||
],
|
||||
},
|
||||
failedDocsFlyout: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'loadfailedDocsErrors',
|
||||
onDone: {
|
||||
target: 'done',
|
||||
actions: ['storeFailedDocsErrors'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: 'unsupported',
|
||||
cond: 'checkIfNotImplemented',
|
||||
},
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'done',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
done: {
|
||||
on: {
|
||||
UPDATE_FAILED_DOCS_ERRORS_TABLE_CRITERIA: {
|
||||
target: 'done',
|
||||
actions: ['storeFailedDocsErrorsTableOptions'],
|
||||
},
|
||||
},
|
||||
},
|
||||
unsupported: {},
|
||||
},
|
||||
},
|
||||
degradedFieldFlyout: {
|
||||
type: 'parallel',
|
||||
states: {
|
||||
ignoredValues: {
|
||||
|
|
@ -376,20 +484,20 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
on: {
|
||||
CLOSE_DEGRADED_FIELD_FLYOUT: {
|
||||
target: 'closed',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
actions: ['storeExpandedQualityIssue'],
|
||||
},
|
||||
UPDATE_TIME_RANGE: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
|
||||
'#DatasetQualityDetailsController.initializing.qualityIssueFlyout.open',
|
||||
},
|
||||
},
|
||||
},
|
||||
closed: {
|
||||
on: {
|
||||
OPEN_DEGRADED_FIELD_FLYOUT: {
|
||||
OPEN_QUALITY_ISSUE_FLYOUT: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
'#DatasetQualityDetailsController.initializing.qualityIssueFlyout.open',
|
||||
actions: ['storeExpandedQualityIssue'],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -402,7 +510,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
},
|
||||
{
|
||||
target: '.closed',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
actions: ['storeExpandedQualityIssue'],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
|
@ -437,6 +545,11 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
}
|
||||
: {};
|
||||
}),
|
||||
storeQualityIssuesChart: assign((_context, event) => {
|
||||
return 'qualityIssuesChart' in event
|
||||
? { qualityIssuesChart: event.qualityIssuesChart }
|
||||
: {};
|
||||
}),
|
||||
storeBreakDownField: assign((_context, event) => {
|
||||
return 'breakdownField' in event ? { breakdownField: event.breakdownField } : {};
|
||||
}),
|
||||
|
|
@ -447,12 +560,55 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
}
|
||||
: {};
|
||||
}),
|
||||
storeFailedDocsDetails: assign((context, event: DoneInvokeEvent<FailedDocsDetails>) => {
|
||||
return 'data' in event
|
||||
? {
|
||||
qualityIssues: {
|
||||
...context.qualityIssues,
|
||||
data: [
|
||||
...(context.qualityIssues.data ?? []).filter(
|
||||
(field) => field.type !== 'failed'
|
||||
),
|
||||
...(event.data.timeSeries.length > 0
|
||||
? [
|
||||
{
|
||||
...event.data,
|
||||
name: 'failedDocs',
|
||||
type: 'failed' as QualityIssueType,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
},
|
||||
}
|
||||
: {};
|
||||
}),
|
||||
storeFailedDocsErrors: assign(
|
||||
(context, event: DoneInvokeEvent<FailedDocsErrorsResponse>) => {
|
||||
return 'data' in event
|
||||
? {
|
||||
failedDocsErrors: {
|
||||
...context.failedDocsErrors,
|
||||
data: event.data.errors,
|
||||
},
|
||||
}
|
||||
: {};
|
||||
}
|
||||
),
|
||||
storeDegradedFields: assign((context, event: DoneInvokeEvent<DegradedFieldResponse>) => {
|
||||
return 'data' in event
|
||||
? {
|
||||
degradedFields: {
|
||||
...context.degradedFields,
|
||||
data: event.data.degradedFields,
|
||||
qualityIssues: {
|
||||
...context.qualityIssues,
|
||||
data: [
|
||||
...(context.qualityIssues.data ?? []).filter(
|
||||
(field) => field.type !== 'degraded'
|
||||
),
|
||||
...(event.data.degradedFields.map((field) => ({
|
||||
...field,
|
||||
type: 'degraded',
|
||||
})) as QualityIssue[]),
|
||||
],
|
||||
},
|
||||
}
|
||||
: {};
|
||||
|
|
@ -471,19 +627,32 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
}
|
||||
: {};
|
||||
}),
|
||||
storeDegradedFieldTableOptions: assign((context, event) => {
|
||||
return 'degraded_field_criteria' in event
|
||||
storeQualityIssuesTableOptions: assign((context, event) => {
|
||||
return 'quality_issues_criteria' in event
|
||||
? {
|
||||
degradedFields: {
|
||||
...context.degradedFields,
|
||||
table: event.degraded_field_criteria,
|
||||
qualityIssues: {
|
||||
...context.qualityIssues,
|
||||
table: event.quality_issues_criteria,
|
||||
},
|
||||
}
|
||||
: {};
|
||||
}),
|
||||
storeExpandedDegradedField: assign((_, event) => {
|
||||
storeFailedDocsErrorsTableOptions: assign((context, event) => {
|
||||
return 'failed_docs_errors_criteria' in event
|
||||
? {
|
||||
failedDocsErrors: {
|
||||
...context.failedDocsErrors,
|
||||
table: event.failed_docs_errors_criteria,
|
||||
},
|
||||
}
|
||||
: {};
|
||||
}),
|
||||
storeExpandedQualityIssue: assign((_, event) => {
|
||||
return {
|
||||
expandedDegradedField: 'fieldName' in event ? event.fieldName : undefined,
|
||||
expandedQualityIssue:
|
||||
'qualityIssue' in event
|
||||
? { name: event.qualityIssue.name, type: event.qualityIssue.type }
|
||||
: undefined,
|
||||
};
|
||||
}),
|
||||
toggleCurrentQualityIssues: assign((context) => {
|
||||
|
|
@ -492,16 +661,6 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
};
|
||||
}),
|
||||
raiseDegradedFieldsLoaded: raise('DEGRADED_FIELDS_LOADED'),
|
||||
resetDegradedFieldPageAndRowsPerPage: assign((context, _event) => ({
|
||||
degradedFields: {
|
||||
...context.degradedFields,
|
||||
table: {
|
||||
...context.degradedFields.table,
|
||||
page: 0,
|
||||
rowsPerPage: 10,
|
||||
},
|
||||
},
|
||||
})),
|
||||
storeDataStreamSettings: assign((_context, event: DoneInvokeEvent<DataStreamSettings>) => {
|
||||
return 'data' in event
|
||||
? {
|
||||
|
|
@ -557,6 +716,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
event.data.statusCode === 403
|
||||
);
|
||||
},
|
||||
checkIfNotImplemented: (_context, event) => {
|
||||
return (
|
||||
'data' in event &&
|
||||
typeof event.data === 'object' &&
|
||||
'statusCode' in event.data! &&
|
||||
event.data.statusCode === 501
|
||||
);
|
||||
},
|
||||
isIndexNotFoundError: (_, event) => {
|
||||
return (
|
||||
('data' in event &&
|
||||
|
|
@ -568,18 +735,23 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
false
|
||||
);
|
||||
},
|
||||
shouldOpenFlyout: (context) => {
|
||||
shouldOpenFlyout: (context, _event, meta) => {
|
||||
return (
|
||||
Boolean(context.expandedDegradedField) &&
|
||||
Boolean(context.expandedQualityIssue) &&
|
||||
Boolean(
|
||||
context.degradedFields.data?.some(
|
||||
(field) => field.name === context.expandedDegradedField
|
||||
context.qualityIssues.data?.some(
|
||||
(field) => field.name === context.expandedQualityIssue?.name
|
||||
)
|
||||
)
|
||||
);
|
||||
},
|
||||
hasNoDegradedFieldsSelected: (context) => {
|
||||
return !Boolean(context.expandedDegradedField);
|
||||
isDegradedFieldFlyout: (context) => {
|
||||
return Boolean(
|
||||
context.expandedQualityIssue && context.expandedQualityIssue.type === 'degraded'
|
||||
);
|
||||
},
|
||||
hasNoQualityIssueSelected: (context) => {
|
||||
return !Boolean(context.expandedQualityIssue);
|
||||
},
|
||||
hasFailedToUpdateLastBackingIndex: (_, event) => {
|
||||
return (
|
||||
|
|
@ -678,6 +850,15 @@ export const createDatasetQualityDetailsControllerStateMachine = ({
|
|||
|
||||
return false;
|
||||
},
|
||||
loadFailedDocsDetails: (context) => {
|
||||
const { startDate: start, endDate: end } = getDateISORange(context.timeRange);
|
||||
|
||||
return dataStreamDetailsClient.getFailedDocsDetails({
|
||||
dataStream: context.dataStream,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
},
|
||||
loadDegradedFields: (context) => {
|
||||
const { startDate: start, endDate: end } = getDateISORange(context.timeRange);
|
||||
|
||||
|
|
@ -699,30 +880,42 @@ export const createDatasetQualityDetailsControllerStateMachine = ({
|
|||
},
|
||||
|
||||
loadDegradedFieldValues: (context) => {
|
||||
if ('expandedDegradedField' in context && context.expandedDegradedField) {
|
||||
if ('expandedQualityIssue' in context && context.expandedQualityIssue) {
|
||||
return dataStreamDetailsClient.getDataStreamDegradedFieldValues({
|
||||
dataStream: context.dataStream,
|
||||
degradedField: context.expandedDegradedField,
|
||||
degradedField: context.expandedQualityIssue.name,
|
||||
});
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
analyzeDegradedField: (context) => {
|
||||
if (context?.degradedFields?.data?.length) {
|
||||
const selectedDegradedField = context.degradedFields.data.find(
|
||||
(field) => field.name === context.expandedDegradedField
|
||||
if (context?.qualityIssues?.data?.length) {
|
||||
const selectedDegradedField = context.qualityIssues.data.find(
|
||||
(field) => field.name === context.expandedQualityIssue?.name
|
||||
);
|
||||
|
||||
if (selectedDegradedField) {
|
||||
if (selectedDegradedField && selectedDegradedField.type === 'degraded') {
|
||||
return dataStreamDetailsClient.analyzeDegradedField({
|
||||
dataStream: context.dataStream,
|
||||
degradedField: context.expandedDegradedField!,
|
||||
lastBackingIndex: selectedDegradedField.indexFieldWasLastPresentIn,
|
||||
degradedField: context.expandedQualityIssue?.name!,
|
||||
lastBackingIndex: selectedDegradedField.indexFieldWasLastPresentIn!,
|
||||
});
|
||||
}
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
loadfailedDocsErrors: (context) => {
|
||||
if ('expandedQualityIssue' in context && context.expandedQualityIssue) {
|
||||
const { startDate: start, endDate: end } = getDateISORange(context.timeRange);
|
||||
|
||||
return dataStreamDetailsClient.getFailedDocsErrors({
|
||||
dataStream: context.dataStream,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
loadDataStreamSettings: (context) => {
|
||||
return dataStreamDetailsClient.getDataStreamSettings({
|
||||
dataStream: context.dataStream,
|
||||
|
|
|
|||
|
|
@ -6,21 +6,26 @@
|
|||
*/
|
||||
|
||||
import type { DoneInvokeEvent } from 'xstate';
|
||||
import type { DegradedFieldSortField } from '../../hooks';
|
||||
import {
|
||||
Dashboard,
|
||||
DataStreamDetails,
|
||||
DataStreamRolloverResponse,
|
||||
DataStreamSettings,
|
||||
DegradedField,
|
||||
DegradedFieldAnalysis,
|
||||
DegradedFieldResponse,
|
||||
DegradedFieldValues,
|
||||
FailedDocsDetails,
|
||||
FailedDocsError,
|
||||
FailedDocsErrorsResponse,
|
||||
NonAggregatableDatasets,
|
||||
QualityIssue,
|
||||
UpdateFieldLimitResponse,
|
||||
} from '../../../common/api_types';
|
||||
import { TableCriteria, TimeRangeConfig } from '../../../common/types';
|
||||
import { IntegrationType } from '../../../common/data_stream_details';
|
||||
import { TableCriteria, TimeRangeConfig } from '../../../common/types';
|
||||
import type { FailedDocsErrorSortField, QualityIssueSortField } from '../../hooks';
|
||||
|
||||
export type QualityIssueType = QualityIssue['type'];
|
||||
|
||||
export interface DataStream {
|
||||
name: string;
|
||||
|
|
@ -29,14 +34,24 @@ export interface DataStream {
|
|||
rawName: string;
|
||||
}
|
||||
|
||||
export interface DegradedFieldsTableConfig {
|
||||
table: TableCriteria<DegradedFieldSortField>;
|
||||
data?: DegradedField[];
|
||||
export interface QualityIssuesTableConfig {
|
||||
table: TableCriteria<QualityIssueSortField>;
|
||||
data?: QualityIssue[];
|
||||
}
|
||||
|
||||
export interface DegradedFieldsWithData {
|
||||
table: TableCriteria<DegradedFieldSortField>;
|
||||
data: DegradedField[];
|
||||
export interface QualityIssuesWithData {
|
||||
table: TableCriteria<QualityIssueSortField>;
|
||||
data: QualityIssue[];
|
||||
}
|
||||
|
||||
export interface FailedDocsErrorsTableConfig {
|
||||
table: TableCriteria<FailedDocsErrorSortField>;
|
||||
data?: FailedDocsError[];
|
||||
}
|
||||
|
||||
export interface FailedDocsErrorsWithData {
|
||||
table: TableCriteria<FailedDocsErrorSortField>;
|
||||
data: QualityIssue[];
|
||||
}
|
||||
|
||||
export interface FieldLimit {
|
||||
|
|
@ -47,14 +62,19 @@ export interface FieldLimit {
|
|||
|
||||
export interface WithDefaultControllerState {
|
||||
dataStream: string;
|
||||
degradedFields: DegradedFieldsTableConfig;
|
||||
qualityIssues: QualityIssuesTableConfig;
|
||||
failedDocsErrors: FailedDocsErrorsTableConfig;
|
||||
timeRange: TimeRangeConfig;
|
||||
showCurrentQualityIssues: boolean;
|
||||
qualityIssuesChart: QualityIssueType;
|
||||
breakdownField?: string;
|
||||
isBreakdownFieldEcs?: boolean;
|
||||
isIndexNotFoundError?: boolean;
|
||||
integration?: IntegrationType;
|
||||
expandedDegradedField?: string;
|
||||
expandedQualityIssue?: {
|
||||
name: string;
|
||||
type: QualityIssueType;
|
||||
};
|
||||
isNonAggregatable?: boolean;
|
||||
fieldLimit?: FieldLimit;
|
||||
}
|
||||
|
|
@ -71,8 +91,12 @@ export interface WithBreakdownInEcsCheck {
|
|||
isBreakdownFieldEcs: boolean;
|
||||
}
|
||||
|
||||
export interface WithDegradedFieldsData {
|
||||
degradedFields: DegradedFieldsWithData;
|
||||
export interface WithQualityIssuesData {
|
||||
qualityIssues: QualityIssuesWithData;
|
||||
}
|
||||
|
||||
export interface WithFailedDocsErrorsData {
|
||||
failedDocsErrors: FailedDocsErrorsWithData;
|
||||
}
|
||||
|
||||
export interface WithNonAggregatableDatasetStatus {
|
||||
|
|
@ -111,7 +135,12 @@ export interface WithNewFieldLimitResponse {
|
|||
|
||||
export type DefaultDatasetQualityDetailsContext = Pick<
|
||||
WithDefaultControllerState,
|
||||
'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues'
|
||||
| 'qualityIssues'
|
||||
| 'failedDocsErrors'
|
||||
| 'timeRange'
|
||||
| 'isIndexNotFoundError'
|
||||
| 'showCurrentQualityIssues'
|
||||
| 'qualityIssuesChart'
|
||||
>;
|
||||
|
||||
export type DatasetQualityDetailsControllerTypeState =
|
||||
|
|
@ -143,13 +172,18 @@ export type DatasetQualityDetailsControllerTypeState =
|
|||
}
|
||||
| {
|
||||
value:
|
||||
| 'initializing.dataStreamSettings.fetchingDataStreamDegradedFields'
|
||||
| 'initializing.dataStreamSettings.errorFetchingDegradedFields';
|
||||
| 'initializing.dataStreamSettings.doneFetchingQualityIssues'
|
||||
| 'initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.fetchingDataStreamDegradedFields'
|
||||
| 'initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.errorFetchingDegradedFields'
|
||||
| 'initializing.dataStreamSettings.qualityIssues.dataStreamFailedDocs.fetchingFailedDocs'
|
||||
| 'initializing.dataStreamSettings.qualityIssues.dataStreamFailedDocs.errorFetchingFailedDocs';
|
||||
context: WithDefaultControllerState & WithDataStreamSettings;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.dataStreamSettings.doneFetchingDegradedFields';
|
||||
context: WithDefaultControllerState & WithDataStreamSettings & WithDegradedFieldsData;
|
||||
value:
|
||||
| 'initializing.dataStreamSettings.qualityIssues.dataStreamDegradedFields.doneFetchingDegradedFields'
|
||||
| 'initializing.dataStreamSettings.qualityIssues.dataStreamFailedDocs.doneFetchingFailedDocs';
|
||||
context: WithDefaultControllerState & WithDataStreamSettings & WithQualityIssuesData;
|
||||
}
|
||||
| {
|
||||
value:
|
||||
|
|
@ -162,33 +196,34 @@ export type DatasetQualityDetailsControllerTypeState =
|
|||
context: WithDefaultControllerState & WithIntegration & WithIntegrationDashboards;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.degradedFieldFlyout.open';
|
||||
value: 'initializing.qualityIssueFlyout.open';
|
||||
context: WithDefaultControllerState;
|
||||
}
|
||||
| {
|
||||
value:
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.fetching'
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzing';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData;
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.ignoredValues.fetching'
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.analyzing'
|
||||
| 'initializing.qualityIssueFlyout.open.failedDocsFlyout.fetching';
|
||||
context: WithDefaultControllerState & WithQualityIssuesData;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.degradedFieldFlyout.open.initialized.ignoredValues.done';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues;
|
||||
value: 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.ignoredValues.done';
|
||||
context: WithDefaultControllerState & WithQualityIssuesData & WithDegradedFieldValues;
|
||||
}
|
||||
| {
|
||||
value:
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.analyzed'
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.mitigating'
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.askingForRollover'
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.rollingOver'
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.success'
|
||||
| 'initializing.degradedFieldFlyout.open.initialized.mitigation.error';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis;
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.analyzed'
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.mitigating'
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.askingForRollover'
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.rollingOver'
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.success'
|
||||
| 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.error';
|
||||
context: WithDefaultControllerState & WithQualityIssuesData & WithDegradeFieldAnalysis;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.degradedFieldFlyout.open.initialized.mitigation.success';
|
||||
value: 'initializing.qualityIssueFlyout.open.degradedFieldFlyout.mitigation.success';
|
||||
context: WithDefaultControllerState &
|
||||
WithDegradedFieldsData &
|
||||
WithQualityIssuesData &
|
||||
WithDegradedFieldValues &
|
||||
WithDegradeFieldAnalysis &
|
||||
WithNewFieldLimit &
|
||||
|
|
@ -204,8 +239,11 @@ export type DatasetQualityDetailsControllerEvent =
|
|||
timeRange: TimeRangeConfig;
|
||||
}
|
||||
| {
|
||||
type: 'OPEN_DEGRADED_FIELD_FLYOUT';
|
||||
fieldName: string | undefined;
|
||||
type: 'OPEN_QUALITY_ISSUE_FLYOUT';
|
||||
qualityIssue: {
|
||||
name: string;
|
||||
type: QualityIssueType;
|
||||
};
|
||||
}
|
||||
| {
|
||||
type: 'CLOSE_DEGRADED_FIELD_FLYOUT';
|
||||
|
|
@ -213,13 +251,21 @@ export type DatasetQualityDetailsControllerEvent =
|
|||
| {
|
||||
type: 'DEGRADED_FIELDS_LOADED';
|
||||
}
|
||||
| {
|
||||
type: 'QUALITY_ISSUES_CHART_CHANGE';
|
||||
qualityIssuesChart: QualityIssueType;
|
||||
}
|
||||
| {
|
||||
type: 'BREAKDOWN_FIELD_CHANGE';
|
||||
breakdownField: string | undefined;
|
||||
}
|
||||
| {
|
||||
type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA';
|
||||
degraded_field_criteria: TableCriteria<DegradedFieldSortField>;
|
||||
type: 'UPDATE_QUALITY_ISSUES_TABLE_CRITERIA';
|
||||
quality_issues_criteria: TableCriteria<QualityIssueSortField>;
|
||||
}
|
||||
| {
|
||||
type: 'UPDATE_FAILED_DOCS_ERRORS_TABLE_CRITERIA';
|
||||
failed_docs_errors_criteria: TableCriteria<FailedDocsErrorSortField>;
|
||||
}
|
||||
| {
|
||||
type: 'SET_NEW_FIELD_LIMIT';
|
||||
|
|
@ -232,6 +278,8 @@ export type DatasetQualityDetailsControllerEvent =
|
|||
| DoneInvokeEvent<DataStreamDetails>
|
||||
| DoneInvokeEvent<Error>
|
||||
| DoneInvokeEvent<boolean>
|
||||
| DoneInvokeEvent<FailedDocsDetails>
|
||||
| DoneInvokeEvent<FailedDocsErrorsResponse>
|
||||
| DoneInvokeEvent<DegradedFieldResponse>
|
||||
| DoneInvokeEvent<DegradedFieldValues>
|
||||
| DoneInvokeEvent<DataStreamSettings>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export function calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) {
|
||||
return totalDocs && count ? (count / totalDocs) * 100 : 0;
|
||||
}
|
||||
|
|
@ -73,18 +73,27 @@ describe('generateDatasets', () => {
|
|||
};
|
||||
|
||||
const degradedDocs = [
|
||||
{
|
||||
dataset: 'logs-system.application-default',
|
||||
count: 0,
|
||||
},
|
||||
{
|
||||
dataset: 'logs-synth-default',
|
||||
count: 6,
|
||||
},
|
||||
];
|
||||
|
||||
it('merges integrations information with dataStreamStats and degradedDocs', () => {
|
||||
const datasets = generateDatasets(dataStreamStats, degradedDocs, integrations, totalDocs);
|
||||
const failedDocs = [
|
||||
{
|
||||
dataset: 'logs-system.application-default',
|
||||
count: 2,
|
||||
},
|
||||
];
|
||||
|
||||
it('merges integrations information with dataStreamStats, degradedDocs and failedDocs', () => {
|
||||
const datasets = generateDatasets(
|
||||
dataStreamStats,
|
||||
degradedDocs,
|
||||
failedDocs,
|
||||
integrations,
|
||||
totalDocs
|
||||
);
|
||||
|
||||
expect(datasets).toEqual([
|
||||
{
|
||||
|
|
@ -101,12 +110,16 @@ describe('generateDatasets', () => {
|
|||
userPrivileges: {
|
||||
canMonitor: true,
|
||||
},
|
||||
docsInTimeRange: 100,
|
||||
quality: 'good',
|
||||
docsInTimeRange: 102,
|
||||
quality: 'degraded',
|
||||
degradedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 1.9607843137254901,
|
||||
count: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'synth',
|
||||
|
|
@ -128,6 +141,10 @@ describe('generateDatasets', () => {
|
|||
count: 6,
|
||||
percentage: 6,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -136,6 +153,7 @@ describe('generateDatasets', () => {
|
|||
const datasets = generateDatasets(
|
||||
dataStreamStats,
|
||||
degradedDocs,
|
||||
failedDocs,
|
||||
integrations,
|
||||
DEFAULT_DICTIONARY_TYPE
|
||||
);
|
||||
|
|
@ -155,12 +173,16 @@ describe('generateDatasets', () => {
|
|||
userPrivileges: {
|
||||
canMonitor: true,
|
||||
},
|
||||
docsInTimeRange: 0,
|
||||
quality: 'good',
|
||||
docsInTimeRange: 2,
|
||||
quality: 'poor',
|
||||
degradedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 100,
|
||||
count: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'synth',
|
||||
|
|
@ -182,12 +204,16 @@ describe('generateDatasets', () => {
|
|||
count: 6,
|
||||
percentage: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('merges integrations information with degradedDocs', () => {
|
||||
const datasets = generateDatasets([], degradedDocs, integrations, totalDocs);
|
||||
const datasets = generateDatasets([], degradedDocs, [], integrations, totalDocs);
|
||||
|
||||
expect(datasets).toEqual([
|
||||
{
|
||||
|
|
@ -208,6 +234,10 @@ describe('generateDatasets', () => {
|
|||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'synth',
|
||||
|
|
@ -227,12 +257,16 @@ describe('generateDatasets', () => {
|
|||
count: 6,
|
||||
percentage: 6,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('merges integrations information with degradedDocs and totalDocs', () => {
|
||||
const datasets = generateDatasets([], degradedDocs, integrations, {
|
||||
const datasets = generateDatasets([], degradedDocs, [], integrations, {
|
||||
...totalDocs,
|
||||
logs: [...totalDocs.logs, { dataset: 'logs-another-default', count: 100 }],
|
||||
});
|
||||
|
|
@ -256,6 +290,10 @@ describe('generateDatasets', () => {
|
|||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'synth',
|
||||
|
|
@ -275,6 +313,10 @@ describe('generateDatasets', () => {
|
|||
count: 6,
|
||||
percentage: 6,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'another',
|
||||
|
|
@ -294,12 +336,16 @@ describe('generateDatasets', () => {
|
|||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('merges integrations information with dataStreamStats', () => {
|
||||
const datasets = generateDatasets(dataStreamStats, [], integrations, totalDocs);
|
||||
const datasets = generateDatasets(dataStreamStats, [], [], integrations, totalDocs);
|
||||
|
||||
expect(datasets).toEqual([
|
||||
{
|
||||
|
|
@ -322,6 +368,10 @@ describe('generateDatasets', () => {
|
|||
count: 0,
|
||||
percentage: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'synth',
|
||||
|
|
@ -343,6 +393,10 @@ describe('generateDatasets', () => {
|
|||
count: 0,
|
||||
percentage: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
@ -360,7 +414,7 @@ describe('generateDatasets', () => {
|
|||
},
|
||||
};
|
||||
|
||||
const datasets = generateDatasets([nonDefaultDataset], [], integrations, totalDocs);
|
||||
const datasets = generateDatasets([nonDefaultDataset], [], [], integrations, totalDocs);
|
||||
|
||||
expect(datasets).toEqual([
|
||||
{
|
||||
|
|
@ -383,6 +437,10 @@ describe('generateDatasets', () => {
|
|||
count: 0,
|
||||
percentage: 0,
|
||||
},
|
||||
failedDocs: {
|
||||
percentage: 0,
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DEFAULT_DEGRADED_DOCS } from '../../common/constants';
|
||||
import { DEFAULT_QUALITY_DOC_STATS } from '../../common/constants';
|
||||
import { DataStreamDocsStat } from '../../common/api_types';
|
||||
import { DataStreamStatType } from '../../common/data_streams_stats/types';
|
||||
import { mapPercentageToQuality } from '../../common/utils';
|
||||
|
|
@ -13,9 +13,12 @@ import { Integration } from '../../common/data_streams_stats/integration';
|
|||
import { DataStreamStat } from '../../common/data_streams_stats/data_stream_stat';
|
||||
import { DictionaryType } from '../state_machines/dataset_quality_controller/src/types';
|
||||
import { flattenStats } from './flatten_stats';
|
||||
import { calculatePercentage } from './calculate_percentage';
|
||||
|
||||
export function generateDatasets(
|
||||
dataStreamStats: DataStreamStatType[] = [],
|
||||
degradedDocStats: DataStreamDocsStat[] = [],
|
||||
failedDocStats: DataStreamDocsStat[] = [],
|
||||
integrations: Integration[],
|
||||
totalDocsStats: DictionaryType<DataStreamDocsStat>
|
||||
): DataStreamStat[] {
|
||||
|
|
@ -51,6 +54,26 @@ export function generateDatasets(
|
|||
const totalDocsMap: Record<DataStreamDocsStat['dataset'], DataStreamDocsStat['count']> =
|
||||
Object.fromEntries(totalDocs.map(({ dataset, count }) => [dataset, count]));
|
||||
|
||||
const failedMap: Record<
|
||||
DataStreamDocsStat['dataset'],
|
||||
{
|
||||
percentage: number;
|
||||
count: DataStreamDocsStat['count'];
|
||||
}
|
||||
> = failedDocStats.reduce(
|
||||
(failedMapAcc, { dataset, count }) =>
|
||||
Object.assign(failedMapAcc, {
|
||||
[dataset]: {
|
||||
count,
|
||||
percentage: calculatePercentage({
|
||||
totalDocs: (totalDocsMap[dataset] ?? 0) + count,
|
||||
count,
|
||||
}),
|
||||
},
|
||||
}),
|
||||
{}
|
||||
);
|
||||
|
||||
const degradedMap: Record<
|
||||
DataStreamDocsStat['dataset'],
|
||||
{
|
||||
|
|
@ -62,8 +85,8 @@ export function generateDatasets(
|
|||
Object.assign(degradedMapAcc, {
|
||||
[dataset]: {
|
||||
count,
|
||||
percentage: DataStreamStat.calculatePercentage({
|
||||
totalDocs: totalDocsMap[dataset],
|
||||
percentage: calculatePercentage({
|
||||
totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0),
|
||||
count,
|
||||
}),
|
||||
},
|
||||
|
|
@ -72,19 +95,30 @@ export function generateDatasets(
|
|||
);
|
||||
|
||||
if (!dataStreamStats.length) {
|
||||
// We want to pick up all datasets even when they don't have degraded docs
|
||||
const dataStreams = [...new Set([...Object.keys(totalDocsMap), ...Object.keys(degradedMap)])];
|
||||
// We want to pick up all datasets even when they don't have degraded docs or failed docs
|
||||
const dataStreams = [
|
||||
...new Set([
|
||||
...Object.keys(totalDocsMap),
|
||||
...Object.keys(degradedMap),
|
||||
...Object.keys(failedMap),
|
||||
]),
|
||||
];
|
||||
return dataStreams.map((dataset) =>
|
||||
DataStreamStat.fromDegradedDocStat({
|
||||
degradedDocStat: { dataset, ...(degradedMap[dataset] || DEFAULT_DEGRADED_DOCS) },
|
||||
DataStreamStat.fromQualityStats({
|
||||
datasetName: dataset,
|
||||
degradedDocStat: degradedMap[dataset] || DEFAULT_QUALITY_DOC_STATS,
|
||||
failedDocStat: failedMap[dataset] || DEFAULT_QUALITY_DOC_STATS,
|
||||
datasetIntegrationMap,
|
||||
totalDocs: totalDocsMap[dataset] ?? 0,
|
||||
totalDocs: (totalDocsMap[dataset] ?? 0) + (failedMap[dataset]?.count ?? 0),
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
return dataStreamStats?.map((dataStream) => {
|
||||
const dataset = DataStreamStat.create(dataStream);
|
||||
const degradedDocs = degradedMap[dataset.rawName] || dataset.degradedDocs;
|
||||
const failedDocs = failedMap[dataset.rawName] || dataset.failedDocs;
|
||||
const qualityStats = [degradedDocs.percentage, failedDocs.percentage];
|
||||
|
||||
return {
|
||||
...dataset,
|
||||
|
|
@ -92,11 +126,11 @@ export function generateDatasets(
|
|||
integration:
|
||||
datasetIntegrationMap[dataset.name]?.integration ??
|
||||
integrationsMap[dataStream.integration ?? ''],
|
||||
degradedDocs: degradedMap[dataset.rawName] || dataset.degradedDocs,
|
||||
docsInTimeRange: totalDocsMap[dataset.rawName] ?? 0,
|
||||
quality: mapPercentageToQuality(
|
||||
(degradedMap[dataset.rawName] || dataset.degradedDocs).percentage
|
||||
),
|
||||
degradedDocs,
|
||||
failedDocs,
|
||||
docsInTimeRange:
|
||||
(totalDocsMap[dataset.rawName] ?? 0) + (failedMap[dataset.rawName]?.count ?? 0),
|
||||
quality: mapPercentageToQuality(qualityStats),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
export * from './calculate_percentage';
|
||||
export * from './filter_inactive_datasets';
|
||||
export * from './generate_datasets';
|
||||
export * from './use_kibana';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,118 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { ElasticsearchClient } from '@kbn/core/server';
|
||||
import { DataStreamDocsStat } from '../../../../common/api_types';
|
||||
import { FAILURE_STORE_SELECTOR } from '../../../../common/constants';
|
||||
import { DataStreamType } from '../../../../common/types';
|
||||
import {
|
||||
extractIndexNameFromBackingIndex,
|
||||
streamPartsToIndexPattern,
|
||||
} from '../../../../common/utils';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client';
|
||||
import { rangeQuery } from '../../../utils/queries';
|
||||
|
||||
const SIZE_LIMIT = 10000;
|
||||
|
||||
async function getPaginatedResults(options: {
|
||||
datasetQualityESClient: DatasetQualityESClient;
|
||||
index: string;
|
||||
start: number;
|
||||
end: number;
|
||||
after?: { dataset: string };
|
||||
prevResults?: Record<string, number>;
|
||||
}) {
|
||||
const { datasetQualityESClient, index, start, end, after, prevResults = {} } = options;
|
||||
|
||||
const bool = {
|
||||
filter: [...rangeQuery(start, end)],
|
||||
};
|
||||
|
||||
const response = await datasetQualityESClient.search({
|
||||
index: `${index}${FAILURE_STORE_SELECTOR}`,
|
||||
size: 0,
|
||||
query: {
|
||||
bool,
|
||||
},
|
||||
aggs: {
|
||||
datasets: {
|
||||
composite: {
|
||||
...(after ? { after } : {}),
|
||||
size: SIZE_LIMIT,
|
||||
sources: [{ dataset: { terms: { field: '_index' } } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const currResults = (response.aggregations?.datasets.buckets ?? []).reduce((acc, curr) => {
|
||||
const datasetName = extractIndexNameFromBackingIndex(curr.key.dataset as string);
|
||||
|
||||
return {
|
||||
...acc,
|
||||
[datasetName]: (acc[datasetName] ?? 0) + curr.doc_count,
|
||||
};
|
||||
}, {} as Record<string, number>);
|
||||
|
||||
const results = {
|
||||
...prevResults,
|
||||
...currResults,
|
||||
};
|
||||
|
||||
if (
|
||||
response.aggregations?.datasets.after_key &&
|
||||
response.aggregations?.datasets.buckets.length === SIZE_LIMIT
|
||||
) {
|
||||
return getPaginatedResults({
|
||||
datasetQualityESClient,
|
||||
index,
|
||||
start,
|
||||
end,
|
||||
after:
|
||||
(response.aggregations?.datasets.after_key as {
|
||||
dataset: string;
|
||||
}) || after,
|
||||
prevResults: results,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function getFailedDocsPaginated(options: {
|
||||
esClient: ElasticsearchClient;
|
||||
types: DataStreamType[];
|
||||
datasetQuery?: string;
|
||||
start: number;
|
||||
end: number;
|
||||
}): Promise<DataStreamDocsStat[]> {
|
||||
const { esClient, types, datasetQuery, start, end } = options;
|
||||
|
||||
const datasetNames = datasetQuery
|
||||
? [datasetQuery]
|
||||
: types.map((type) =>
|
||||
streamPartsToIndexPattern({
|
||||
typePattern: type,
|
||||
datasetPattern: '*-*',
|
||||
})
|
||||
);
|
||||
|
||||
const datasetQualityESClient = createDatasetQualityESClient(esClient);
|
||||
|
||||
const datasets = await getPaginatedResults({
|
||||
datasetQualityESClient,
|
||||
index: datasetNames.join(','),
|
||||
start,
|
||||
end,
|
||||
});
|
||||
|
||||
return Object.entries(datasets).map(([dataset, count]) => ({
|
||||
dataset,
|
||||
count,
|
||||
}));
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { FailedDocsDetails } from '../../../../common/api_types';
|
||||
import { FAILURE_STORE_SELECTOR } from '../../../../common/constants';
|
||||
import { TIMESTAMP } from '../../../../common/es_fields';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { getFieldIntervalInSeconds } from '../../../utils/get_interval';
|
||||
import { rangeQuery } from '../../../utils/queries';
|
||||
|
||||
export async function getFailedDocsDetails({
|
||||
esClient,
|
||||
start,
|
||||
end,
|
||||
dataStream,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
start: number;
|
||||
end: number;
|
||||
dataStream: string;
|
||||
}): Promise<FailedDocsDetails> {
|
||||
const fieldInterval = getFieldIntervalInSeconds({ start, end });
|
||||
const datasetQualityESClient = createDatasetQualityESClient(esClient);
|
||||
|
||||
const filterQuery = [...rangeQuery(start, end)];
|
||||
|
||||
const aggs = {
|
||||
lastOccurrence: {
|
||||
max: {
|
||||
field: TIMESTAMP,
|
||||
},
|
||||
},
|
||||
timeSeries: {
|
||||
date_histogram: {
|
||||
field: TIMESTAMP,
|
||||
fixed_interval: `${fieldInterval}s`,
|
||||
min_doc_count: 0,
|
||||
extended_bounds: {
|
||||
min: start,
|
||||
max: end,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const response = await datasetQualityESClient.search({
|
||||
index: `${dataStream}${FAILURE_STORE_SELECTOR}`,
|
||||
track_total_hits: true,
|
||||
size: 0,
|
||||
query: {
|
||||
bool: {
|
||||
filter: filterQuery,
|
||||
},
|
||||
},
|
||||
aggs,
|
||||
});
|
||||
|
||||
return {
|
||||
count: response.hits.total.value,
|
||||
lastOccurrence: response.aggregations?.lastOccurrence.value,
|
||||
timeSeries:
|
||||
response.aggregations?.timeSeries.buckets.map((timeSeriesBucket) => ({
|
||||
x: timeSeriesBucket.key,
|
||||
y: timeSeriesBucket.doc_count,
|
||||
})) ?? [],
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { SearchHit } from '@kbn/es-types';
|
||||
import { FAILURE_STORE_SELECTOR } from '../../../../common/constants';
|
||||
import { TIMESTAMP } from '../../../../common/es_fields';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { rangeQuery } from '../../../utils/queries';
|
||||
|
||||
export async function getFailedDocsErrors({
|
||||
esClient,
|
||||
start,
|
||||
end,
|
||||
dataStream,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
start: number;
|
||||
end: number;
|
||||
dataStream: string;
|
||||
}): Promise<{ errors: Array<{ type: string; message: string }> }> {
|
||||
const datasetQualityESClient = createDatasetQualityESClient(esClient);
|
||||
|
||||
const bool = {
|
||||
filter: [...rangeQuery(start, end)],
|
||||
};
|
||||
|
||||
const response = await datasetQualityESClient.search({
|
||||
index: `${dataStream}${FAILURE_STORE_SELECTOR}`,
|
||||
size: 10000,
|
||||
query: {
|
||||
bool,
|
||||
},
|
||||
sort: [
|
||||
{
|
||||
[TIMESTAMP]: {
|
||||
order: 'desc',
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const errors = extractAndDeduplicateValues(response.hits.hits);
|
||||
|
||||
return {
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
function extractAndDeduplicateValues(
|
||||
searchHits: SearchHit[]
|
||||
): Array<{ type: string; message: string }> {
|
||||
const values: Record<string, Set<string>> = {};
|
||||
|
||||
searchHits.forEach((hit: any) => {
|
||||
const fieldKey = hit._source?.error?.type;
|
||||
const fieldValue = hit._source?.error?.message;
|
||||
if (!values[fieldKey]) {
|
||||
// Here we will create a set if not already present
|
||||
values[fieldKey] = new Set();
|
||||
}
|
||||
// here set.add will take care of dedupe
|
||||
values[fieldKey].add(fieldValue);
|
||||
});
|
||||
|
||||
return Object.entries(values).flatMap(([key, messages]) =>
|
||||
Array.from(messages).map((message) => ({ type: key, message }))
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,131 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { notImplemented } from '@hapi/boom';
|
||||
import * as t from 'io-ts';
|
||||
import {
|
||||
DataStreamDocsStat,
|
||||
FailedDocsDetails,
|
||||
FailedDocsErrorsResponse,
|
||||
} from '../../../../common/api_types';
|
||||
import { rangeRt, typesRt } from '../../../types/default_api_types';
|
||||
import { createDatasetQualityServerRoute } from '../../create_datasets_quality_server_route';
|
||||
import { getFailedDocsPaginated } from './get_failed_docs';
|
||||
import { getFailedDocsDetails } from './get_failed_docs_details';
|
||||
import { getFailedDocsErrors } from './get_failed_docs_errors';
|
||||
|
||||
const failedDocsRoute = createDatasetQualityServerRoute({
|
||||
endpoint: 'GET /internal/dataset_quality/data_streams/failed_docs',
|
||||
params: t.type({
|
||||
query: t.intersection([
|
||||
rangeRt,
|
||||
t.type({ types: typesRt }),
|
||||
t.partial({
|
||||
datasetQuery: t.string,
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
options: {
|
||||
tags: [],
|
||||
},
|
||||
async handler(resources): Promise<{
|
||||
failedDocs: DataStreamDocsStat[];
|
||||
}> {
|
||||
const { context, params, logger, getEsCapabilities } = resources;
|
||||
const coreContext = await context.core;
|
||||
const isServerless = (await getEsCapabilities()).serverless;
|
||||
|
||||
if (isServerless) {
|
||||
throw notImplemented('Failure store is not available in serverless mode');
|
||||
}
|
||||
|
||||
const esClient = coreContext.elasticsearch.client.asCurrentUser;
|
||||
|
||||
try {
|
||||
const failedDocs = await getFailedDocsPaginated({
|
||||
esClient,
|
||||
...params.query,
|
||||
});
|
||||
|
||||
return {
|
||||
failedDocs,
|
||||
};
|
||||
} catch (e) {
|
||||
logger.error(`Failed to get failed docs: ${e}`);
|
||||
|
||||
return {
|
||||
failedDocs: [],
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const failedDocsDetailsRoute = createDatasetQualityServerRoute({
|
||||
endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs',
|
||||
params: t.type({
|
||||
path: t.type({
|
||||
dataStream: t.string,
|
||||
}),
|
||||
query: rangeRt,
|
||||
}),
|
||||
options: {
|
||||
tags: [],
|
||||
},
|
||||
async handler(resources): Promise<FailedDocsDetails> {
|
||||
const { context, params, getEsCapabilities } = resources;
|
||||
const coreContext = await context.core;
|
||||
const { dataStream } = params.path;
|
||||
const isServerless = (await getEsCapabilities()).serverless;
|
||||
|
||||
if (isServerless) {
|
||||
throw notImplemented('Failure store is not available in serverless mode');
|
||||
}
|
||||
|
||||
const esClient = coreContext.elasticsearch.client.asCurrentUser;
|
||||
|
||||
return await getFailedDocsDetails({
|
||||
esClient,
|
||||
dataStream,
|
||||
...params.query,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const failedDocsErrorsRoute = createDatasetQualityServerRoute({
|
||||
endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/failed_docs/errors',
|
||||
params: t.type({
|
||||
path: t.type({
|
||||
dataStream: t.string,
|
||||
}),
|
||||
query: rangeRt,
|
||||
}),
|
||||
options: {
|
||||
tags: [],
|
||||
},
|
||||
async handler(resources): Promise<FailedDocsErrorsResponse> {
|
||||
const { context, params, getEsCapabilities } = resources;
|
||||
const coreContext = await context.core;
|
||||
const esClient = coreContext.elasticsearch.client.asCurrentUser;
|
||||
const isServerless = (await getEsCapabilities()).serverless;
|
||||
|
||||
if (isServerless) {
|
||||
throw notImplemented('Failure store is not available in serverless mode');
|
||||
}
|
||||
|
||||
return await getFailedDocsErrors({
|
||||
esClient,
|
||||
dataStream: params.path.dataStream,
|
||||
...params.query,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const failedDocsRouteRepository = {
|
||||
...failedDocsRoute,
|
||||
...failedDocsDetailsRoute,
|
||||
...failedDocsErrorsRoute,
|
||||
};
|
||||
|
|
@ -13,6 +13,7 @@ import { _IGNORED } from '../../../../common/es_fields';
|
|||
import { datasetQualityPrivileges } from '../../../services';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { rangeQuery } from '../../../utils/queries';
|
||||
import { getFailedDocsPaginated } from '../failed_docs/get_failed_docs';
|
||||
import { getDataStreams } from '../get_data_streams';
|
||||
import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats';
|
||||
|
||||
|
|
@ -60,6 +61,18 @@ export async function getDataStreamDetails({
|
|||
end
|
||||
);
|
||||
|
||||
const failedDocs = isServerless
|
||||
? undefined
|
||||
: (
|
||||
await getFailedDocsPaginated({
|
||||
esClient: esClientAsCurrentUser,
|
||||
types: [],
|
||||
datasetQuery: dataStream,
|
||||
start,
|
||||
end,
|
||||
})
|
||||
)?.[0];
|
||||
|
||||
const avgDocSizeInBytes =
|
||||
hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
|
||||
? isServerless
|
||||
|
|
@ -71,6 +84,7 @@ export async function getDataStreamDetails({
|
|||
|
||||
return {
|
||||
...dataStreamSummaryStats,
|
||||
failedDocsCount: failedDocs?.count,
|
||||
sizeBytes,
|
||||
lastActivity: esDataStream?.lastActivity,
|
||||
userPrivileges: {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { MAX_DEGRADED_FIELDS } from '../../../../common/constants';
|
|||
import { INDEX, TIMESTAMP, _IGNORED } from '../../../../common/es_fields';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { existsQuery, rangeQuery } from '../../../utils/queries';
|
||||
import { getFieldIntervalInSeconds } from './get_interval';
|
||||
import { getFieldIntervalInSeconds } from '../../../utils/get_interval';
|
||||
|
||||
export async function getDegradedFields({
|
||||
esClient,
|
||||
|
|
|
|||
|
|
@ -7,36 +7,37 @@
|
|||
|
||||
import * as t from 'io-ts';
|
||||
import {
|
||||
CheckAndLoadIntegrationResponse,
|
||||
DataStreamDetails,
|
||||
DataStreamDocsStat,
|
||||
DataStreamRolloverResponse,
|
||||
DataStreamSettings,
|
||||
DataStreamStat,
|
||||
NonAggregatableDatasets,
|
||||
DegradedFieldResponse,
|
||||
DatasetUserPrivileges,
|
||||
DegradedFieldValues,
|
||||
DegradedFieldAnalysis,
|
||||
DataStreamDocsStat,
|
||||
DegradedFieldResponse,
|
||||
DegradedFieldValues,
|
||||
NonAggregatableDatasets,
|
||||
UpdateFieldLimitResponse,
|
||||
DataStreamRolloverResponse,
|
||||
CheckAndLoadIntegrationResponse,
|
||||
} from '../../../common/api_types';
|
||||
import { rangeRt, typeRt, typesRt } from '../../types/default_api_types';
|
||||
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
|
||||
import { datasetQualityPrivileges } from '../../services';
|
||||
import { rangeRt, typeRt, typesRt } from '../../types/default_api_types';
|
||||
import { createDatasetQualityESClient } from '../../utils';
|
||||
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
|
||||
import { checkAndLoadIntegration } from './check_and_load_integration';
|
||||
import { failedDocsRouteRepository } from './failed_docs/routes';
|
||||
import { getDataStreamDetails } from './get_data_stream_details';
|
||||
import { getDataStreams } from './get_data_streams';
|
||||
import { getDataStreamsStats } from './get_data_streams_stats';
|
||||
import { getDegradedDocsPaginated } from './get_degraded_docs';
|
||||
import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams';
|
||||
import { getDegradedFields } from './get_degraded_fields';
|
||||
import { getDegradedFieldValues } from './get_degraded_field_values';
|
||||
import { analyzeDegradedField } from './get_degraded_field_analysis';
|
||||
import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats';
|
||||
import { getDataStreamsStats } from './get_data_streams_stats';
|
||||
import { getAggregatedDatasetPaginatedResults } from './get_dataset_aggregated_paginated_results';
|
||||
import { updateFieldLimit } from './update_field_limit';
|
||||
import { createDatasetQualityESClient } from '../../utils';
|
||||
import { getDataStreamSettings } from './get_datastream_settings';
|
||||
import { checkAndLoadIntegration } from './check_and_load_integration';
|
||||
import { getDegradedDocsPaginated } from './get_degraded_docs';
|
||||
import { analyzeDegradedField } from './get_degraded_field_analysis';
|
||||
import { getDegradedFieldValues } from './get_degraded_field_values';
|
||||
import { getDegradedFields } from './get_degraded_fields';
|
||||
import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams';
|
||||
import { updateFieldLimit } from './update_field_limit';
|
||||
|
||||
const statsRoute = createDatasetQualityServerRoute({
|
||||
endpoint: 'GET /internal/dataset_quality/data_streams/stats',
|
||||
|
|
@ -457,4 +458,5 @@ export const dataStreamsRouteRepository = {
|
|||
...analyzeDegradedFieldRoute,
|
||||
...updateFieldLimitRoute,
|
||||
...rolloverDataStream,
|
||||
...failedDocsRouteRepository,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -56,7 +56,8 @@
|
|||
"@kbn/rison",
|
||||
"@kbn/task-manager-plugin",
|
||||
"@kbn/field-utils",
|
||||
"@kbn/logging"
|
||||
"@kbn/logging",
|
||||
"@kbn/ui-theme"
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*"
|
||||
|
|
|
|||
|
|
@ -66,10 +66,10 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
|||
describe('gets limit analysis for a given datastream and degraded field', () => {
|
||||
before(async () => {
|
||||
synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient();
|
||||
await synthtraceLogsEsClient.createComponentTemplate(
|
||||
customComponentTemplateName,
|
||||
logsSynthMappings(dataset)
|
||||
);
|
||||
await synthtraceLogsEsClient.createComponentTemplate({
|
||||
name: customComponentTemplateName,
|
||||
mappings: logsSynthMappings(dataset),
|
||||
});
|
||||
await esClient.indices.putIndexTemplate({
|
||||
name: dataStreamName,
|
||||
_meta: {
|
||||
|
|
|
|||
|
|
@ -387,7 +387,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
|
||||
const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
|
||||
const countColumn = table['Docs count'];
|
||||
const countColumn = table[PageObjects.datasetQuality.texts.datasetDocsCountColumn];
|
||||
const cellTexts = await countColumn.getCellTexts();
|
||||
|
||||
await countColumn.sort('ascending');
|
||||
|
|
@ -402,7 +402,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
});
|
||||
|
||||
const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
const countColumn = table['Docs count'];
|
||||
const countColumn = table[PageObjects.datasetQuality.texts.datasetDocsCountColumn];
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const currentUrl = await browser.getCurrentUrl();
|
||||
|
|
@ -438,7 +438,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
|
||||
const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
|
||||
const countColumn = table['Docs count'];
|
||||
const countColumn = table[PageObjects.datasetQuality.texts.datasetDocsCountColumn];
|
||||
const cellTexts = await countColumn.getCellTexts();
|
||||
|
||||
await synthtrace.index([
|
||||
|
|
@ -452,7 +452,8 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
await PageObjects.datasetQuality.refreshDetailsPageData();
|
||||
|
||||
const updatedTable = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
const updatedCountColumn = updatedTable['Docs count'];
|
||||
const updatedCountColumn =
|
||||
updatedTable[PageObjects.datasetQuality.texts.datasetDocsCountColumn];
|
||||
|
||||
const updatedCellTexts = await updatedCountColumn.getCellTexts();
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
|
||||
it('shows sort by dataset name and show namespace', async () => {
|
||||
const cols = await PageObjects.datasetQuality.parseDatasetTable();
|
||||
const datasetNameCol = cols['Data Set Name'];
|
||||
const datasetNameCol = cols[PageObjects.datasetQuality.texts.datasetNameColumn];
|
||||
await datasetNameCol.sort('descending');
|
||||
const datasetNameColCellTexts = await datasetNameCol.getCellTexts();
|
||||
expect(datasetNameColCellTexts).to.eql(
|
||||
|
|
@ -88,7 +88,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
|
||||
it('shows the last activity', async () => {
|
||||
const cols = await PageObjects.datasetQuality.parseDatasetTable();
|
||||
const lastActivityCol = cols['Last Activity'];
|
||||
const lastActivityCol = cols[PageObjects.datasetQuality.texts.datasetLastActivityColumn];
|
||||
const activityCells = await lastActivityCol.getCellTexts();
|
||||
const lastActivityCell = activityCells[activityCells.length - 1];
|
||||
const restActivityCells = activityCells.slice(0, -1);
|
||||
|
|
@ -106,7 +106,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
it('shows degraded docs percentage', async () => {
|
||||
const cols = await PageObjects.datasetQuality.parseDatasetTable();
|
||||
|
||||
const degradedDocsCol = cols['Degraded Docs (%)'];
|
||||
const degradedDocsCol = cols[PageObjects.datasetQuality.texts.datasetDegradedDocsColumn];
|
||||
const degradedDocsColCellTexts = await degradedDocsCol.getCellTexts();
|
||||
expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']);
|
||||
});
|
||||
|
|
@ -124,7 +124,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
|
||||
it('shows dataset from integration', async () => {
|
||||
const cols = await PageObjects.datasetQuality.parseDatasetTable();
|
||||
const datasetNameCol = cols['Data Set Name'];
|
||||
const datasetNameCol = cols[PageObjects.datasetQuality.texts.datasetNameColumn];
|
||||
|
||||
const datasetNameColCellTexts = await datasetNameCol.getCellTexts();
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
it('goes to log explorer page when opened', async () => {
|
||||
const rowIndexToOpen = 1;
|
||||
const cols = await PageObjects.datasetQuality.parseDatasetTable();
|
||||
const datasetNameCol = cols['Data Set Name'];
|
||||
const datasetNameCol = cols[PageObjects.datasetQuality.texts.datasetNameColumn];
|
||||
const actionsCol = cols.Actions;
|
||||
|
||||
const datasetName = (await datasetNameCol.getCellTexts())[rowIndexToOpen];
|
||||
|
|
@ -152,7 +152,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
it('hides inactive datasets', async () => {
|
||||
// Get number of rows with Last Activity not equal to "No activity in the selected timeframe"
|
||||
const cols = await PageObjects.datasetQuality.parseDatasetTable();
|
||||
const lastActivityCol = cols['Last Activity'];
|
||||
const lastActivityCol = cols[PageObjects.datasetQuality.texts.datasetLastActivityColumn];
|
||||
const lastActivityColCellTexts = await lastActivityCol.getCellTexts();
|
||||
const activeDatasets = lastActivityColCellTexts.filter(
|
||||
(activity: string) => activity !== PageObjects.datasetQuality.texts.noActivityText
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue