mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
# Backport This will backport the following commits from `main` to `8.x`: - [[Dataset Quality] Implement _ignored root cause identification flow (#192370)](https://github.com/elastic/kibana/pull/192370) <!--- Backport version: 9.4.3 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Achyut Jhunjhunwala","email":"achyut.jhunjhunwala@elastic.co"},"sourceCommit":{"committedDate":"2024-10-04T07:41:55Z","message":"[Dataset Quality] Implement _ignored root cause identification flow (#192370)\n\n## Summary\r\n\r\nCloses - https://github.com/elastic/kibana/issues/192471\r\nCloses - https://github.com/elastic/kibana/issues/191055\r\n\r\nThe PR adds Flyout to the Degraded Fields inside the Dataset Quality\r\nDetails page where the Root Cause of the Degraded Field is diagnosed.\r\n\r\n## Pending Items\r\n\r\n- [x] API Tests for 1 new and 2 old API modifications\r\n- [x] E2E Tests for the Flyout\r\n\r\n## How to test this\r\n\r\nNOTE (Below guide is for Stateful, you can do the same for serverless)\r\n\r\n- Checkout the PR using - `gh pr checkout 192370`\r\n\r\n1. Start the FTR server using the command below\r\n\r\n```\r\n yarn test:ftr:server --config ./x-pack/test/functional/apps/dataset_quality/config.ts\r\n ```\r\n \r\n 2. Go to the following path - `x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts`\r\n 3. Comment out the 2 `after` blocks present at Line - 54-56 and 414-416\r\n 4. Run the FTR runner using the command below\r\n \r\n ```\r\nyarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts\r\n```\r\n\r\nLet the test run and go green\r\n\r\n5. Navigate to `http://localhost:5620/app/management/data/data_quality/`\r\nusername - `test_user` and password - `changeme`\r\n\r\n6. Select the `degraded.dataset.rca` dataset\r\n\r\nYou will have an environment ready to test the flyout different\r\nscenarios\r\n\r\n## Demo\r\n\r\n## Field Limit and Ignore above isse\r\n\r\n\r\n\r\n## Warning about not current quality issue\r\n\r\n\r\n\r\n## Blocker\r\n\r\nThere is an Elasticsearch issue on Serverless, which becomes a blocker\r\nfor merging this PR\r\n\r\nhttps://github.com/elastic/elasticsearch-serverless/issues/2815","sha":"0d19367fdfad5526b5220dfdf18b4991fe6b3abd","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["v9.0.0","release_note:feature","backport:prev-minor","ci:project-deploy-observability","Team:obs-ux-logs","Feature:Dataset Health"],"title":"[Dataset Quality] Implement _ignored root cause identification flow","number":192370,"url":"https://github.com/elastic/kibana/pull/192370","mergeCommit":{"message":"[Dataset Quality] Implement _ignored root cause identification flow (#192370)\n\n## Summary\r\n\r\nCloses - https://github.com/elastic/kibana/issues/192471\r\nCloses - https://github.com/elastic/kibana/issues/191055\r\n\r\nThe PR adds Flyout to the Degraded Fields inside the Dataset Quality\r\nDetails page where the Root Cause of the Degraded Field is diagnosed.\r\n\r\n## Pending Items\r\n\r\n- [x] API Tests for 1 new and 2 old API modifications\r\n- [x] E2E Tests for the Flyout\r\n\r\n## How to test this\r\n\r\nNOTE (Below guide is for Stateful, you can do the same for serverless)\r\n\r\n- Checkout the PR using - `gh pr checkout 192370`\r\n\r\n1. Start the FTR server using the command below\r\n\r\n```\r\n yarn test:ftr:server --config ./x-pack/test/functional/apps/dataset_quality/config.ts\r\n ```\r\n \r\n 2. Go to the following path - `x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts`\r\n 3. Comment out the 2 `after` blocks present at Line - 54-56 and 414-416\r\n 4. Run the FTR runner using the command below\r\n \r\n ```\r\nyarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts\r\n```\r\n\r\nLet the test run and go green\r\n\r\n5. Navigate to `http://localhost:5620/app/management/data/data_quality/`\r\nusername - `test_user` and password - `changeme`\r\n\r\n6. Select the `degraded.dataset.rca` dataset\r\n\r\nYou will have an environment ready to test the flyout different\r\nscenarios\r\n\r\n## Demo\r\n\r\n## Field Limit and Ignore above isse\r\n\r\n\r\n\r\n## Warning about not current quality issue\r\n\r\n\r\n\r\n## Blocker\r\n\r\nThere is an Elasticsearch issue on Serverless, which becomes a blocker\r\nfor merging this PR\r\n\r\nhttps://github.com/elastic/elasticsearch-serverless/issues/2815","sha":"0d19367fdfad5526b5220dfdf18b4991fe6b3abd"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/192370","number":192370,"mergeCommit":{"message":"[Dataset Quality] Implement _ignored root cause identification flow (#192370)\n\n## Summary\r\n\r\nCloses - https://github.com/elastic/kibana/issues/192471\r\nCloses - https://github.com/elastic/kibana/issues/191055\r\n\r\nThe PR adds Flyout to the Degraded Fields inside the Dataset Quality\r\nDetails page where the Root Cause of the Degraded Field is diagnosed.\r\n\r\n## Pending Items\r\n\r\n- [x] API Tests for 1 new and 2 old API modifications\r\n- [x] E2E Tests for the Flyout\r\n\r\n## How to test this\r\n\r\nNOTE (Below guide is for Stateful, you can do the same for serverless)\r\n\r\n- Checkout the PR using - `gh pr checkout 192370`\r\n\r\n1. Start the FTR server using the command below\r\n\r\n```\r\n yarn test:ftr:server --config ./x-pack/test/functional/apps/dataset_quality/config.ts\r\n ```\r\n \r\n 2. Go to the following path - `x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts`\r\n 3. Comment out the 2 `after` blocks present at Line - 54-56 and 414-416\r\n 4. Run the FTR runner using the command below\r\n \r\n ```\r\nyarn test:ftr:runner --config ./x-pack/test/functional/apps/dataset_quality/config.ts --include ./x-pack/test/functional/apps/dataset_quality/degraded_field_flyout.ts\r\n```\r\n\r\nLet the test run and go green\r\n\r\n5. Navigate to `http://localhost:5620/app/management/data/data_quality/`\r\nusername - `test_user` and password - `changeme`\r\n\r\n6. Select the `degraded.dataset.rca` dataset\r\n\r\nYou will have an environment ready to test the flyout different\r\nscenarios\r\n\r\n## Demo\r\n\r\n## Field Limit and Ignore above isse\r\n\r\n\r\n\r\n## Warning about not current quality issue\r\n\r\n\r\n\r\n## Blocker\r\n\r\nThere is an Elasticsearch issue on Serverless, which becomes a blocker\r\nfor merging this PR\r\n\r\nhttps://github.com/elastic/elasticsearch-serverless/issues/2815","sha":"0d19367fdfad5526b5220dfdf18b4991fe6b3abd"}}]}] BACKPORT--> Co-authored-by: Achyut Jhunjhunwala <achyut.jhunjhunwala@elastic.co>
This commit is contained in:
Kibana Machine
GitHub
parent
268e07f5ba
commit
29be4a5bcc
58 changed files with 2299 additions and 464 deletions
|
|
@ -42,4 +42,5 @@ export interface DataQualityDetailsLocatorParams extends SerializableRecord {
|
|||
table?: DegradedFieldsTable;
|
||||
};
|
||||
expandedDegradedField?: string;
|
||||
showCurrentQualityIssues?: boolean;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ pageLoadAssetSize:
|
|||
dashboardEnhanced: 65646
|
||||
data: 454087
|
||||
dataQuality: 19384
|
||||
datasetQuality: 52000
|
||||
datasetQuality: 55000
|
||||
dataUsage: 30000
|
||||
dataViewEditor: 28082
|
||||
dataViewFieldEditor: 42021
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@
|
|||
"settings": {
|
||||
"index": {
|
||||
"codec": "best_compression",
|
||||
"final_pipeline": "logs@custom",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
"limit": 2000
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@
|
|||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"final_pipeline": "logs@custom",
|
||||
"codec": "best_compression",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@
|
|||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"final_pipeline": "logs@custom",
|
||||
"codec": "best_compression",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@
|
|||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"final_pipeline": "logs@custom",
|
||||
"codec": "best_compression",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@
|
|||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"final_pipeline": "logs@custom",
|
||||
"codec": "best_compression",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@
|
|||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"final_pipeline": "logs@custom",
|
||||
"codec": "best_compression",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@
|
|||
},
|
||||
"settings": {
|
||||
"index": {
|
||||
"final_pipeline": "logs@custom",
|
||||
"codec": "best_compression",
|
||||
"mapping": {
|
||||
"total_fields": {
|
||||
|
|
|
|||
|
|
@ -13,12 +13,10 @@ import { installAssets } from './lib/install_assets';
|
|||
import { indexSchedule } from './lib/index_schedule';
|
||||
import { installIndexTemplate } from './lib/install_index_template';
|
||||
import { indices } from './lib/indices';
|
||||
import { installDefaultIngestPipeline } from './lib/install_default_ingest_pipeline';
|
||||
import { installDefaultComponentTemplate } from './lib/install_default_component_template';
|
||||
|
||||
export async function run(config: Config, client: Client, logger: ToolingLog) {
|
||||
await installDefaultComponentTemplate(config, client, logger);
|
||||
await installDefaultIngestPipeline(config, client, logger);
|
||||
await installIndexTemplate(config, client, logger);
|
||||
if (config.elasticsearch.installKibanaUser) {
|
||||
await setupKibanaSystemUser(config, client, logger);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export const urlSchemaRT = rt.exact(
|
|||
breakdownField: rt.string,
|
||||
degradedFields: degradedFieldRT,
|
||||
expandedDegradedField: rt.string,
|
||||
showCurrentQualityIssues: rt.boolean,
|
||||
}),
|
||||
])
|
||||
);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export const getStateFromUrlValue = (
|
|||
degradedFields: urlValue.degradedFields,
|
||||
breakdownField: urlValue.breakdownField,
|
||||
expandedDegradedField: urlValue.expandedDegradedField,
|
||||
showCurrentQualityIssues: urlValue.showCurrentQualityIssues,
|
||||
});
|
||||
|
||||
export const getUrlValueFromState = (
|
||||
|
|
@ -30,6 +31,7 @@ export const getUrlValueFromState = (
|
|||
degradedFields: state.degradedFields,
|
||||
breakdownField: state.breakdownField,
|
||||
expandedDegradedField: state.expandedDegradedField,
|
||||
showCurrentQualityIssues: state.showCurrentQualityIssues,
|
||||
v: 1,
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ The deployment-agnostic API tests are located in [`x-pack/test/api_integration/d
|
|||
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts
|
||||
|
||||
# run tests
|
||||
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --grep=$
|
||||
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts --include ./x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/$
|
||||
```
|
||||
|
||||
#### Start server and run test (serverless)
|
||||
|
|
@ -39,7 +39,7 @@ node scripts/functional_test_runner --config x-pack/test/api_integration/deploym
|
|||
node scripts/functional_tests_server --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
|
||||
|
||||
# run tests
|
||||
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --grep=$
|
||||
node scripts/functional_test_runner --config x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts --include ./x-pack/test/api_integration/deployment_agnostic/apis/observability/dataset_quality/$
|
||||
```
|
||||
|
||||
### API integration tests
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ export const degradedFieldRt = rt.type({
|
|||
y: rt.number,
|
||||
})
|
||||
),
|
||||
indexFieldWasLastPresentIn: rt.string,
|
||||
});
|
||||
|
||||
export type DegradedField = rt.TypeOf<typeof degradedFieldRt>;
|
||||
|
|
@ -120,11 +121,34 @@ export const degradedFieldValuesRt = rt.type({
|
|||
|
||||
export type DegradedFieldValues = rt.TypeOf<typeof degradedFieldValuesRt>;
|
||||
|
||||
export const dataStreamSettingsRt = rt.partial({
|
||||
createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
|
||||
integration: rt.string,
|
||||
datasetUserPrivileges: datasetUserPrivilegesRt,
|
||||
});
|
||||
export const degradedFieldAnalysisRt = rt.intersection([
|
||||
rt.type({
|
||||
isFieldLimitIssue: rt.boolean,
|
||||
fieldCount: rt.number,
|
||||
totalFieldLimit: rt.number,
|
||||
}),
|
||||
rt.partial({
|
||||
ignoreMalformed: rt.boolean,
|
||||
nestedFieldLimit: rt.number,
|
||||
fieldMapping: rt.partial({
|
||||
type: rt.string,
|
||||
ignore_above: rt.number,
|
||||
}),
|
||||
}),
|
||||
]);
|
||||
|
||||
export type DegradedFieldAnalysis = rt.TypeOf<typeof degradedFieldAnalysisRt>;
|
||||
|
||||
export const dataStreamSettingsRt = rt.intersection([
|
||||
rt.type({
|
||||
lastBackingIndexName: rt.string,
|
||||
}),
|
||||
rt.partial({
|
||||
createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless
|
||||
integration: rt.string,
|
||||
datasetUserPrivileges: datasetUserPrivilegesRt,
|
||||
}),
|
||||
]);
|
||||
|
||||
export type DataStreamSettings = rt.TypeOf<typeof dataStreamSettingsRt>;
|
||||
|
||||
|
|
|
|||
|
|
@ -8,3 +8,9 @@
|
|||
export interface GetDataStreamIntegrationParams {
|
||||
integrationName: string;
|
||||
}
|
||||
|
||||
export interface AnalyzeDegradedFieldsParams {
|
||||
dataStream: string;
|
||||
lastBackingIndex: string;
|
||||
degradedField: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
export const _IGNORED = '_ignored';
|
||||
export const TIMESTAMP = '@timestamp';
|
||||
export const INDEX = '_index';
|
||||
|
||||
export const DATA_STREAM_DATASET = 'data_stream.dataset';
|
||||
export const DATA_STREAM_NAMESPACE = 'data_stream.namespace';
|
||||
|
|
|
|||
|
|
@ -50,6 +50,13 @@ export const openInLogsExplorerText = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const logsExplorerAriaText = i18n.translate(
|
||||
'xpack.datasetQuality.details.logsExplorerAriaText',
|
||||
{
|
||||
defaultMessage: 'Logs Explorer',
|
||||
}
|
||||
);
|
||||
|
||||
export const openInDiscoverText = i18n.translate(
|
||||
'xpack.datasetQuality.details.openInDiscoverText',
|
||||
{
|
||||
|
|
@ -57,6 +64,10 @@ export const openInDiscoverText = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const discoverAriaText = i18n.translate('xpack.datasetQuality.details.discoverAriaText', {
|
||||
defaultMessage: 'Discover',
|
||||
});
|
||||
|
||||
export const flyoutDatasetDetailsText = i18n.translate(
|
||||
'xpack.datasetQuality.flyoutDatasetDetailsText',
|
||||
{
|
||||
|
|
@ -329,6 +340,21 @@ export const overviewDegradedFieldsSectionTitle = i18n.translate(
|
|||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldToggleSwitch = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldToggleSwitch',
|
||||
{
|
||||
defaultMessage: 'Current quality issues only',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldToggleSwitchTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldToggleSwitchTooltip',
|
||||
{
|
||||
defaultMessage:
|
||||
'Enable to only show issues detected in the most recent version of the data set. Disable to show all issues detected within the configured time range.',
|
||||
}
|
||||
);
|
||||
|
||||
export const overviewDegradedFieldsSectionTitleTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedFieldsSectionTooltip',
|
||||
{
|
||||
|
|
@ -402,3 +428,75 @@ export const fieldIgnoredText = i18n.translate(
|
|||
defaultMessage: 'field ignored',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldPotentialCauseColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.potentialCause',
|
||||
{
|
||||
defaultMessage: 'Potential cause',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCurrentFieldLimitColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.currentFieldLimit',
|
||||
{
|
||||
defaultMessage: 'Field limit',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldMaximumCharacterLimitColumnName = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.maximumCharacterLimit',
|
||||
{
|
||||
defaultMessage: 'Maximum character length',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCauseFieldLimitExceeded = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.cause.fieldLimitExceeded',
|
||||
{
|
||||
defaultMessage: 'field limit exceeded',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCauseFieldLimitExceededTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.cause.fieldLimitExceededTooltip',
|
||||
{
|
||||
defaultMessage: 'The number of fields in this index has exceeded the maximum allowed limit.',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCauseFieldIgnored = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.cause.fieldIgnored',
|
||||
{
|
||||
defaultMessage: 'field character limit exceeded',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCauseFieldIgnoredTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.cause.fieldIgnoredTooltip',
|
||||
{
|
||||
defaultMessage:
|
||||
'One or more values for this field exceeded the maximum allowed character length. Characters above will be ignored.',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCauseFieldMalformed = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.cause.fieldMalformed',
|
||||
{
|
||||
defaultMessage: 'field malformed',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldCauseFieldMalformedTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.cause.fieldMalformedTooltip',
|
||||
{
|
||||
defaultMessage: 'Data type for the field not set correctly.',
|
||||
}
|
||||
);
|
||||
|
||||
export const degradedFieldMessageIssueDoesNotExistInLatestIndex = i18n.translate(
|
||||
'xpack.datasetQuality.details.degradedField.message.issueDoesNotExistInLatestIndex',
|
||||
{
|
||||
defaultMessage:
|
||||
'This issue was detected in an older version of the dataset, but not in the most recent version.',
|
||||
}
|
||||
);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ const DegradedFieldFlyout = dynamic(() => import('./degraded_field_flyout'));
|
|||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DatasetQualityDetails() {
|
||||
const { isIndexNotFoundError, dataStream, expandedDegradedField } =
|
||||
const { isIndexNotFoundError, dataStream, isDegradedFieldFlyoutOpen } =
|
||||
useDatasetQualityDetailsState();
|
||||
const { startTracking } = useDatasetDetailsTelemetry();
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ export default function DatasetQualityDetails() {
|
|||
<Details />
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
{expandedDegradedField && <DegradedFieldFlyout />}
|
||||
{isDegradedFieldFlyoutOpen && <DegradedFieldFlyout />}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,16 +5,16 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useMemo } from 'react';
|
||||
import React from 'react';
|
||||
import {
|
||||
EuiBadge,
|
||||
EuiBadgeGroup,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiHorizontalRule,
|
||||
EuiSkeletonRectangle,
|
||||
EuiTextColor,
|
||||
EuiTitle,
|
||||
EuiToolTip,
|
||||
formatNumber,
|
||||
} from '@elastic/eui';
|
||||
import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
|
||||
|
|
@ -22,41 +22,42 @@ import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types';
|
|||
import { NUMBER_FORMAT } from '../../../../common/constants';
|
||||
import {
|
||||
countColumnName,
|
||||
degradedFieldCurrentFieldLimitColumnName,
|
||||
degradedFieldMaximumCharacterLimitColumnName,
|
||||
degradedFieldPotentialCauseColumnName,
|
||||
degradedFieldValuesColumnName,
|
||||
lastOccurrenceColumnName,
|
||||
} from '../../../../common/translations';
|
||||
import { useDegradedFields } from '../../../hooks';
|
||||
import { SparkPlot } from '../../common/spark_plot';
|
||||
import { DegradedField } from '../../../../common/api_types';
|
||||
|
||||
export const DegradedFieldInfo = () => {
|
||||
export const DegradedFieldInfo = ({ fieldList }: { fieldList?: DegradedField }) => {
|
||||
const {
|
||||
renderedItems,
|
||||
fieldFormats,
|
||||
expandedDegradedField,
|
||||
degradedFieldValues,
|
||||
isDegradedFieldsLoading,
|
||||
isDegradedFieldsValueLoading,
|
||||
isAnalysisInProgress,
|
||||
degradedFieldAnalysisResult,
|
||||
degradedFieldAnalysis,
|
||||
} = useDegradedFields();
|
||||
|
||||
const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [
|
||||
ES_FIELD_TYPES.DATE,
|
||||
]);
|
||||
|
||||
const fieldList = useMemo(() => {
|
||||
return renderedItems.find((item) => {
|
||||
return item.name === expandedDegradedField;
|
||||
});
|
||||
}, [renderedItems, expandedDegradedField]);
|
||||
|
||||
return (
|
||||
<EuiFlexGroup direction="column" gutterSize="none">
|
||||
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount`}>
|
||||
<EuiFlexItem>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{countColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-docCount">
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-docCount"
|
||||
grow={2}
|
||||
>
|
||||
<SparkPlot
|
||||
series={fieldList?.timeSeries}
|
||||
valueLabel={formatNumber(fieldList?.count, NUMBER_FORMAT)}
|
||||
|
|
@ -68,41 +69,108 @@ export const DegradedFieldInfo = () => {
|
|||
<EuiFlexGroup
|
||||
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-lastOccurrence`}
|
||||
>
|
||||
<EuiFlexItem>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{lastOccurrenceColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-lastOccurrence">
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-lastOccurrence"
|
||||
grow={2}
|
||||
>
|
||||
<span>{dateFormatter.convert(fieldList?.lastOccurrence)}</span>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-values`}>
|
||||
<EuiFlexItem>
|
||||
|
||||
<EuiFlexGroup data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-cause`}>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{degradedFieldValuesColumnName}</span>
|
||||
<span>{degradedFieldPotentialCauseColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-values"
|
||||
grow={false}
|
||||
css={{ maxWidth: '49%' }}
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause"
|
||||
grow={2}
|
||||
>
|
||||
<EuiSkeletonRectangle isLoading={isDegradedFieldsValueLoading} width="300px">
|
||||
<EuiBadgeGroup gutterSize="s">
|
||||
{degradedFieldValues?.values.map((value) => (
|
||||
<EuiBadge color="hollow">
|
||||
<EuiTextColor color="#765B96">
|
||||
<strong>{value}</strong>
|
||||
</EuiTextColor>
|
||||
</EuiBadge>
|
||||
))}
|
||||
</EuiBadgeGroup>
|
||||
</EuiSkeletonRectangle>
|
||||
<div>
|
||||
<EuiToolTip position="top" content={degradedFieldAnalysisResult?.tooltipContent}>
|
||||
<EuiBadge color="hollow">
|
||||
<strong>{degradedFieldAnalysisResult?.potentialCause}</strong>
|
||||
</EuiBadge>
|
||||
</EuiToolTip>
|
||||
</div>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
|
||||
{!isAnalysisInProgress && degradedFieldAnalysis?.isFieldLimitIssue && (
|
||||
<>
|
||||
<EuiFlexGroup
|
||||
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-mappingLimit`}
|
||||
>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{degradedFieldCurrentFieldLimitColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-mappingLimit"
|
||||
grow={2}
|
||||
>
|
||||
<span>{degradedFieldAnalysis.totalFieldLimit}</span>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
</>
|
||||
)}
|
||||
|
||||
{!isAnalysisInProgress && degradedFieldAnalysisResult?.shouldDisplayValues && (
|
||||
<>
|
||||
<EuiFlexGroup
|
||||
data-test-subj={'datasetQualityDetailsDegradedFieldFlyoutFieldsList-characterLimit'}
|
||||
>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{degradedFieldMaximumCharacterLimitColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-characterLimit"
|
||||
css={{ maxWidth: '64%' }}
|
||||
grow={2}
|
||||
>
|
||||
<span>{degradedFieldAnalysis?.fieldMapping?.ignore_above}</span>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
<EuiFlexGroup
|
||||
data-test-subj={`datasetQualityDetailsDegradedFieldFlyoutFieldsList-values`}
|
||||
>
|
||||
<EuiFlexItem grow={1}>
|
||||
<EuiTitle size="xxs">
|
||||
<span>{degradedFieldValuesColumnName}</span>
|
||||
</EuiTitle>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutFieldValue-values"
|
||||
css={{ maxWidth: '64%' }}
|
||||
grow={2}
|
||||
>
|
||||
<EuiBadgeGroup gutterSize="s">
|
||||
{degradedFieldValues?.values.map((value, idx) => (
|
||||
<EuiBadge color="hollow" key={idx}>
|
||||
<EuiTextColor color="#765B96">
|
||||
<strong>{value}</strong>
|
||||
</EuiTextColor>
|
||||
</EuiBadge>
|
||||
))}
|
||||
</EuiBadgeGroup>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
<EuiHorizontalRule margin="s" />
|
||||
</>
|
||||
)}
|
||||
</EuiFlexGroup>
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import React from 'react';
|
||||
import React, { useMemo } from 'react';
|
||||
import {
|
||||
EuiBadge,
|
||||
EuiFlyout,
|
||||
|
|
@ -15,22 +15,60 @@ import {
|
|||
EuiText,
|
||||
EuiTitle,
|
||||
useGeneratedHtmlId,
|
||||
EuiTextColor,
|
||||
EuiFlexGroup,
|
||||
EuiButtonIcon,
|
||||
EuiToolTip,
|
||||
} from '@elastic/eui';
|
||||
import { useDegradedFields } from '../../../hooks';
|
||||
import { NavigationSource } from '../../../services/telemetry';
|
||||
import {
|
||||
useDatasetDetailsRedirectLinkTelemetry,
|
||||
useDatasetQualityDetailsState,
|
||||
useDegradedFields,
|
||||
useRedirectLink,
|
||||
} from '../../../hooks';
|
||||
import {
|
||||
degradedFieldMessageIssueDoesNotExistInLatestIndex,
|
||||
discoverAriaText,
|
||||
fieldIgnoredText,
|
||||
logsExplorerAriaText,
|
||||
openInDiscoverText,
|
||||
openInLogsExplorerText,
|
||||
overviewDegradedFieldsSectionTitle,
|
||||
} from '../../../../common/translations';
|
||||
import { DegradedFieldInfo } from './field_info';
|
||||
import { _IGNORED } from '../../../../common/es_fields';
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function DegradedFieldFlyout() {
|
||||
const { closeDegradedFieldFlyout, expandedDegradedField } = useDegradedFields();
|
||||
const { closeDegradedFieldFlyout, expandedDegradedField, renderedItems } = useDegradedFields();
|
||||
const { dataStreamSettings, datasetDetails, timeRange } = useDatasetQualityDetailsState();
|
||||
const pushedFlyoutTitleId = useGeneratedHtmlId({
|
||||
prefix: 'pushedFlyoutTitle',
|
||||
});
|
||||
|
||||
const fieldList = useMemo(() => {
|
||||
return renderedItems.find((item) => {
|
||||
return item.name === expandedDegradedField;
|
||||
});
|
||||
}, [renderedItems, expandedDegradedField]);
|
||||
|
||||
const isUserViewingTheIssueOnLatestBackingIndex =
|
||||
dataStreamSettings?.lastBackingIndexName === fieldList?.indexFieldWasLastPresentIn;
|
||||
|
||||
const { sendTelemetry } = useDatasetDetailsRedirectLinkTelemetry({
|
||||
query: { language: 'kuery', query: `${_IGNORED}: ${expandedDegradedField}` },
|
||||
navigationSource: NavigationSource.DegradedFieldFlyoutHeader,
|
||||
});
|
||||
|
||||
const redirectLinkProps = useRedirectLink({
|
||||
dataStreamStat: datasetDetails,
|
||||
timeRangeConfig: timeRange,
|
||||
query: { language: 'kuery', query: `${_IGNORED}: ${expandedDegradedField}` },
|
||||
sendTelemetry,
|
||||
});
|
||||
|
||||
return (
|
||||
<EuiFlyout
|
||||
type="push"
|
||||
|
|
@ -42,14 +80,47 @@ export default function DegradedFieldFlyout() {
|
|||
<EuiFlyoutHeader hasBorder>
|
||||
<EuiBadge color="warning">{overviewDegradedFieldsSectionTitle}</EuiBadge>
|
||||
<EuiSpacer size="s" />
|
||||
<EuiTitle size="m">
|
||||
<EuiText>
|
||||
{expandedDegradedField} <span style={{ fontWeight: 400 }}>{fieldIgnoredText}</span>
|
||||
</EuiText>
|
||||
</EuiTitle>
|
||||
<EuiFlexGroup justifyContent="spaceBetween" gutterSize="s">
|
||||
<EuiTitle size="m">
|
||||
<EuiText>
|
||||
{expandedDegradedField} <span style={{ fontWeight: 400 }}>{fieldIgnoredText}</span>
|
||||
</EuiText>
|
||||
</EuiTitle>
|
||||
<EuiToolTip
|
||||
content={
|
||||
redirectLinkProps.isLogsExplorerAvailable
|
||||
? openInLogsExplorerText
|
||||
: openInDiscoverText
|
||||
}
|
||||
>
|
||||
<EuiButtonIcon
|
||||
display="base"
|
||||
iconType={
|
||||
redirectLinkProps.isLogsExplorerAvailable ? 'logoObservability' : 'discoverApp'
|
||||
}
|
||||
aria-label={
|
||||
redirectLinkProps.isLogsExplorerAvailable ? logsExplorerAriaText : discoverAriaText
|
||||
}
|
||||
size="s"
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutTitleLinkToDiscover"
|
||||
{...redirectLinkProps.linkProps}
|
||||
/>
|
||||
</EuiToolTip>
|
||||
</EuiFlexGroup>
|
||||
{!isUserViewingTheIssueOnLatestBackingIndex && (
|
||||
<>
|
||||
<EuiSpacer size="s" />
|
||||
<EuiTextColor
|
||||
color="danger"
|
||||
data-test-subj="datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist"
|
||||
>
|
||||
{degradedFieldMessageIssueDoesNotExistInLatestIndex}
|
||||
</EuiTextColor>
|
||||
</>
|
||||
)}
|
||||
</EuiFlyoutHeader>
|
||||
<EuiFlyoutBody>
|
||||
<DegradedFieldInfo />
|
||||
<DegradedFieldInfo fieldList={fieldList} />
|
||||
</EuiFlyoutBody>
|
||||
</EuiFlyout>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -15,10 +15,13 @@ import {
|
|||
useGeneratedHtmlId,
|
||||
EuiBadge,
|
||||
EuiBetaBadge,
|
||||
EuiSwitch,
|
||||
} from '@elastic/eui';
|
||||
import {
|
||||
overviewDegradedFieldsSectionTitle,
|
||||
overviewDegradedFieldsSectionTitleTooltip,
|
||||
overviewDegradedFieldToggleSwitch,
|
||||
overviewDegradedFieldToggleSwitchTooltip,
|
||||
overviewQualityIssuesAccordionTechPreviewBadge,
|
||||
} from '../../../../../common/translations';
|
||||
import { DegradedFieldTable } from './table';
|
||||
|
|
@ -28,8 +31,24 @@ export function DegradedFields() {
|
|||
const accordionId = useGeneratedHtmlId({
|
||||
prefix: overviewDegradedFieldsSectionTitle,
|
||||
});
|
||||
const toggleTextSwitchId = useGeneratedHtmlId({ prefix: 'toggleTextSwitch' });
|
||||
|
||||
const { totalItemCount } = useDegradedFields();
|
||||
const { totalItemCount, toggleCurrentQualityIssues, showCurrentQualityIssues } =
|
||||
useDegradedFields();
|
||||
|
||||
const latestBackingIndexToggle = (
|
||||
<>
|
||||
<EuiSwitch
|
||||
label={overviewDegradedFieldToggleSwitch}
|
||||
checked={showCurrentQualityIssues}
|
||||
onChange={toggleCurrentQualityIssues}
|
||||
aria-describedby={toggleTextSwitchId}
|
||||
compressed
|
||||
data-test-subj="datasetQualityDetailsOverviewDegradedFieldToggleSwitch"
|
||||
/>
|
||||
<EuiIconTip content={overviewDegradedFieldToggleSwitchTooltip} position="top" />
|
||||
</>
|
||||
);
|
||||
|
||||
const accordionTitle = (
|
||||
<EuiFlexGroup alignItems="center" gutterSize="s" direction="row">
|
||||
|
|
@ -58,6 +77,7 @@ export function DegradedFields() {
|
|||
buttonContent={accordionTitle}
|
||||
paddingSize="none"
|
||||
initialIsOpen={true}
|
||||
extraAction={latestBackingIndexToggle}
|
||||
data-test-subj="datasetQualityDetailsOverviewDocumentTrends"
|
||||
>
|
||||
<DegradedFieldTable />
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
*/
|
||||
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import {
|
||||
EuiAccordion,
|
||||
|
|
@ -27,6 +26,9 @@ import type { DataViewField } from '@kbn/data-views-plugin/common';
|
|||
import { css } from '@emotion/react';
|
||||
import { UnifiedBreakdownFieldSelector } from '@kbn/unified-histogram-plugin/public';
|
||||
import {
|
||||
discoverAriaText,
|
||||
logsExplorerAriaText,
|
||||
openInDiscoverText,
|
||||
openInLogsExplorerText,
|
||||
overviewDegradedDocsText,
|
||||
} from '../../../../../../common/translations';
|
||||
|
|
@ -130,14 +132,25 @@ export default function DegradedDocs({ lastReloadTime }: { lastReloadTime: numbe
|
|||
onBreakdownFieldChange={breakdown.onChange}
|
||||
/>
|
||||
</EuiSkeletonRectangle>
|
||||
<EuiToolTip content={openInLogsExplorerText}>
|
||||
<EuiToolTip
|
||||
content={
|
||||
degradedDocLinkLogsExplorer.isLogsExplorerAvailable
|
||||
? openInLogsExplorerText
|
||||
: openInDiscoverText
|
||||
}
|
||||
>
|
||||
<EuiButtonIcon
|
||||
display="base"
|
||||
iconType="discoverApp"
|
||||
aria-label={i18n.translate(
|
||||
'xpack.datasetQuality.degradedDocs.euiButtonIcon.discoverLabel',
|
||||
{ defaultMessage: 'Discover' }
|
||||
)}
|
||||
iconType={
|
||||
degradedDocLinkLogsExplorer.isLogsExplorerAvailable
|
||||
? 'logoObservability'
|
||||
: 'discoverApp'
|
||||
}
|
||||
aria-label={
|
||||
degradedDocLinkLogsExplorer.isLogsExplorerAvailable
|
||||
? logsExplorerAriaText
|
||||
: discoverAriaText
|
||||
}
|
||||
size="s"
|
||||
data-test-subj="datasetQualityDetailsLinkToDiscover"
|
||||
{...degradedDocLinkLogsExplorer.linkProps}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ export const getPublicStateFromContext = (
|
|||
breakdownField: context.breakdownField,
|
||||
integration: context.integration,
|
||||
expandedDegradedField: context.expandedDegradedField,
|
||||
showCurrentQualityIssues: context.showCurrentQualityIssues,
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -51,4 +52,6 @@ export const getContextFromPublicState = (
|
|||
},
|
||||
dataStream: publicState.dataStream,
|
||||
expandedDegradedField: publicState.expandedDegradedField,
|
||||
showCurrentQualityIssues:
|
||||
publicState.showCurrentQualityIssues ?? DEFAULT_CONTEXT.showCurrentQualityIssues,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -28,7 +28,10 @@ export type DatasetQualityDetailsPublicState = WithDefaultControllerState;
|
|||
// a must and everything else can be optional. The table inside the
|
||||
// degradedFields must accept field property as string
|
||||
export type DatasetQualityDetailsPublicStateUpdate = Partial<
|
||||
Pick<WithDefaultControllerState, 'timeRange' | 'breakdownField' | 'expandedDegradedField'>
|
||||
Pick<
|
||||
WithDefaultControllerState,
|
||||
'timeRange' | 'breakdownField' | 'expandedDegradedField' | 'showCurrentQualityIssues'
|
||||
>
|
||||
> & {
|
||||
dataStream: string;
|
||||
} & {
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ export const useDatasetQualityDetailsState = () => {
|
|||
);
|
||||
|
||||
const dataStreamSettings = useSelector(service, (state) =>
|
||||
state.matches('initializing.dataStreamSettings.initializeIntegrations')
|
||||
state.matches('initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields')
|
||||
? state.context.dataStreamSettings
|
||||
: undefined
|
||||
);
|
||||
|
|
@ -59,14 +59,14 @@ export const useDatasetQualityDetailsState = () => {
|
|||
const integrationDetails = {
|
||||
integration: useSelector(service, (state) =>
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.done'
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done'
|
||||
)
|
||||
? state.context.integration
|
||||
: undefined
|
||||
),
|
||||
dashboard: useSelector(service, (state) =>
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.done'
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done'
|
||||
)
|
||||
? state.context.integrationDashboards
|
||||
: undefined
|
||||
|
|
@ -77,7 +77,7 @@ export const useDatasetQualityDetailsState = () => {
|
|||
service,
|
||||
(state) =>
|
||||
!state.matches(
|
||||
'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.unauthorized'
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized'
|
||||
)
|
||||
);
|
||||
|
||||
|
|
@ -103,18 +103,24 @@ export const useDatasetQualityDetailsState = () => {
|
|||
const loadingState = useSelector(service, (state) => ({
|
||||
nonAggregatableDatasetLoading: state.matches('initializing.nonAggregatableDataset.fetching'),
|
||||
dataStreamDetailsLoading: state.matches('initializing.dataStreamDetails.fetching'),
|
||||
dataStreamSettingsLoading: state.matches('initializing.dataStreamSettings.fetching'),
|
||||
dataStreamSettingsLoading: state.matches(
|
||||
'initializing.dataStreamSettings.fetchingDataStreamSettings'
|
||||
),
|
||||
integrationDetailsLoadings: state.matches(
|
||||
'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.fetching'
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching'
|
||||
),
|
||||
integrationDetailsLoaded: state.matches(
|
||||
'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.done'
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done'
|
||||
),
|
||||
integrationDashboardsLoading: state.matches(
|
||||
'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.fetching'
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching'
|
||||
),
|
||||
}));
|
||||
|
||||
const isDegradedFieldFlyoutOpen = useSelector(service, (state) =>
|
||||
state.matches('initializing.degradedFieldFlyout.open')
|
||||
);
|
||||
|
||||
const updateTimeRange = useCallback(
|
||||
({ start, end, refreshInterval }: OnRefreshProps) => {
|
||||
service.send({
|
||||
|
|
@ -150,5 +156,6 @@ export const useDatasetQualityDetailsState = () => {
|
|||
canUserAccessDashboards,
|
||||
canUserViewIntegrations,
|
||||
expandedDegradedField,
|
||||
isDegradedFieldFlyoutOpen,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -15,6 +15,14 @@ import {
|
|||
} from '../../common/constants';
|
||||
import { useKibanaContextForPlugin } from '../utils';
|
||||
import { useDatasetQualityDetailsState } from './use_dataset_quality_details_state';
|
||||
import {
|
||||
degradedFieldCauseFieldIgnored,
|
||||
degradedFieldCauseFieldIgnoredTooltip,
|
||||
degradedFieldCauseFieldLimitExceeded,
|
||||
degradedFieldCauseFieldLimitExceededTooltip,
|
||||
degradedFieldCauseFieldMalformed,
|
||||
degradedFieldCauseFieldMalformedTooltip,
|
||||
} from '../../common/translations';
|
||||
|
||||
export type DegradedFieldSortField = keyof DegradedField;
|
||||
|
||||
|
|
@ -24,7 +32,10 @@ export function useDegradedFields() {
|
|||
services: { fieldFormats },
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const { degradedFields, expandedDegradedField } = useSelector(service, (state) => state.context);
|
||||
const { degradedFields, expandedDegradedField, showCurrentQualityIssues } = useSelector(
|
||||
service,
|
||||
(state) => state.context
|
||||
);
|
||||
const { data, table } = degradedFields ?? {};
|
||||
const { page, rowsPerPage, sort } = table;
|
||||
|
||||
|
|
@ -62,8 +73,14 @@ export function useDegradedFields() {
|
|||
return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage);
|
||||
}, [data, sort.field, sort.direction, page, rowsPerPage]);
|
||||
|
||||
const expandedRenderedItem = useMemo(() => {
|
||||
return renderedItems.find((item) => item.name === expandedDegradedField);
|
||||
}, [expandedDegradedField, renderedItems]);
|
||||
|
||||
const isDegradedFieldsLoading = useSelector(service, (state) =>
|
||||
state.matches('initializing.dataStreamDegradedFields.fetching')
|
||||
state.matches(
|
||||
'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching'
|
||||
)
|
||||
);
|
||||
|
||||
const closeDegradedFieldFlyout = useCallback(
|
||||
|
|
@ -82,14 +99,71 @@ export function useDegradedFields() {
|
|||
[expandedDegradedField, service]
|
||||
);
|
||||
|
||||
const toggleCurrentQualityIssues = useCallback(() => {
|
||||
service.send('TOGGLE_CURRENT_QUALITY_ISSUES');
|
||||
}, [service]);
|
||||
|
||||
const degradedFieldValues = useSelector(service, (state) =>
|
||||
state.matches('initializing.initializeFixItFlow.ignoredValues.done')
|
||||
state.matches('initializing.degradedFieldFlyout.open.ignoredValues.done')
|
||||
? state.context.degradedFieldValues
|
||||
: undefined
|
||||
);
|
||||
|
||||
const degradedFieldAnalysis = useSelector(service, (state) =>
|
||||
state.matches('initializing.degradedFieldFlyout.open.analyze.done')
|
||||
? state.context.degradedFieldAnalysis
|
||||
: undefined
|
||||
);
|
||||
|
||||
// This piece only cater field limit issue at the moment.
|
||||
// In future this will cater the other 2 reasons as well
|
||||
const degradedFieldAnalysisResult = useMemo(() => {
|
||||
if (!degradedFieldAnalysis) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// 1st check if it's a field limit issue
|
||||
if (degradedFieldAnalysis.isFieldLimitIssue) {
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldLimitExceeded,
|
||||
tooltipContent: degradedFieldCauseFieldLimitExceededTooltip,
|
||||
shouldDisplayMitigation: true,
|
||||
shouldDisplayValues: false,
|
||||
};
|
||||
}
|
||||
|
||||
// 2nd check if it's a ignored above issue
|
||||
const fieldMapping = degradedFieldAnalysis.fieldMapping;
|
||||
|
||||
if (fieldMapping && fieldMapping?.type === 'keyword' && fieldMapping?.ignore_above) {
|
||||
const isAnyValueExceedingIgnoreAbove = degradedFieldValues?.values.some(
|
||||
(value) => value.length > fieldMapping.ignore_above!
|
||||
);
|
||||
if (isAnyValueExceedingIgnoreAbove) {
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldIgnored,
|
||||
tooltipContent: degradedFieldCauseFieldIgnoredTooltip,
|
||||
shouldDisplayMitigation: false,
|
||||
shouldDisplayValues: true,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 3rd check if its a ignore_malformed issue. There is no check, at the moment.
|
||||
return {
|
||||
potentialCause: degradedFieldCauseFieldMalformed,
|
||||
tooltipContent: degradedFieldCauseFieldMalformedTooltip,
|
||||
shouldDisplayMitigation: false,
|
||||
shouldDisplayValues: false,
|
||||
};
|
||||
}, [degradedFieldAnalysis, degradedFieldValues]);
|
||||
|
||||
const isDegradedFieldsValueLoading = useSelector(service, (state) => {
|
||||
return !state.matches('initializing.initializeFixItFlow.ignoredValues.done');
|
||||
return state.matches('initializing.degradedFieldFlyout.open.ignoredValues.fetching');
|
||||
});
|
||||
|
||||
const isAnalysisInProgress = useSelector(service, (state) => {
|
||||
return state.matches('initializing.degradedFieldFlyout.open.analyze.fetching');
|
||||
});
|
||||
|
||||
return {
|
||||
|
|
@ -105,5 +179,11 @@ export function useDegradedFields() {
|
|||
closeDegradedFieldFlyout,
|
||||
degradedFieldValues,
|
||||
isDegradedFieldsValueLoading,
|
||||
isAnalysisInProgress,
|
||||
degradedFieldAnalysis,
|
||||
degradedFieldAnalysisResult,
|
||||
toggleCurrentQualityIssues,
|
||||
showCurrentQualityIssues,
|
||||
expandedRenderedItem,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@
|
|||
import { HttpStart } from '@kbn/core/public';
|
||||
import { decodeOrThrow } from '@kbn/io-ts-utils';
|
||||
import {
|
||||
DegradedFieldAnalysis,
|
||||
degradedFieldAnalysisRt,
|
||||
DegradedFieldValues,
|
||||
degradedFieldValuesRt,
|
||||
getDataStreamDegradedFieldsResponseRt,
|
||||
|
|
@ -32,7 +34,10 @@ import {
|
|||
} from '../../../common/data_streams_stats';
|
||||
import { IDataStreamDetailsClient } from './types';
|
||||
import { Integration } from '../../../common/data_streams_stats/integration';
|
||||
import { GetDataStreamIntegrationParams } from '../../../common/data_stream_details/types';
|
||||
import {
|
||||
AnalyzeDegradedFieldsParams,
|
||||
GetDataStreamIntegrationParams,
|
||||
} from '../../../common/data_stream_details/types';
|
||||
import { DatasetQualityError } from '../../../common/errors';
|
||||
|
||||
export class DataStreamDetailsClient implements IDataStreamDetailsClient {
|
||||
|
|
@ -167,4 +172,28 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient {
|
|||
|
||||
if (integration) return Integration.create(integration);
|
||||
}
|
||||
|
||||
public async analyzeDegradedField({
|
||||
dataStream,
|
||||
degradedField,
|
||||
lastBackingIndex,
|
||||
}: AnalyzeDegradedFieldsParams): Promise<DegradedFieldAnalysis> {
|
||||
const response = await this.http
|
||||
.get<DegradedFieldAnalysis>(
|
||||
`/internal/dataset_quality/data_streams/${dataStream}/degraded_field/${degradedField}/analyze`,
|
||||
{ query: { lastBackingIndex } }
|
||||
)
|
||||
.catch((error) => {
|
||||
throw new DatasetQualityError(
|
||||
`Failed to analyze degraded field: ${degradedField} for datastream: ${dataStream}`,
|
||||
error
|
||||
);
|
||||
});
|
||||
|
||||
return decodeOrThrow(
|
||||
degradedFieldAnalysisRt,
|
||||
(message: string) =>
|
||||
new DatasetQualityError(`Failed to decode the analysis response: ${message}`)
|
||||
)(response);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,8 +17,11 @@ import {
|
|||
DegradedFieldResponse,
|
||||
GetDataStreamDegradedFieldValuesPathParams,
|
||||
} from '../../../common/data_streams_stats';
|
||||
import { GetDataStreamIntegrationParams } from '../../../common/data_stream_details/types';
|
||||
import { Dashboard, DegradedFieldValues } from '../../../common/api_types';
|
||||
import {
|
||||
AnalyzeDegradedFieldsParams,
|
||||
GetDataStreamIntegrationParams,
|
||||
} from '../../../common/data_stream_details/types';
|
||||
import { Dashboard, DegradedFieldAnalysis, DegradedFieldValues } from '../../../common/api_types';
|
||||
|
||||
export type DataStreamDetailsServiceSetup = void;
|
||||
|
||||
|
|
@ -43,4 +46,5 @@ export interface IDataStreamDetailsClient {
|
|||
getDataStreamIntegration(
|
||||
params: GetDataStreamIntegrationParams
|
||||
): Promise<Integration | undefined>;
|
||||
analyzeDegradedField(params: AnalyzeDegradedFieldsParams): Promise<DegradedFieldAnalysis>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ export enum NavigationSource {
|
|||
Trend = 'trend',
|
||||
Table = 'table',
|
||||
ActionMenu = 'action_menu',
|
||||
DegradedFieldFlyoutHeader = 'degraded_field_flyout_header',
|
||||
}
|
||||
|
||||
export interface WithTrackingId {
|
||||
|
|
|
|||
|
|
@ -29,4 +29,5 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityDetailsContext = {
|
|||
...DEFAULT_TIME_RANGE,
|
||||
refresh: DEFAULT_DATEPICKER_REFRESH,
|
||||
},
|
||||
showCurrentQualityIssues: false,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate';
|
||||
import { assign, createMachine, DoneInvokeEvent, InterpreterFrom, raise } from 'xstate';
|
||||
import { getDateISORange } from '@kbn/timerange';
|
||||
import type { IToasts } from '@kbn/core-notifications-browser';
|
||||
import {
|
||||
|
|
@ -21,6 +21,7 @@ import {
|
|||
Dashboard,
|
||||
DataStreamDetails,
|
||||
DataStreamSettings,
|
||||
DegradedFieldAnalysis,
|
||||
DegradedFieldResponse,
|
||||
DegradedFieldValues,
|
||||
NonAggregatableDatasets,
|
||||
|
|
@ -47,13 +48,8 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
id: 'DatasetQualityDetailsController',
|
||||
context: initialContext,
|
||||
predictableActionArguments: true,
|
||||
initial: 'uninitialized',
|
||||
initial: 'initializing',
|
||||
states: {
|
||||
uninitialized: {
|
||||
always: {
|
||||
target: 'initializing',
|
||||
},
|
||||
},
|
||||
initializing: {
|
||||
type: 'parallel',
|
||||
states: {
|
||||
|
|
@ -145,58 +141,14 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
done: {},
|
||||
},
|
||||
},
|
||||
dataStreamDegradedFields: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'loadDegradedFields',
|
||||
onDone: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFields'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'done',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
done: {
|
||||
on: {
|
||||
UPDATE_TIME_RANGE: {
|
||||
target: 'fetching',
|
||||
actions: ['resetDegradedFieldPageAndRowsPerPage'],
|
||||
},
|
||||
UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFieldTableOptions'],
|
||||
},
|
||||
OPEN_DEGRADED_FIELD_FLYOUT: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.initializeFixItFlow.ignoredValues',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
},
|
||||
CLOSE_DEGRADED_FIELD_FLYOUT: {
|
||||
target: 'done',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
dataStreamSettings: {
|
||||
initial: 'fetching',
|
||||
initial: 'fetchingDataStreamSettings',
|
||||
states: {
|
||||
fetching: {
|
||||
fetchingDataStreamSettings: {
|
||||
invoke: {
|
||||
src: 'loadDataStreamSettings',
|
||||
onDone: {
|
||||
target: 'initializeIntegrations',
|
||||
target: 'loadingIntegrationsAndDegradedFields',
|
||||
actions: ['storeDataStreamSettings'],
|
||||
},
|
||||
onError: [
|
||||
|
|
@ -211,9 +163,53 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
],
|
||||
},
|
||||
},
|
||||
initializeIntegrations: {
|
||||
loadingIntegrationsAndDegradedFields: {
|
||||
type: 'parallel',
|
||||
states: {
|
||||
dataStreamDegradedFields: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'loadDegradedFields',
|
||||
onDone: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFields', 'raiseDegradedFieldsLoaded'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'done',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
done: {
|
||||
on: {
|
||||
UPDATE_TIME_RANGE: {
|
||||
target: 'fetching',
|
||||
actions: ['resetDegradedFieldPageAndRowsPerPage'],
|
||||
},
|
||||
UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFieldTableOptions'],
|
||||
},
|
||||
OPEN_DEGRADED_FIELD_FLYOUT: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
},
|
||||
TOGGLE_CURRENT_QUALITY_ISSUES: {
|
||||
target: 'fetching',
|
||||
actions: ['toggleCurrentQualityIssues'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
integrationDetails: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
|
|
@ -230,9 +226,7 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
},
|
||||
},
|
||||
},
|
||||
done: {
|
||||
type: 'final',
|
||||
},
|
||||
done: {},
|
||||
},
|
||||
},
|
||||
integrationDashboards: {
|
||||
|
|
@ -257,60 +251,114 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
],
|
||||
},
|
||||
},
|
||||
done: {
|
||||
type: 'final',
|
||||
},
|
||||
done: {},
|
||||
unauthorized: {
|
||||
type: 'final',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
onDone: {
|
||||
target: 'done',
|
||||
},
|
||||
},
|
||||
done: {
|
||||
done: {},
|
||||
},
|
||||
on: {
|
||||
UPDATE_TIME_RANGE: {
|
||||
target: '.fetchingDataStreamSettings',
|
||||
},
|
||||
},
|
||||
},
|
||||
degradedFieldFlyout: {
|
||||
initial: 'pending',
|
||||
states: {
|
||||
pending: {
|
||||
always: [
|
||||
{
|
||||
target: 'closed',
|
||||
cond: 'hasNoDegradedFieldsSelected',
|
||||
},
|
||||
],
|
||||
},
|
||||
open: {
|
||||
type: 'parallel',
|
||||
states: {
|
||||
ignoredValues: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'loadDegradedFieldValues',
|
||||
onDone: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFieldValues'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'done',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
done: {},
|
||||
},
|
||||
},
|
||||
analyze: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'analyzeDegradedField',
|
||||
onDone: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFieldAnalysis'],
|
||||
},
|
||||
onError: {
|
||||
target: 'done',
|
||||
},
|
||||
},
|
||||
},
|
||||
done: {},
|
||||
},
|
||||
},
|
||||
},
|
||||
on: {
|
||||
CLOSE_DEGRADED_FIELD_FLYOUT: {
|
||||
target: 'closed',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
},
|
||||
UPDATE_TIME_RANGE: {
|
||||
target: 'fetching',
|
||||
actions: ['resetDegradedFieldPageAndRowsPerPage'],
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
|
||||
},
|
||||
},
|
||||
},
|
||||
closed: {
|
||||
on: {
|
||||
OPEN_DEGRADED_FIELD_FLYOUT: {
|
||||
target:
|
||||
'#DatasetQualityDetailsController.initializing.degradedFieldFlyout.open',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
initializeFixItFlow: {
|
||||
initial: 'closed',
|
||||
type: 'parallel',
|
||||
states: {
|
||||
ignoredValues: {
|
||||
initial: 'fetching',
|
||||
states: {
|
||||
fetching: {
|
||||
invoke: {
|
||||
src: 'loadDegradedFieldValues',
|
||||
onDone: {
|
||||
target: 'done',
|
||||
actions: ['storeDegradedFieldValues'],
|
||||
},
|
||||
onError: [
|
||||
{
|
||||
target: '#DatasetQualityDetailsController.indexNotFound',
|
||||
cond: 'isIndexNotFoundError',
|
||||
},
|
||||
{
|
||||
target: 'done',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
done: {
|
||||
on: {
|
||||
UPDATE_TIME_RANGE: {
|
||||
target: 'fetching',
|
||||
},
|
||||
},
|
||||
},
|
||||
on: {
|
||||
DEGRADED_FIELDS_LOADED: [
|
||||
{
|
||||
target: '.open',
|
||||
cond: 'shouldOpenFlyout',
|
||||
},
|
||||
},
|
||||
{
|
||||
target: '.closed',
|
||||
actions: ['storeExpandedDegradedField'],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -370,6 +418,13 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
}
|
||||
: {};
|
||||
}),
|
||||
storeDegradedFieldAnalysis: assign((_, event: DoneInvokeEvent<DegradedFieldAnalysis>) => {
|
||||
return 'data' in event
|
||||
? {
|
||||
degradedFieldAnalysis: event.data,
|
||||
}
|
||||
: {};
|
||||
}),
|
||||
storeDegradedFieldTableOptions: assign((context, event) => {
|
||||
return 'degraded_field_criteria' in event
|
||||
? {
|
||||
|
|
@ -380,11 +435,17 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
}
|
||||
: {};
|
||||
}),
|
||||
storeExpandedDegradedField: assign((context, event) => {
|
||||
storeExpandedDegradedField: assign((_, event) => {
|
||||
return {
|
||||
expandedDegradedField: 'fieldName' in event ? event.fieldName : undefined,
|
||||
};
|
||||
}),
|
||||
toggleCurrentQualityIssues: assign((context) => {
|
||||
return {
|
||||
showCurrentQualityIssues: !context.showCurrentQualityIssues,
|
||||
};
|
||||
}),
|
||||
raiseDegradedFieldsLoaded: raise('DEGRADED_FIELDS_LOADED'),
|
||||
resetDegradedFieldPageAndRowsPerPage: assign((context, _event) => ({
|
||||
degradedFields: {
|
||||
...context.degradedFields,
|
||||
|
|
@ -442,6 +503,19 @@ export const createPureDatasetQualityDetailsControllerStateMachine = (
|
|||
false
|
||||
);
|
||||
},
|
||||
shouldOpenFlyout: (context) => {
|
||||
return (
|
||||
Boolean(context.expandedDegradedField) &&
|
||||
Boolean(
|
||||
context.degradedFields.data?.some(
|
||||
(field) => field.name === context.expandedDegradedField
|
||||
)
|
||||
)
|
||||
);
|
||||
},
|
||||
hasNoDegradedFieldsSelected: (context) => {
|
||||
return !Boolean(context.expandedDegradedField);
|
||||
},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
|
@ -524,18 +598,46 @@ export const createDatasetQualityDetailsControllerStateMachine = ({
|
|||
loadDegradedFields: (context) => {
|
||||
const { startDate: start, endDate: end } = getDateISORange(context.timeRange);
|
||||
|
||||
return dataStreamDetailsClient.getDataStreamDegradedFields({
|
||||
dataStream: context.dataStream,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
if (!context?.isNonAggregatable) {
|
||||
return dataStreamDetailsClient.getDataStreamDegradedFields({
|
||||
dataStream:
|
||||
context.showCurrentQualityIssues &&
|
||||
'dataStreamSettings' in context &&
|
||||
context.dataStreamSettings
|
||||
? context.dataStreamSettings.lastBackingIndexName
|
||||
: context.dataStream,
|
||||
start,
|
||||
end,
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.resolve();
|
||||
},
|
||||
|
||||
loadDegradedFieldValues: (context) => {
|
||||
return dataStreamDetailsClient.getDataStreamDegradedFieldValues({
|
||||
dataStream: context.dataStream,
|
||||
degradedField: context.expandedDegradedField!,
|
||||
});
|
||||
if ('expandedDegradedField' in context && context.expandedDegradedField) {
|
||||
return dataStreamDetailsClient.getDataStreamDegradedFieldValues({
|
||||
dataStream: context.dataStream,
|
||||
degradedField: context.expandedDegradedField,
|
||||
});
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
analyzeDegradedField: (context) => {
|
||||
if (context?.degradedFields?.data?.length) {
|
||||
const selectedDegradedField = context.degradedFields.data.find(
|
||||
(field) => field.name === context.expandedDegradedField
|
||||
);
|
||||
|
||||
if (selectedDegradedField) {
|
||||
return dataStreamDetailsClient.analyzeDegradedField({
|
||||
dataStream: context.dataStream,
|
||||
degradedField: context.expandedDegradedField!,
|
||||
lastBackingIndex: selectedDegradedField.indexFieldWasLastPresentIn,
|
||||
});
|
||||
}
|
||||
}
|
||||
return Promise.resolve();
|
||||
},
|
||||
loadDataStreamSettings: (context) => {
|
||||
return dataStreamDetailsClient.getDataStreamSettings({
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {
|
|||
DataStreamDetails,
|
||||
DataStreamSettings,
|
||||
DegradedField,
|
||||
DegradedFieldAnalysis,
|
||||
DegradedFieldResponse,
|
||||
DegradedFieldValues,
|
||||
NonAggregatableDatasets,
|
||||
|
|
@ -40,11 +41,13 @@ export interface WithDefaultControllerState {
|
|||
dataStream: string;
|
||||
degradedFields: DegradedFieldsTableConfig;
|
||||
timeRange: TimeRangeConfig;
|
||||
showCurrentQualityIssues: boolean;
|
||||
breakdownField?: string;
|
||||
isBreakdownFieldEcs?: boolean;
|
||||
isIndexNotFoundError?: boolean;
|
||||
integration?: Integration;
|
||||
expandedDegradedField?: string;
|
||||
isNonAggregatable?: boolean;
|
||||
}
|
||||
|
||||
export interface WithDataStreamDetails {
|
||||
|
|
@ -80,24 +83,29 @@ export interface WithDegradedFieldValues {
|
|||
degradedFieldValues: DegradedFieldValues;
|
||||
}
|
||||
|
||||
export interface WithDegradeFieldAnalysis {
|
||||
degradedFieldAnalysis: DegradedFieldAnalysis;
|
||||
}
|
||||
|
||||
export type DefaultDatasetQualityDetailsContext = Pick<
|
||||
WithDefaultControllerState,
|
||||
'degradedFields' | 'timeRange' | 'isIndexNotFoundError'
|
||||
'degradedFields' | 'timeRange' | 'isIndexNotFoundError' | 'showCurrentQualityIssues'
|
||||
>;
|
||||
|
||||
export type DatasetQualityDetailsControllerTypeState =
|
||||
| {
|
||||
value:
|
||||
| 'initializing'
|
||||
| 'uninitialized'
|
||||
| 'initializing.nonAggregatableDataset.fetching'
|
||||
| 'initializing.dataStreamDegradedFields.fetching'
|
||||
| 'initializing.dataStreamSettings.fetching'
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching'
|
||||
| 'initializing.dataStreamSettings.fetchingDataStreamSettings'
|
||||
| 'initializing.dataStreamDetails.fetching';
|
||||
context: WithDefaultControllerState;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.nonAggregatableDataset.done';
|
||||
value:
|
||||
| 'initializing.nonAggregatableDataset.done'
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.fetching';
|
||||
context: WithDefaultControllerState & WithNonAggregatableDatasetStatus;
|
||||
}
|
||||
| {
|
||||
|
|
@ -113,29 +121,44 @@ export type DatasetQualityDetailsControllerTypeState =
|
|||
context: WithDefaultControllerState & WithBreakdownInEcsCheck;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.dataStreamDegradedFields.done';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.initializeFixItFlow.ignoredValues.fetching';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.initializeFixItFlow.ignoredValues.done';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues;
|
||||
value: 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.dataStreamDegradedFields.done';
|
||||
context: WithDefaultControllerState &
|
||||
WithNonAggregatableDatasetStatus &
|
||||
WithDegradedFieldsData;
|
||||
}
|
||||
| {
|
||||
value:
|
||||
| 'initializing.dataStreamSettings.initializeIntegrations'
|
||||
| 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.fetching'
|
||||
| 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.fetching'
|
||||
| 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.unauthorized';
|
||||
| 'initializing.degradedFieldFlyout.open.ignoredValues.fetching'
|
||||
| 'initializing.degradedFieldFlyout.open.analyze.fetching';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.degradedFieldFlyout.open.ignoredValues.done';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradedFieldValues;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.degradedFieldFlyout.open.analyze.done';
|
||||
context: WithDefaultControllerState & WithDegradedFieldsData & WithDegradeFieldAnalysis;
|
||||
}
|
||||
| {
|
||||
value: 'initializing.degradedFieldFlyout.open';
|
||||
context: WithDefaultControllerState &
|
||||
WithDegradedFieldsData &
|
||||
WithDegradedFieldValues &
|
||||
WithDegradeFieldAnalysis;
|
||||
}
|
||||
| {
|
||||
value:
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields'
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.fetching'
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.fetching'
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.unauthorized';
|
||||
context: WithDefaultControllerState & WithDataStreamSettings;
|
||||
}
|
||||
| {
|
||||
value:
|
||||
| 'initializing.dataStreamSettings.initializeIntegrations.integrationDetails.done'
|
||||
| 'initializing.dataStreamSettings.initializeIntegrations.integrationDashboards.done';
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDetails.done'
|
||||
| 'initializing.dataStreamSettings.loadingIntegrationsAndDegradedFields.integrationDashboards.done';
|
||||
context: WithDefaultControllerState & WithDataStreamSettings & WithIntegration;
|
||||
};
|
||||
|
||||
|
|
@ -154,6 +177,9 @@ export type DatasetQualityDetailsControllerEvent =
|
|||
| {
|
||||
type: 'CLOSE_DEGRADED_FIELD_FLYOUT';
|
||||
}
|
||||
| {
|
||||
type: 'DEGRADED_FIELDS_LOADED';
|
||||
}
|
||||
| {
|
||||
type: 'BREAKDOWN_FIELD_CHANGE';
|
||||
breakdownField: string | undefined;
|
||||
|
|
@ -170,4 +196,5 @@ export type DatasetQualityDetailsControllerEvent =
|
|||
| DoneInvokeEvent<DegradedFieldValues>
|
||||
| DoneInvokeEvent<DataStreamSettings>
|
||||
| DoneInvokeEvent<Integration>
|
||||
| DoneInvokeEvent<Dashboard[]>;
|
||||
| DoneInvokeEvent<Dashboard[]>
|
||||
| DoneInvokeEvent<DegradedFieldAnalysis>;
|
||||
|
|
|
|||
|
|
@ -36,12 +36,15 @@ export async function getDataStreamSettings({
|
|||
dataStreamService.getMatchingDataStreams(esClient, dataStream),
|
||||
datasetQualityPrivileges.getDatasetPrivileges(esClient, dataStream),
|
||||
]);
|
||||
|
||||
const integration = dataStreamInfo?._meta?.package?.name;
|
||||
const lastBackingIndex = dataStreamInfo?.indices?.slice(-1)[0];
|
||||
|
||||
return {
|
||||
createdOn,
|
||||
integration,
|
||||
datasetUserPrivileges,
|
||||
lastBackingIndexName: lastBackingIndex?.index_name,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
MappingTypeMapping,
|
||||
MappingProperty,
|
||||
PropertyName,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client';
|
||||
|
||||
export interface DataStreamMappingResponse {
|
||||
fieldCount: number;
|
||||
fieldPresent: boolean;
|
||||
fieldMapping?: {
|
||||
type?: string;
|
||||
ignore_above?: number;
|
||||
};
|
||||
}
|
||||
|
||||
type MappingWithProperty = MappingTypeMapping & {
|
||||
properties: Record<PropertyName, MappingProperty>;
|
||||
};
|
||||
|
||||
type MappingWithFields = MappingTypeMapping & {
|
||||
fields: Record<PropertyName, MappingProperty>;
|
||||
};
|
||||
export async function getDataStreamMapping({
|
||||
field,
|
||||
datasetQualityESClient,
|
||||
dataStream,
|
||||
lastBackingIndex,
|
||||
}: {
|
||||
field: string;
|
||||
datasetQualityESClient: DatasetQualityESClient;
|
||||
dataStream: string;
|
||||
lastBackingIndex: string;
|
||||
}): Promise<DataStreamMappingResponse> {
|
||||
const mappings = await datasetQualityESClient.mappings({ index: dataStream });
|
||||
const properties = mappings[lastBackingIndex]?.mappings?.properties;
|
||||
const { count: fieldCount, capturedMapping: mapping } = countFields(properties ?? {}, field);
|
||||
const fieldPresent = mapping !== undefined;
|
||||
const fieldMapping = fieldPresent
|
||||
? {
|
||||
type: mapping?.type,
|
||||
ignore_above: (mapping as any)?.ignore_above,
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
fieldCount,
|
||||
fieldPresent,
|
||||
fieldMapping,
|
||||
};
|
||||
}
|
||||
|
||||
function isNestedProperty(property: MappingProperty): property is MappingWithProperty {
|
||||
return 'properties' in property && property.properties !== undefined;
|
||||
}
|
||||
|
||||
function isNestedField(property: MappingProperty): property is MappingWithFields {
|
||||
return 'fields' in property && property.fields !== undefined;
|
||||
}
|
||||
|
||||
function countFields(
|
||||
mappings: Record<PropertyName, MappingProperty>,
|
||||
captureField?: string,
|
||||
prefix = ''
|
||||
): { count: number; capturedMapping?: any } {
|
||||
let fieldCount = 0;
|
||||
let capturedMapping;
|
||||
|
||||
for (const field in mappings) {
|
||||
if (Object.prototype.hasOwnProperty.call(mappings, field)) {
|
||||
const mappingField = mappings[field];
|
||||
const currentPath = [prefix, field].filter(Boolean).join('.');
|
||||
|
||||
// Capture the value if the current path matches the captureField
|
||||
if (captureField && currentPath === captureField) {
|
||||
capturedMapping = mappingField;
|
||||
}
|
||||
|
||||
fieldCount++; // Count the current field
|
||||
|
||||
// If there are properties, recursively count nested fields
|
||||
if (isNestedProperty(mappingField)) {
|
||||
const { count, capturedMapping: nestedCapturedValue } = countFields(
|
||||
mappingField.properties,
|
||||
captureField,
|
||||
currentPath
|
||||
);
|
||||
fieldCount += count;
|
||||
if (nestedCapturedValue !== undefined) {
|
||||
capturedMapping = nestedCapturedValue;
|
||||
}
|
||||
}
|
||||
|
||||
// If there are fields, recursively count nested fields
|
||||
if (isNestedField(mappingField)) {
|
||||
const { count, capturedMapping: nestedCapturedValue } = countFields(
|
||||
mappingField.fields,
|
||||
captureField,
|
||||
currentPath
|
||||
);
|
||||
fieldCount += count;
|
||||
if (nestedCapturedValue !== undefined) {
|
||||
capturedMapping = nestedCapturedValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: fieldCount, capturedMapping };
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { DatasetQualityESClient } from '../../../utils/create_dataset_quality_es_client';
|
||||
import { toBoolean } from '../../../utils/to_boolean';
|
||||
|
||||
export interface DataStreamSettingResponse {
|
||||
nestedFieldLimit?: number;
|
||||
totalFieldLimit: number;
|
||||
ignoreDynamicBeyondLimit?: boolean;
|
||||
ignoreMalformed?: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_FIELD_LIMIT = 1000;
|
||||
const DEFAULT_NESTED_FIELD_LIMIT = 50;
|
||||
|
||||
export async function getDataStreamSettings({
|
||||
datasetQualityESClient,
|
||||
dataStream,
|
||||
lastBackingIndex,
|
||||
}: {
|
||||
datasetQualityESClient: DatasetQualityESClient;
|
||||
dataStream: string;
|
||||
lastBackingIndex: string;
|
||||
}): Promise<DataStreamSettingResponse> {
|
||||
const settings = await datasetQualityESClient.settings({ index: dataStream });
|
||||
const indexSettings = settings[lastBackingIndex]?.settings?.index?.mapping;
|
||||
|
||||
return {
|
||||
nestedFieldLimit: indexSettings?.nested_fields?.limit
|
||||
? Number(indexSettings?.nested_fields?.limit)
|
||||
: DEFAULT_NESTED_FIELD_LIMIT,
|
||||
totalFieldLimit: indexSettings?.total_fields?.limit
|
||||
? Number(indexSettings?.total_fields?.limit)
|
||||
: DEFAULT_FIELD_LIMIT,
|
||||
ignoreDynamicBeyondLimit: toBoolean(indexSettings?.total_fields?.ignore_dynamic_beyond_limit),
|
||||
ignoreMalformed: toBoolean(indexSettings?.ignore_malformed),
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
|
||||
import { DegradedFieldAnalysis } from '../../../../common/api_types';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { getDataStreamMapping } from './get_datastream_mappings';
|
||||
import { getDataStreamSettings } from './get_datastream_settings';
|
||||
|
||||
// TODO: The API should also in future return some analysis around the ignore_malformed check.
|
||||
// As this check is expensive and steps are not very concrete, its not done for the initial iteration
|
||||
export async function analyzeDegradedField({
|
||||
esClient,
|
||||
dataStream,
|
||||
degradedField,
|
||||
lastBackingIndex,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
dataStream: string;
|
||||
degradedField: string;
|
||||
lastBackingIndex: string;
|
||||
}): Promise<DegradedFieldAnalysis> {
|
||||
const datasetQualityESClient = createDatasetQualityESClient(esClient);
|
||||
|
||||
const [
|
||||
{ fieldCount, fieldPresent, fieldMapping },
|
||||
{ nestedFieldLimit, totalFieldLimit, ignoreDynamicBeyondLimit, ignoreMalformed },
|
||||
] = await Promise.all([
|
||||
getDataStreamMapping({
|
||||
datasetQualityESClient,
|
||||
dataStream,
|
||||
field: degradedField,
|
||||
lastBackingIndex,
|
||||
}),
|
||||
getDataStreamSettings({ datasetQualityESClient, dataStream, lastBackingIndex }),
|
||||
]);
|
||||
|
||||
return {
|
||||
isFieldLimitIssue: Boolean(
|
||||
!fieldPresent && ignoreDynamicBeyondLimit && fieldCount === totalFieldLimit
|
||||
),
|
||||
fieldCount,
|
||||
fieldMapping,
|
||||
totalFieldLimit,
|
||||
ignoreMalformed,
|
||||
nestedFieldLimit,
|
||||
};
|
||||
}
|
||||
|
|
@ -10,7 +10,7 @@ import { rangeQuery, existsQuery } from '@kbn/observability-plugin/server';
|
|||
import { DegradedFieldResponse } from '../../../../common/api_types';
|
||||
import { MAX_DEGRADED_FIELDS } from '../../../../common/constants';
|
||||
import { createDatasetQualityESClient } from '../../../utils';
|
||||
import { _IGNORED, TIMESTAMP } from '../../../../common/es_fields';
|
||||
import { _IGNORED, INDEX, TIMESTAMP } from '../../../../common/es_fields';
|
||||
import { getFieldIntervalInSeconds } from './get_interval';
|
||||
|
||||
export async function getDegradedFields({
|
||||
|
|
@ -43,6 +43,15 @@ export async function getDegradedFields({
|
|||
field: TIMESTAMP,
|
||||
},
|
||||
},
|
||||
index: {
|
||||
terms: {
|
||||
size: 1,
|
||||
field: INDEX,
|
||||
order: {
|
||||
_key: 'desc',
|
||||
},
|
||||
},
|
||||
},
|
||||
timeSeries: {
|
||||
date_histogram: {
|
||||
field: TIMESTAMP,
|
||||
|
|
@ -80,6 +89,7 @@ export async function getDegradedFields({
|
|||
x: timeSeriesBucket.key,
|
||||
y: timeSeriesBucket.doc_count,
|
||||
})),
|
||||
indexFieldWasLastPresentIn: bucket.index.buckets[0].key as string,
|
||||
})) ?? [],
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import {
|
|||
DegradedFieldResponse,
|
||||
DatasetUserPrivileges,
|
||||
DegradedFieldValues,
|
||||
DegradedFieldAnalysis,
|
||||
} from '../../../common/api_types';
|
||||
import { rangeRt, typeRt, typesRt } from '../../types/default_api_types';
|
||||
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
|
||||
|
|
@ -26,6 +27,7 @@ import { getDegradedDocsPaginated } from './get_degraded_docs';
|
|||
import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams';
|
||||
import { getDegradedFields } from './get_degraded_fields';
|
||||
import { getDegradedFieldValues } from './get_degraded_field_values';
|
||||
import { analyzeDegradedField } from './get_degraded_field_analysis';
|
||||
import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats';
|
||||
|
||||
const statsRoute = createDatasetQualityServerRoute({
|
||||
|
|
@ -291,6 +293,37 @@ const dataStreamDetailsRoute = createDatasetQualityServerRoute({
|
|||
},
|
||||
});
|
||||
|
||||
const analyzeDegradedFieldRoute = createDatasetQualityServerRoute({
|
||||
endpoint:
|
||||
'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_field/{degradedField}/analyze',
|
||||
params: t.type({
|
||||
path: t.type({
|
||||
dataStream: t.string,
|
||||
degradedField: t.string,
|
||||
}),
|
||||
query: t.type({
|
||||
lastBackingIndex: t.string,
|
||||
}),
|
||||
}),
|
||||
options: {
|
||||
tags: [],
|
||||
},
|
||||
async handler(resources): Promise<DegradedFieldAnalysis> {
|
||||
const { context, params } = resources;
|
||||
const coreContext = await context.core;
|
||||
const esClient = coreContext.elasticsearch.client.asCurrentUser;
|
||||
|
||||
const degradedFieldAnalysis = await analyzeDegradedField({
|
||||
esClient,
|
||||
dataStream: params.path.dataStream,
|
||||
degradedField: params.path.degradedField,
|
||||
lastBackingIndex: params.query.lastBackingIndex,
|
||||
});
|
||||
|
||||
return degradedFieldAnalysis;
|
||||
},
|
||||
});
|
||||
|
||||
export const dataStreamsRouteRepository = {
|
||||
...statsRoute,
|
||||
...degradedDocsRoute,
|
||||
|
|
@ -300,4 +333,5 @@ export const dataStreamsRouteRepository = {
|
|||
...degradedFieldValuesRoute,
|
||||
...dataStreamDetailsRoute,
|
||||
...dataStreamSettingsRoute,
|
||||
...analyzeDegradedFieldRoute,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,7 +7,13 @@
|
|||
|
||||
import { ESSearchRequest, InferSearchResponseOf } from '@kbn/es-types';
|
||||
import { ElasticsearchClient } from '@kbn/core/server';
|
||||
import { FieldCapsRequest, FieldCapsResponse, Indices } from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
FieldCapsRequest,
|
||||
FieldCapsResponse,
|
||||
Indices,
|
||||
IndicesGetMappingResponse,
|
||||
IndicesGetSettingsResponse,
|
||||
} from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
type DatasetQualityESSearchParams = ESSearchRequest & {
|
||||
size: number;
|
||||
|
|
@ -35,5 +41,11 @@ export function createDatasetQualityESClient(esClient: ElasticsearchClient) {
|
|||
async fieldCaps(params: FieldCapsRequest): Promise<FieldCapsResponse> {
|
||||
return esClient.fieldCaps(params) as Promise<any>;
|
||||
},
|
||||
async mappings(params: { index: string }): Promise<IndicesGetMappingResponse> {
|
||||
return esClient.indices.getMapping(params);
|
||||
},
|
||||
async settings(params: { index: string }): Promise<IndicesGetSettingsResponse> {
|
||||
return esClient.indices.getSettings(params);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export function toBoolean(value?: string | boolean): boolean {
|
||||
if (typeof value === 'string') {
|
||||
return value.toLowerCase() === 'true';
|
||||
}
|
||||
return Boolean(value);
|
||||
}
|
||||
|
|
@ -0,0 +1,157 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import { log, timerange } from '@kbn/apm-synthtrace-client';
|
||||
import { SupertestWithRoleScopeType } from '../../../services';
|
||||
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { createBackingIndexNameWithoutVersion, setDataStreamSettings } from './es_utils';
|
||||
|
||||
const MORE_THAN_1024_CHARS =
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
|
||||
|
||||
export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
||||
const roleScopedSupertest = getService('roleScopedSupertest');
|
||||
const synthtrace = getService('logsSynthtraceEsClient');
|
||||
const esClient = getService('es');
|
||||
const start = '2024-09-20T11:00:00.000Z';
|
||||
const end = '2024-09-20T11:01:00.000Z';
|
||||
const type = 'logs';
|
||||
const dataset = 'synth.good';
|
||||
const namespace = 'default';
|
||||
const serviceName = 'my-service';
|
||||
const hostName = 'synth-host';
|
||||
const dataStreamName = `${type}-${dataset}-${namespace}`;
|
||||
|
||||
async function callApiAs({
|
||||
roleScopedSupertestWithCookieCredentials,
|
||||
apiParams: { dataStream, degradedField, lastBackingIndex },
|
||||
}: {
|
||||
roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType;
|
||||
apiParams: {
|
||||
dataStream: string;
|
||||
degradedField: string;
|
||||
lastBackingIndex: string;
|
||||
};
|
||||
}) {
|
||||
return roleScopedSupertestWithCookieCredentials
|
||||
.get(
|
||||
`/internal/dataset_quality/data_streams/${dataStream}/degraded_field/${degradedField}/analyze`
|
||||
)
|
||||
.query({ lastBackingIndex });
|
||||
}
|
||||
|
||||
describe('Degraded field analyze', () => {
|
||||
let supertestAdminWithCookieCredentials: SupertestWithRoleScopeType;
|
||||
|
||||
before(async () => {
|
||||
supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
|
||||
'admin',
|
||||
{
|
||||
useCookieHeader: true,
|
||||
withInternalHeaders: true,
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe('gets limit analysis for a given datastream and degraded field', () => {
|
||||
before(async () => {
|
||||
await synthtrace.index([
|
||||
timerange(start, end)
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) =>
|
||||
log
|
||||
.create()
|
||||
.message('This is a log message')
|
||||
.timestamp(timestamp)
|
||||
.dataset(dataset)
|
||||
.namespace(namespace)
|
||||
.defaults({
|
||||
'log.file.path': '/my-service.log',
|
||||
'service.name': serviceName,
|
||||
'host.name': hostName,
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
})
|
||||
),
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return default limits and should return isFieldLimitIssue as false', async () => {
|
||||
const resp = await callApiAs({
|
||||
roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials,
|
||||
apiParams: {
|
||||
dataStream: dataStreamName,
|
||||
degradedField: 'test_field',
|
||||
lastBackingIndex: `${createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset,
|
||||
namespace,
|
||||
})}-000001`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(resp.body.isFieldLimitIssue).to.be(false);
|
||||
expect(resp.body.fieldCount).to.be(25);
|
||||
expect(resp.body.fieldMapping).to.eql({ type: 'keyword', ignore_above: 1024 });
|
||||
expect(resp.body.totalFieldLimit).to.be(1000);
|
||||
expect(resp.body.ignoreMalformed).to.be(true);
|
||||
expect(resp.body.nestedFieldLimit).to.be(50);
|
||||
});
|
||||
|
||||
it('should return updated limits and should return isFieldLimitIssue as true', async () => {
|
||||
await setDataStreamSettings(esClient, dataStreamName, {
|
||||
'mapping.total_fields.limit': 25,
|
||||
});
|
||||
await synthtrace.index([
|
||||
timerange(start, end)
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) =>
|
||||
log
|
||||
.create()
|
||||
.message('This is a log message')
|
||||
.timestamp(timestamp)
|
||||
.dataset(dataset)
|
||||
.namespace(namespace)
|
||||
.defaults({
|
||||
'log.file.path': '/my-service.log',
|
||||
'service.name': serviceName,
|
||||
'host.name': hostName,
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
'cloud.region': 'us-east-1',
|
||||
})
|
||||
),
|
||||
]);
|
||||
|
||||
const resp = await callApiAs({
|
||||
roleScopedSupertestWithCookieCredentials: supertestAdminWithCookieCredentials,
|
||||
apiParams: {
|
||||
dataStream: dataStreamName,
|
||||
degradedField: 'cloud.region',
|
||||
lastBackingIndex: `${createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset,
|
||||
namespace,
|
||||
})}-000001`,
|
||||
},
|
||||
});
|
||||
|
||||
expect(resp.body.isFieldLimitIssue).to.be(true);
|
||||
expect(resp.body.fieldCount).to.be(25);
|
||||
expect(resp.body.fieldMapping).to.be(undefined); // As the field limit was reached, field cannot be mapped
|
||||
expect(resp.body.totalFieldLimit).to.be(25);
|
||||
expect(resp.body.ignoreMalformed).to.be(true);
|
||||
expect(resp.body.nestedFieldLimit).to.be(50);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
function getCurrentDateFormatted() {
|
||||
const date = new Date();
|
||||
const year = date.getFullYear();
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getDate()).padStart(2, '0');
|
||||
|
||||
return `${year}.${month}.${day}`;
|
||||
}
|
||||
|
||||
export function createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset,
|
||||
namespace = 'default',
|
||||
}: {
|
||||
type: string;
|
||||
dataset: string;
|
||||
namespace: string;
|
||||
}) {
|
||||
return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`;
|
||||
}
|
||||
|
||||
export async function setDataStreamSettings(
|
||||
esClient: Client,
|
||||
name: string,
|
||||
settings: IndicesIndexSettings
|
||||
) {
|
||||
return esClient.indices.putSettings({
|
||||
index: name,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_cont
|
|||
|
||||
export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
|
||||
describe('Dataset quality', () => {
|
||||
loadTestFile(require.resolve('./integrations/integrations'));
|
||||
loadTestFile(require.resolve('./integrations'));
|
||||
loadTestFile(require.resolve('./degraded_field_analyze'));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@
|
|||
import { RoleCredentials, InternalRequestHeader } from '@kbn/ftr-common-functional-services';
|
||||
import expect from '@kbn/expect';
|
||||
import { APIReturnType } from '@kbn/dataset-quality-plugin/common/rest';
|
||||
import { CustomIntegration } from '../../../../services/package_api';
|
||||
import { DeploymentAgnosticFtrProviderContext } from '../../../../ftr_provider_context';
|
||||
import { CustomIntegration } from '../../../services/package_api';
|
||||
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
||||
const samlAuth = getService('samlAuth');
|
||||
|
|
@ -12,6 +12,7 @@ import { deploymentAgnosticServices } from './deployment_agnostic_services';
|
|||
import { PackageApiProvider } from './package_api';
|
||||
import { RoleScopedSupertestProvider, SupertestWithRoleScope } from './role_scoped_supertest';
|
||||
import { SloApiProvider } from './slo_api';
|
||||
import { LogsSynthtraceEsClientProvider } from './logs_synthtrace_es_client';
|
||||
|
||||
export type {
|
||||
InternalRequestHeader,
|
||||
|
|
@ -28,6 +29,7 @@ export const services = {
|
|||
packageApi: PackageApiProvider,
|
||||
sloApi: SloApiProvider,
|
||||
roleScopedSupertest: RoleScopedSupertestProvider,
|
||||
logsSynthtraceEsClient: LogsSynthtraceEsClientProvider,
|
||||
// create a new deployment-agnostic service and load here
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { createLogger, LogLevel, LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
|
||||
import { DeploymentAgnosticFtrProviderContext } from '../ftr_provider_context';
|
||||
|
||||
export function LogsSynthtraceEsClientProvider({
|
||||
getService,
|
||||
}: DeploymentAgnosticFtrProviderContext) {
|
||||
return new LogsSynthtraceEsClient({
|
||||
client: getService('es'),
|
||||
logger: createLogger(LogLevel.info),
|
||||
refreshAfterIndex: true,
|
||||
});
|
||||
}
|
||||
|
|
@ -15,6 +15,7 @@ import {
|
|||
getDataStreamSettingsOfEarliestIndex,
|
||||
rolloverDataStream,
|
||||
} from '../../utils';
|
||||
import { createBackingIndexNameWithoutVersion } from './es_utils';
|
||||
|
||||
export default function ApiTest({ getService }: FtrProviderContext) {
|
||||
const registry = getService('registry');
|
||||
|
|
@ -110,32 +111,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
|
|||
expect(resp.body).eql(defaultDataStreamPrivileges);
|
||||
});
|
||||
|
||||
it('returns "createdOn" correctly', async () => {
|
||||
const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex(
|
||||
esClient,
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
const resp = await callApiAs(
|
||||
'datasetQualityMonitorUser',
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date));
|
||||
});
|
||||
|
||||
it('returns "createdOn" correctly for rolled over dataStream', async () => {
|
||||
await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`);
|
||||
const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex(
|
||||
esClient,
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
const resp = await callApiAs(
|
||||
'datasetQualityMonitorUser',
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date));
|
||||
});
|
||||
|
||||
it('returns "createdOn" and "integration" correctly when available', async () => {
|
||||
it('returns "createdOn", "integration" and "lastBackingIndexName" correctly when available', async () => {
|
||||
const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex(
|
||||
esClient,
|
||||
`${type}-${integrationDataset}-${namespace}`
|
||||
|
|
@ -146,11 +122,34 @@ export default function ApiTest({ getService }: FtrProviderContext) {
|
|||
);
|
||||
expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date));
|
||||
expect(resp.body.integration).to.be('apache');
|
||||
expect(resp.body.lastBackingIndexName).to.be(
|
||||
`${createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset: integrationDataset,
|
||||
namespace,
|
||||
})}-000001`
|
||||
);
|
||||
expect(resp.body.datasetUserPrivileges).to.eql(
|
||||
defaultDataStreamPrivileges.datasetUserPrivileges
|
||||
);
|
||||
});
|
||||
|
||||
it('returns "createdOn" and "lastBackingIndexName" for rolled over dataStream', async () => {
|
||||
await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`);
|
||||
const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex(
|
||||
esClient,
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
const resp = await callApiAs(
|
||||
'datasetQualityMonitorUser',
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date));
|
||||
expect(resp.body.lastBackingIndexName).to.be(
|
||||
`${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002`
|
||||
);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
await pkgService.uninstallPackage(pkg);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import expect from '@kbn/expect';
|
|||
import { DegradedField } from '@kbn/dataset-quality-plugin/common/api_types';
|
||||
import { DatasetQualityApiClientKey } from '../../common/config';
|
||||
import { FtrProviderContext } from '../../common/ftr_provider_context';
|
||||
import { rolloverDataStream } from '../../utils';
|
||||
import { createBackingIndexNameWithoutVersion } from './es_utils';
|
||||
|
||||
const MORE_THAN_1024_CHARS =
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
|
||||
|
|
@ -18,6 +20,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
|
|||
const registry = getService('registry');
|
||||
const synthtrace = getService('logSynthtraceEsClient');
|
||||
const datasetQualityApiClient = getService('datasetQualityApiClient');
|
||||
const esClient = getService('es');
|
||||
const start = '2024-05-22T08:00:00.000Z';
|
||||
const end = '2024-05-23T08:02:00.000Z';
|
||||
const type = 'logs';
|
||||
|
|
@ -130,6 +133,56 @@ export default function ApiTest({ getService }: FtrProviderContext) {
|
|||
|
||||
expect(logLevelTimeSeries).to.eql(logsTimeSeriesData);
|
||||
});
|
||||
|
||||
it('should return the backing index where the ignored field was last seen', async () => {
|
||||
await rolloverDataStream(esClient, `${type}-${degradedFieldDataset}-${namespace}`);
|
||||
await synthtrace.index([
|
||||
timerange(start, end)
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) =>
|
||||
log
|
||||
.create()
|
||||
.message('This is a error message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.timestamp(timestamp)
|
||||
.dataset(degradedFieldDataset)
|
||||
.namespace(namespace)
|
||||
.defaults({
|
||||
'log.file.path': '/error.log',
|
||||
'service.name': serviceName + 1,
|
||||
})
|
||||
),
|
||||
]);
|
||||
|
||||
const resp = await callApiAs(
|
||||
'datasetQualityMonitorUser',
|
||||
`${type}-${degradedFieldDataset}-${namespace}`
|
||||
);
|
||||
|
||||
const logLevelLastBackingIndex = resp.body.degradedFields.find(
|
||||
(dFields) => dFields.name === 'log.level'
|
||||
)?.indexFieldWasLastPresentIn;
|
||||
|
||||
const traceIdLastBackingIndex = resp.body.degradedFields.find(
|
||||
(dFields) => dFields.name === 'trace.id'
|
||||
)?.indexFieldWasLastPresentIn;
|
||||
|
||||
expect(logLevelLastBackingIndex).to.be(
|
||||
`${createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset: degradedFieldDataset,
|
||||
namespace,
|
||||
})}-000002`
|
||||
);
|
||||
expect(traceIdLastBackingIndex).to.be(
|
||||
`${createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset: degradedFieldDataset,
|
||||
namespace,
|
||||
})}-000001`
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export async function addIntegrationToLogIndexTemplate({
|
||||
esClient,
|
||||
|
|
@ -52,3 +53,35 @@ export async function cleanLogIndexTemplate({ esClient }: { esClient: Client })
|
|||
},
|
||||
});
|
||||
}
|
||||
|
||||
function getCurrentDateFormatted() {
|
||||
const date = new Date();
|
||||
const year = date.getFullYear();
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getDate()).padStart(2, '0');
|
||||
|
||||
return `${year}.${month}.${day}`;
|
||||
}
|
||||
|
||||
export function createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset,
|
||||
namespace = 'default',
|
||||
}: {
|
||||
type: string;
|
||||
dataset: string;
|
||||
namespace: string;
|
||||
}) {
|
||||
return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`;
|
||||
}
|
||||
|
||||
export async function setDataStreamSettings(
|
||||
esClient: Client,
|
||||
name: string,
|
||||
settings: IndicesIndexSettings
|
||||
) {
|
||||
return esClient.indices.putSettings({
|
||||
index: name,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,108 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import { DatasetQualityFtrProviderContext } from './config';
|
||||
import {
|
||||
createDegradedFieldsRecord,
|
||||
datasetNames,
|
||||
defaultNamespace,
|
||||
getInitialTestLogs,
|
||||
ANOTHER_1024_CHARS,
|
||||
MORE_THAN_1024_CHARS,
|
||||
} from './data';
|
||||
|
||||
export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) {
|
||||
const PageObjects = getPageObjects([
|
||||
'common',
|
||||
'navigationalSearch',
|
||||
'observabilityLogsExplorer',
|
||||
'datasetQuality',
|
||||
]);
|
||||
const testSubjects = getService('testSubjects');
|
||||
const synthtrace = getService('logSynthtraceEsClient');
|
||||
const retry = getService('retry');
|
||||
const to = '2024-01-01T12:00:00.000Z';
|
||||
const degradedDatasetName = datasetNames[2];
|
||||
const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`;
|
||||
|
||||
describe('Degraded fields flyout', () => {
|
||||
before(async () => {
|
||||
await synthtrace.index([
|
||||
// Ingest basic logs
|
||||
getInitialTestLogs({ to, count: 4 }),
|
||||
// Ingest Degraded Logs
|
||||
createDegradedFieldsRecord({
|
||||
to: new Date().toISOString(),
|
||||
count: 2,
|
||||
dataset: degradedDatasetName,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
|
||||
describe('degraded field flyout open-close', () => {
|
||||
it('should open and close the flyout when user clicks on the expand button', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field');
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
|
||||
await testSubjects.missingOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
});
|
||||
|
||||
it('should open the flyout when navigating to the page with degradedField in URL State', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('values exist', () => {
|
||||
it('should display the degraded field values', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
ANOTHER_1024_CHARS
|
||||
);
|
||||
const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
MORE_THAN_1024_CHARS
|
||||
);
|
||||
expect(cloudAvailabilityZoneValueExists).to.be(true);
|
||||
expect(cloudAvailabilityZoneValue2Exists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,419 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import moment from 'moment/moment';
|
||||
import { generateShortId, log, timerange } from '@kbn/apm-synthtrace-client';
|
||||
import { DatasetQualityFtrProviderContext } from './config';
|
||||
import {
|
||||
createDegradedFieldsRecord,
|
||||
datasetNames,
|
||||
defaultNamespace,
|
||||
getInitialTestLogs,
|
||||
ANOTHER_1024_CHARS,
|
||||
MORE_THAN_1024_CHARS,
|
||||
} from './data';
|
||||
|
||||
export default function ({ getService, getPageObjects }: DatasetQualityFtrProviderContext) {
|
||||
const PageObjects = getPageObjects([
|
||||
'common',
|
||||
'navigationalSearch',
|
||||
'observabilityLogsExplorer',
|
||||
'datasetQuality',
|
||||
]);
|
||||
const testSubjects = getService('testSubjects');
|
||||
const synthtrace = getService('logSynthtraceEsClient');
|
||||
const retry = getService('retry');
|
||||
const to = new Date().toISOString();
|
||||
const degradedDatasetName = datasetNames[2];
|
||||
const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`;
|
||||
|
||||
const degradedDatasetWithLimitsName = 'degraded.dataset.rca';
|
||||
const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`;
|
||||
const serviceName = 'test_service';
|
||||
const count = 5;
|
||||
|
||||
describe('Degraded fields flyout', () => {
|
||||
before(async () => {
|
||||
await synthtrace.index([
|
||||
// Ingest basic logs
|
||||
getInitialTestLogs({ to, count: 4 }),
|
||||
// Ingest Degraded Logs
|
||||
createDegradedFieldsRecord({
|
||||
to: new Date().toISOString(),
|
||||
count: 2,
|
||||
dataset: degradedDatasetName,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
|
||||
describe('degraded field flyout open-close', () => {
|
||||
it('should open and close the flyout when user clicks on the expand button', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field');
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
|
||||
await testSubjects.missingOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
});
|
||||
|
||||
it('should open the flyout when navigating to the page with degradedField in URL State', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('values exist', () => {
|
||||
it('should display the degraded field values', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
ANOTHER_1024_CHARS
|
||||
);
|
||||
const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
MORE_THAN_1024_CHARS
|
||||
);
|
||||
expect(cloudAvailabilityZoneValueExists).to.be(true);
|
||||
expect(cloudAvailabilityZoneValue2Exists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('testing root cause for ignored fields', () => {
|
||||
before(async () => {
|
||||
// Ingest Degraded Logs with 25 fields
|
||||
await synthtrace.index([
|
||||
timerange(moment(to).subtract(count, 'minute'), moment(to))
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) => {
|
||||
return Array(1)
|
||||
.fill(0)
|
||||
.flatMap(() =>
|
||||
log
|
||||
.create()
|
||||
.dataset(degradedDatasetWithLimitsName)
|
||||
.message('a log message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.service(serviceName)
|
||||
.namespace(defaultNamespace)
|
||||
.defaults({
|
||||
'service.name': serviceName,
|
||||
'trace.id': generateShortId(),
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
// Set Limit of 25
|
||||
await PageObjects.datasetQuality.setDataStreamSettings(
|
||||
degradedDatasetWithLimitDataStreamName,
|
||||
{
|
||||
'mapping.total_fields.limit': 25,
|
||||
}
|
||||
);
|
||||
|
||||
// Ingest Degraded Logs with 26 field
|
||||
await synthtrace.index([
|
||||
timerange(moment(to).subtract(count, 'minute'), moment(to))
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) => {
|
||||
return Array(1)
|
||||
.fill(0)
|
||||
.flatMap(() =>
|
||||
log
|
||||
.create()
|
||||
.dataset(degradedDatasetWithLimitsName)
|
||||
.message('a log message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.service(serviceName)
|
||||
.namespace(defaultNamespace)
|
||||
.defaults({
|
||||
'service.name': serviceName,
|
||||
'trace.id': generateShortId(),
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
'cloud.region': 'us-east-1',
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
// Rollover Datastream to reset the limit to default which is 1000
|
||||
await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName);
|
||||
|
||||
// Ingest docs with 26 fields again
|
||||
await synthtrace.index([
|
||||
timerange(moment(to).subtract(count, 'minute'), moment(to))
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) => {
|
||||
return Array(1)
|
||||
.fill(0)
|
||||
.flatMap(() =>
|
||||
log
|
||||
.create()
|
||||
.dataset(degradedDatasetWithLimitsName)
|
||||
.message('a log message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.service(serviceName)
|
||||
.namespace(defaultNamespace)
|
||||
.defaults({
|
||||
'log.file.path': '/my-service.log',
|
||||
'service.name': serviceName,
|
||||
'trace.id': generateShortId(),
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
'cloud.region': 'us-east-1',
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
);
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
describe('field character limit exceeded', () => {
|
||||
it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const fieldIgnoredMessageExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause',
|
||||
'field character limit exceeded'
|
||||
);
|
||||
expect(fieldIgnoredMessageExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should display values when cause is "field ignored"', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
MORE_THAN_1024_CHARS
|
||||
);
|
||||
expect(testFieldValueExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('field limit exceeded', () => {
|
||||
it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const fieldLimitMessageExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause',
|
||||
'field limit exceeded'
|
||||
);
|
||||
expect(fieldLimitMessageExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should display the limit when the cause is "field limit exceeded"', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const limitExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-mappingLimit',
|
||||
'25'
|
||||
);
|
||||
expect(limitExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should warn users about the issue not present in latest backing index', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('current quality issues', () => {
|
||||
it('should display issues only from latest backing index when current issues toggle is on', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
});
|
||||
|
||||
const currentIssuesToggleState =
|
||||
await PageObjects.datasetQuality.getQualityIssueSwitchState();
|
||||
|
||||
expect(currentIssuesToggleState).to.be(false);
|
||||
|
||||
const rows =
|
||||
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
|
||||
|
||||
expect(rows.length).to.eql(3);
|
||||
|
||||
await testSubjects.click(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsOverviewDegradedFieldToggleSwitch
|
||||
);
|
||||
|
||||
const newCurrentIssuesToggleState =
|
||||
await PageObjects.datasetQuality.getQualityIssueSwitchState();
|
||||
|
||||
expect(newCurrentIssuesToggleState).to.be(true);
|
||||
|
||||
const newRows =
|
||||
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
|
||||
|
||||
expect(newRows.length).to.eql(2);
|
||||
});
|
||||
|
||||
it('should keep the toggle on when url state says so', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
showCurrentQualityIssues: true,
|
||||
});
|
||||
|
||||
const currentIssuesToggleState =
|
||||
await PageObjects.datasetQuality.getQualityIssueSwitchState();
|
||||
|
||||
expect(currentIssuesToggleState).to.be(true);
|
||||
});
|
||||
|
||||
it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
showCurrentQualityIssues: true,
|
||||
});
|
||||
|
||||
// Check value in Table
|
||||
const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
const countColumn = table['Docs count'];
|
||||
expect(await countColumn.getCellTexts()).to.eql(['5', '5']);
|
||||
|
||||
// Check value in Flyout
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const countValue = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount',
|
||||
'5'
|
||||
);
|
||||
expect(countValue).to.be(true);
|
||||
});
|
||||
|
||||
// Toggle the switch
|
||||
await testSubjects.click(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsOverviewDegradedFieldToggleSwitch
|
||||
);
|
||||
|
||||
// Check value in Table
|
||||
const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
const newCountColumn = newTable['Docs count'];
|
||||
expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']);
|
||||
|
||||
// Check value in Flyout
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const newCountValue = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount',
|
||||
'15'
|
||||
);
|
||||
expect(newCountValue).to.be(true);
|
||||
});
|
||||
});
|
||||
|
||||
it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
showCurrentQualityIssues: true,
|
||||
});
|
||||
|
||||
await testSubjects.missingOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
});
|
||||
|
||||
it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await testSubjects.click(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsOverviewDegradedFieldToggleSwitch
|
||||
);
|
||||
|
||||
await testSubjects.missingOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -15,6 +15,6 @@ export default function ({ loadTestFile }: DatasetQualityFtrProviderContext) {
|
|||
loadTestFile(require.resolve('./dataset_quality_table_filters'));
|
||||
loadTestFile(require.resolve('./dataset_quality_privileges'));
|
||||
loadTestFile(require.resolve('./dataset_quality_details'));
|
||||
loadTestFile(require.resolve('./dataset_quality_details_degraded_field_flyout'));
|
||||
loadTestFile(require.resolve('./degraded_field_flyout'));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import expect from '@kbn/expect';
|
|||
import querystring from 'querystring';
|
||||
import rison from '@kbn/rison';
|
||||
import { WebElementWrapper } from '@kbn/ftr-common-functional-ui-services';
|
||||
import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types';
|
||||
import {
|
||||
DATA_QUALITY_URL_STATE_KEY,
|
||||
datasetQualityUrlSchemaV1,
|
||||
|
|
@ -77,6 +78,7 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
|
|||
const euiSelectable = getService('selectable');
|
||||
const find = getService('find');
|
||||
const retry = getService('retry');
|
||||
const es = getService('es');
|
||||
|
||||
const selectors = {
|
||||
datasetQualityTable: '[data-test-subj="datasetQualityTable"]',
|
||||
|
|
@ -132,6 +134,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
|
|||
unifiedHistogramBreakdownSelectorSelectable: 'unifiedHistogramBreakdownSelectorSelectable',
|
||||
managementHome: 'managementHome',
|
||||
euiFlyoutCloseButton: 'euiFlyoutCloseButton',
|
||||
datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist:
|
||||
'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist',
|
||||
datasetQualityDetailsOverviewDegradedFieldToggleSwitch:
|
||||
'datasetQualityDetailsOverviewDegradedFieldToggleSwitch',
|
||||
};
|
||||
|
||||
return {
|
||||
|
|
@ -440,6 +446,27 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
|
|||
return testSubjects.click(testSubjectSelectors.euiFlyoutCloseButton);
|
||||
},
|
||||
|
||||
async setDataStreamSettings(name: string, settings: IndicesIndexSettings) {
|
||||
return es.indices.putSettings({
|
||||
index: name,
|
||||
settings,
|
||||
});
|
||||
},
|
||||
|
||||
async rolloverDataStream(name: string) {
|
||||
return es.indices.rollover({
|
||||
alias: name,
|
||||
});
|
||||
},
|
||||
|
||||
async getQualityIssueSwitchState() {
|
||||
const isSelected = await testSubjects.getAttribute(
|
||||
testSubjectSelectors.datasetQualityDetailsOverviewDegradedFieldToggleSwitch,
|
||||
'aria-checked'
|
||||
);
|
||||
return isSelected === 'true';
|
||||
},
|
||||
|
||||
async parseTable(tableWrapper: WebElementWrapper, columnNamesOrIndexes: string[]) {
|
||||
const headerElementWrappers = await tableWrapper.findAllByCssSelector('thead th, thead td');
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
DatasetQualityApiError,
|
||||
} from './common/dataset_quality_api_supertest';
|
||||
import { DatasetQualityFtrContextProvider } from './common/services';
|
||||
import { createBackingIndexNameWithoutVersion } from './utils';
|
||||
|
||||
export default function ({ getService }: DatasetQualityFtrContextProvider) {
|
||||
const datasetQualityApiClient: DatasetQualityApiClient = getService('datasetQualityApiClient');
|
||||
|
|
@ -97,16 +98,23 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) {
|
|||
expect(resp.body).eql(defaultDataStreamPrivileges);
|
||||
});
|
||||
|
||||
it('returns "createdOn" correctly', async () => {
|
||||
it('returns "createdOn" and "lastBackingIndexName" correctly', async () => {
|
||||
const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex(
|
||||
esClient,
|
||||
`${type}-${dataset}-${namespace}`
|
||||
);
|
||||
const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader);
|
||||
expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date));
|
||||
expect(resp.body.lastBackingIndexName).to.be(
|
||||
`${createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset,
|
||||
namespace,
|
||||
})}-000001`
|
||||
);
|
||||
});
|
||||
|
||||
it('returns "createdOn" correctly for rolled over dataStream', async () => {
|
||||
it('returns "createdOn" and "lastBackingIndexName" correctly for rolled over dataStream', async () => {
|
||||
await rolloverDataStream(esClient, `${type}-${dataset}-${namespace}`);
|
||||
const dataStreamSettings = await getDataStreamSettingsOfEarliestIndex(
|
||||
esClient,
|
||||
|
|
@ -114,6 +122,9 @@ export default function ({ getService }: DatasetQualityFtrContextProvider) {
|
|||
);
|
||||
const resp = await callApi(`${type}-${dataset}-${namespace}`, roleAuthc, internalReqHeader);
|
||||
expect(resp.body.createdOn).to.be(Number(dataStreamSettings?.index?.creation_date));
|
||||
expect(resp.body.lastBackingIndexName).to.be(
|
||||
`${createBackingIndexNameWithoutVersion({ type, dataset, namespace })}-000002`
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
import { Client } from '@elastic/elasticsearch';
|
||||
import { IndicesIndexSettings } from '@elastic/elasticsearch/lib/api/types';
|
||||
|
||||
export async function rolloverDataStream(es: Client, name: string) {
|
||||
return es.indices.rollover({ alias: name });
|
||||
|
|
@ -24,3 +25,35 @@ export async function getDataStreamSettingsOfEarliestIndex(es: Client, name: str
|
|||
|
||||
return matchingIndexesObj[matchingIndexes[0]].settings;
|
||||
}
|
||||
|
||||
function getCurrentDateFormatted() {
|
||||
const date = new Date();
|
||||
const year = date.getFullYear();
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getDate()).padStart(2, '0');
|
||||
|
||||
return `${year}.${month}.${day}`;
|
||||
}
|
||||
|
||||
export function createBackingIndexNameWithoutVersion({
|
||||
type,
|
||||
dataset,
|
||||
namespace = 'default',
|
||||
}: {
|
||||
type: string;
|
||||
dataset: string;
|
||||
namespace: string;
|
||||
}) {
|
||||
return `.ds-${type}-${dataset}-${namespace}-${getCurrentDateFormatted()}`;
|
||||
}
|
||||
|
||||
export async function setDataStreamSettings(
|
||||
esClient: Client,
|
||||
name: string,
|
||||
settings: IndicesIndexSettings
|
||||
) {
|
||||
return esClient.indices.putSettings({
|
||||
index: name,
|
||||
settings,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,106 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import {
|
||||
createDegradedFieldsRecord,
|
||||
datasetNames,
|
||||
defaultNamespace,
|
||||
getInitialTestLogs,
|
||||
ANOTHER_1024_CHARS,
|
||||
MORE_THAN_1024_CHARS,
|
||||
} from './data';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects([
|
||||
'common',
|
||||
'navigationalSearch',
|
||||
'observabilityLogsExplorer',
|
||||
'datasetQuality',
|
||||
'svlCommonPage',
|
||||
]);
|
||||
const testSubjects = getService('testSubjects');
|
||||
const synthtrace = getService('svlLogsSynthtraceClient');
|
||||
const retry = getService('retry');
|
||||
const to = '2024-01-01T12:00:00.000Z';
|
||||
const degradedDatasetName = datasetNames[2];
|
||||
const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`;
|
||||
|
||||
describe('Degraded fields flyout', () => {
|
||||
before(async () => {
|
||||
await synthtrace.index([
|
||||
// Ingest basic logs
|
||||
getInitialTestLogs({ to, count: 4 }),
|
||||
// Ingest Degraded Logs
|
||||
createDegradedFieldsRecord({
|
||||
to: new Date().toISOString(),
|
||||
count: 2,
|
||||
dataset: degradedDatasetName,
|
||||
}),
|
||||
]);
|
||||
await PageObjects.svlCommonPage.loginWithPrivilegedRole();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
|
||||
describe('degraded field flyout open-close', () => {
|
||||
it('should open and close the flyout when user clicks on the expand button', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field');
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should open the flyout when navigating to the page with degradedField in URL State', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('values exist', () => {
|
||||
it('should display the degraded field values', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
ANOTHER_1024_CHARS
|
||||
);
|
||||
const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
MORE_THAN_1024_CHARS
|
||||
);
|
||||
expect(cloudAvailabilityZoneValueExists).to.be(true);
|
||||
expect(cloudAvailabilityZoneValue2Exists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,417 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import expect from '@kbn/expect';
|
||||
import moment from 'moment';
|
||||
import { generateShortId, log, timerange } from '@kbn/apm-synthtrace-client';
|
||||
import {
|
||||
createDegradedFieldsRecord,
|
||||
datasetNames,
|
||||
defaultNamespace,
|
||||
getInitialTestLogs,
|
||||
ANOTHER_1024_CHARS,
|
||||
MORE_THAN_1024_CHARS,
|
||||
} from './data';
|
||||
import { FtrProviderContext } from '../../../ftr_provider_context';
|
||||
|
||||
export default function ({ getService, getPageObjects }: FtrProviderContext) {
|
||||
const PageObjects = getPageObjects([
|
||||
'common',
|
||||
'navigationalSearch',
|
||||
'observabilityLogsExplorer',
|
||||
'datasetQuality',
|
||||
'svlCommonPage',
|
||||
]);
|
||||
const testSubjects = getService('testSubjects');
|
||||
const synthtrace = getService('svlLogsSynthtraceClient');
|
||||
const retry = getService('retry');
|
||||
const to = new Date().toISOString();
|
||||
const degradedDatasetName = datasetNames[2];
|
||||
const degradedDataStreamName = `logs-${degradedDatasetName}-${defaultNamespace}`;
|
||||
|
||||
const degradedDatasetWithLimitsName = 'degraded.dataset.rca';
|
||||
const degradedDatasetWithLimitDataStreamName = `logs-${degradedDatasetWithLimitsName}-${defaultNamespace}`;
|
||||
const serviceName = 'test_service';
|
||||
const count = 5;
|
||||
|
||||
describe('Degraded fields flyout', () => {
|
||||
before(async () => {
|
||||
await synthtrace.index([
|
||||
// Ingest basic logs
|
||||
getInitialTestLogs({ to, count: 4 }),
|
||||
// Ingest Degraded Logs
|
||||
createDegradedFieldsRecord({
|
||||
to: new Date().toISOString(),
|
||||
count: 2,
|
||||
dataset: degradedDatasetName,
|
||||
}),
|
||||
]);
|
||||
await PageObjects.svlCommonPage.loginWithPrivilegedRole();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
|
||||
describe('degraded field flyout open-close', () => {
|
||||
it('should open and close the flyout when user clicks on the expand button', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.openDegradedFieldFlyout('test_field');
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should open the flyout when navigating to the page with degradedField in URL State', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('values exist', () => {
|
||||
it('should display the degraded field values', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const cloudAvailabilityZoneValueExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
ANOTHER_1024_CHARS
|
||||
);
|
||||
const cloudAvailabilityZoneValue2Exists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
MORE_THAN_1024_CHARS
|
||||
);
|
||||
expect(cloudAvailabilityZoneValueExists).to.be(true);
|
||||
expect(cloudAvailabilityZoneValue2Exists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('testing root cause for ignored fields', () => {
|
||||
before(async () => {
|
||||
// Ingest Degraded Logs with 25 fields
|
||||
await synthtrace.index([
|
||||
timerange(moment(to).subtract(count, 'minute'), moment(to))
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) => {
|
||||
return Array(1)
|
||||
.fill(0)
|
||||
.flatMap(() =>
|
||||
log
|
||||
.create()
|
||||
.dataset(degradedDatasetWithLimitsName)
|
||||
.message('a log message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.service(serviceName)
|
||||
.namespace(defaultNamespace)
|
||||
.defaults({
|
||||
'service.name': serviceName,
|
||||
'trace.id': generateShortId(),
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
// Set Limit of 25
|
||||
await PageObjects.datasetQuality.setDataStreamSettings(
|
||||
degradedDatasetWithLimitDataStreamName,
|
||||
{
|
||||
'mapping.total_fields.limit': 25,
|
||||
}
|
||||
);
|
||||
|
||||
// Ingest Degraded Logs with 26 field
|
||||
await synthtrace.index([
|
||||
timerange(moment(to).subtract(count, 'minute'), moment(to))
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) => {
|
||||
return Array(1)
|
||||
.fill(0)
|
||||
.flatMap(() =>
|
||||
log
|
||||
.create()
|
||||
.dataset(degradedDatasetWithLimitsName)
|
||||
.message('a log message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.service(serviceName)
|
||||
.namespace(defaultNamespace)
|
||||
.defaults({
|
||||
'service.name': serviceName,
|
||||
'trace.id': generateShortId(),
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
'cloud.region': 'us-east-1',
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
);
|
||||
}),
|
||||
]);
|
||||
|
||||
// Rollover Datastream to reset the limit to default which is 1000
|
||||
await PageObjects.datasetQuality.rolloverDataStream(degradedDatasetWithLimitDataStreamName);
|
||||
|
||||
// Ingest docs with 26 fields again
|
||||
await synthtrace.index([
|
||||
timerange(moment(to).subtract(count, 'minute'), moment(to))
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp) => {
|
||||
return Array(1)
|
||||
.fill(0)
|
||||
.flatMap(() =>
|
||||
log
|
||||
.create()
|
||||
.dataset(degradedDatasetWithLimitsName)
|
||||
.message('a log message')
|
||||
.logLevel(MORE_THAN_1024_CHARS)
|
||||
.service(serviceName)
|
||||
.namespace(defaultNamespace)
|
||||
.defaults({
|
||||
'log.file.path': '/my-service.log',
|
||||
'service.name': serviceName,
|
||||
'trace.id': generateShortId(),
|
||||
test_field: [MORE_THAN_1024_CHARS, 'hello world'],
|
||||
'cloud.region': 'us-east-1',
|
||||
})
|
||||
.timestamp(timestamp)
|
||||
);
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
describe('field character limit exceeded', () => {
|
||||
it('should display cause as "field ignored" when a field is ignored due to field above issue', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const fieldIgnoredMessageExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause',
|
||||
'field character limit exceeded'
|
||||
);
|
||||
expect(fieldIgnoredMessageExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should display values when cause is "field ignored"', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const testFieldValueExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-values',
|
||||
MORE_THAN_1024_CHARS
|
||||
);
|
||||
expect(testFieldValueExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
});
|
||||
|
||||
describe('field limit exceeded', () => {
|
||||
it('should display cause as "field limit exceeded" when a field is ignored due to field limit issue', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const fieldLimitMessageExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-cause',
|
||||
'field limit exceeded'
|
||||
);
|
||||
expect(fieldLimitMessageExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should display the limit when the cause is "field limit exceeded"', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const limitExists = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldValue-mappingLimit',
|
||||
'25'
|
||||
);
|
||||
expect(limitExists).to.be(true);
|
||||
});
|
||||
|
||||
await PageObjects.datasetQuality.closeFlyout();
|
||||
});
|
||||
|
||||
it('should warn users about the issue not present in latest backing index', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('current quality issues', () => {
|
||||
it('should display issues only from latest backing index when current issues toggle is on', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
});
|
||||
|
||||
const currentIssuesToggleState =
|
||||
await PageObjects.datasetQuality.getQualityIssueSwitchState();
|
||||
|
||||
expect(currentIssuesToggleState).to.be(false);
|
||||
|
||||
const rows =
|
||||
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
|
||||
|
||||
expect(rows.length).to.eql(3);
|
||||
|
||||
await testSubjects.click(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsOverviewDegradedFieldToggleSwitch
|
||||
);
|
||||
|
||||
const newCurrentIssuesToggleState =
|
||||
await PageObjects.datasetQuality.getQualityIssueSwitchState();
|
||||
|
||||
expect(newCurrentIssuesToggleState).to.be(true);
|
||||
|
||||
const newRows =
|
||||
await PageObjects.datasetQuality.getDatasetQualityDetailsDegradedFieldTableRows();
|
||||
|
||||
expect(newRows.length).to.eql(2);
|
||||
});
|
||||
|
||||
it('should keep the toggle on when url state says so', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
showCurrentQualityIssues: true,
|
||||
});
|
||||
|
||||
const currentIssuesToggleState =
|
||||
await PageObjects.datasetQuality.getQualityIssueSwitchState();
|
||||
|
||||
expect(currentIssuesToggleState).to.be(true);
|
||||
});
|
||||
|
||||
it('should display count from latest backing index when current issues toggle is on in the table and in the flyout', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'test_field',
|
||||
showCurrentQualityIssues: true,
|
||||
});
|
||||
|
||||
// Check value in Table
|
||||
const table = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
const countColumn = table['Docs count'];
|
||||
expect(await countColumn.getCellTexts()).to.eql(['5', '5']);
|
||||
|
||||
// Check value in Flyout
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const countValue = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount',
|
||||
'5'
|
||||
);
|
||||
expect(countValue).to.be(true);
|
||||
});
|
||||
|
||||
// Toggle the switch
|
||||
await testSubjects.click(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsOverviewDegradedFieldToggleSwitch
|
||||
);
|
||||
|
||||
// Check value in Table
|
||||
const newTable = await PageObjects.datasetQuality.parseDegradedFieldTable();
|
||||
const newCountColumn = newTable['Docs count'];
|
||||
expect(await newCountColumn.getCellTexts()).to.eql(['15', '15', '5']);
|
||||
|
||||
// Check value in Flyout
|
||||
await retry.tryForTime(5000, async () => {
|
||||
const newCountValue = await PageObjects.datasetQuality.doesTextExist(
|
||||
'datasetQualityDetailsDegradedFieldFlyoutFieldsList-docCount',
|
||||
'15'
|
||||
);
|
||||
expect(newCountValue).to.be(true);
|
||||
});
|
||||
});
|
||||
|
||||
it('should close the flyout if passed value in URL no more exists in latest backing index and current quality toggle is switched on', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
showCurrentQualityIssues: true,
|
||||
});
|
||||
|
||||
await testSubjects.missingOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
});
|
||||
|
||||
it('should close the flyout when current quality switch is toggled on and the flyout is already open with an old field ', async () => {
|
||||
await PageObjects.datasetQuality.navigateToDetails({
|
||||
dataStream: degradedDatasetWithLimitDataStreamName,
|
||||
expandedDegradedField: 'cloud',
|
||||
});
|
||||
|
||||
await testSubjects.existOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
|
||||
await testSubjects.click(
|
||||
PageObjects.datasetQuality.testSubjectSelectors
|
||||
.datasetQualityDetailsOverviewDegradedFieldToggleSwitch
|
||||
);
|
||||
|
||||
await testSubjects.missingOrFail(
|
||||
PageObjects.datasetQuality.testSubjectSelectors.datasetQualityDetailsDegradedFieldFlyout
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtrace.clean();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -15,6 +15,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
|
|||
loadTestFile(require.resolve('./dataset_quality_table_filters'));
|
||||
loadTestFile(require.resolve('./dataset_quality_privileges'));
|
||||
loadTestFile(require.resolve('./dataset_quality_details'));
|
||||
loadTestFile(require.resolve('./dataset_quality_details_degraded_field_flyout'));
|
||||
loadTestFile(require.resolve('./degraded_field_flyout'));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue