[Dataset quality] degradedDocs rule (#216026)

Closes https://github.com/elastic/kibana/issues/179173.

## Rule type
A dedicated stack rule type was created `datasetQuality.degradedDocs`.
<img width="1759" alt="image"
src="https://github.com/user-attachments/assets/5004a08d-6f12-4f5e-b27f-5f4db242dcf0"
/>
<img width="2318" alt="image"
src="https://github.com/user-attachments/assets/f8b2664a-f1c6-48c5-a617-c6f1b79bf0f7"
/>


This new rule is aggregated by default using `_index` and could be
further configured by the user (e.g. user can also aggregate by
`cloud.provider`).

A new rule type was needed to be created since there is no actual way to
aggregate all documents in a dataStream if we use a DataView like
`logs-*-*`. Inside datasStream documents there is no indication about
the dataStream where they belong to, instead we just have `_index` which
contains backingIndexName instead of actual index.

It's important to note, that this rule type is also visible from
`Observability > Alerts`, which is useful specially for serverless.


https://github.com/user-attachments/assets/000aee51-4895-4f4c-9484-924ace4325c5


## Role Based Access-Control (RBAC)
RBAC for dataset quality alerts is defined within dataQuality kibana
feature.
We have three privileges defined:
1. `all`: This privilege now contains a subFeature `manage_rules` that
will allow for more granularity on alerting level. It's by default
assigned to `all` but can be disabled.
2. `read`: This privilege is only related to serverless (when we don't
have yet custom roles).

https://github.com/user-attachments/assets/70ed5bde-bf45-4024-b448-228799fcaf71
3. `none`: This privilege is only relevant for stateful (in serverless
we don't have custom roles).

## 🎥 Demo
### Serverless
#### `all` privileges

https://github.com/user-attachments/assets/8dad6e30-a261-4a69-979f-6dfc2a41c888

#### `read` privileges

https://github.com/user-attachments/assets/e1cb108d-22a0-4e7f-b252-9cc12d1e9d65

### Stateful
#### `all` privileges

https://github.com/user-attachments/assets/d96f3b70-35b2-466b-aa59-a07190d24d93

#### `all` privileges with subFeature disabled

https://github.com/user-attachments/assets/808ab811-9320-43e4-b2a6-06d530a78b82

#### `none` privileges (Stateful)

https://github.com/user-attachments/assets/18f2a2d6-d825-4713-acea-0d72f451e9ab

## How to test?
1. run synthrace scenario `degraded_logs` in live mode
```
node scripts/synthtrace degraded_logs --live
```
2. Open dataset quality page (/app/management/data/data_quality)
3. Select `synth.3` dataset
(/app/management/data/data_quality/details?pageState=(dataStream:logs-synth.3-default)
4. Click on `Actions` and then select `Create rule`
5. Fill out the alert form
6. Go to `Observability > Alerts` or `Stack management > Alerts`
(/app/observability/alerts)

## Release note
Adds the Create alert rule action to dataset quality page and dataset
quality details. This allows you to generate an alert when the
percentage of degraded docs on the chart crosses a certain threshold.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Aleh Zasypkin <aleh.zasypkin@elastic.co>
Co-authored-by: Faisal Kanout <faisal.kanout@elastic.co>
This commit is contained in:
Yngrid Coello 2025-06-23 11:30:14 +02:00 committed by GitHub
parent 2c8ec79abf
commit 64df229998
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
59 changed files with 2964 additions and 119 deletions

View file

@ -0,0 +1,84 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
// ---------------------------------- WARNING ----------------------------------
// this file was generated, and should not be edited by hand
// ---------------------------------- WARNING ----------------------------------
import * as rt from 'io-ts';
import type { Either } from 'fp-ts/Either';
import { AlertSchema } from './alert_schema';
import { EcsSchema } from './ecs_schema';
const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/;
export const IsoDateString = new rt.Type<string, string, unknown>(
'IsoDateString',
rt.string.is,
(input, context): Either<rt.Errors, string> => {
if (typeof input === 'string' && ISO_DATE_PATTERN.test(input)) {
return rt.success(input);
} else {
return rt.failure(input, context);
}
},
rt.identity
);
export type IsoDateStringC = typeof IsoDateString;
export const schemaUnknown = rt.unknown;
export const schemaUnknownArray = rt.array(rt.unknown);
export const schemaString = rt.string;
export const schemaStringArray = rt.array(schemaString);
export const schemaNumber = rt.number;
export const schemaNumberArray = rt.array(schemaNumber);
export const schemaDate = rt.union([IsoDateString, schemaNumber]);
export const schemaDateArray = rt.array(schemaDate);
export const schemaDateRange = rt.partial({
gte: schemaDate,
lte: schemaDate,
});
export const schemaDateRangeArray = rt.array(schemaDateRange);
export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]);
export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber);
export const schemaBoolean = rt.boolean;
export const schemaBooleanArray = rt.array(schemaBoolean);
const schemaGeoPointCoords = rt.type({
type: schemaString,
coordinates: schemaNumberArray,
});
const schemaGeoPointString = schemaString;
const schemaGeoPointLatLon = rt.type({
lat: schemaNumber,
lon: schemaNumber,
});
const schemaGeoPointLocation = rt.type({
location: schemaNumberArray,
});
const schemaGeoPointLocationString = rt.type({
location: schemaString,
});
export const schemaGeoPoint = rt.union([
schemaGeoPointCoords,
schemaGeoPointString,
schemaGeoPointLatLon,
schemaGeoPointLocation,
schemaGeoPointLocationString,
]);
export const schemaGeoPointArray = rt.array(schemaGeoPoint);
// prettier-ignore
const DatasetQualityAlertRequired = rt.type({
});
// prettier-ignore
const DatasetQualityAlertOptional = rt.partial({
'kibana.alert.evaluation.threshold': schemaStringOrNumber,
'kibana.alert.evaluation.value': schemaString,
'kibana.alert.grouping': schemaUnknown,
'kibana.alert.reason': schemaString,
});
// prettier-ignore
export const DatasetQualityAlertSchema = rt.intersection([DatasetQualityAlertRequired, DatasetQualityAlertOptional, AlertSchema, EcsSchema]);
// prettier-ignore
export type DatasetQualityAlert = rt.TypeOf<typeof DatasetQualityAlertSchema>;

View file

@ -9,19 +9,18 @@
export interface RouterLinkProps {
href: string | undefined;
onClick: (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement, MouseEvent>) => void;
onClick: (event: React.MouseEvent<Element, MouseEvent>) => void;
}
interface GetRouterLinkPropsDeps {
href?: string;
onClick(event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>): void;
onClick(event: React.MouseEvent<Element, MouseEvent>): void;
}
const isModifiedEvent = (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) =>
const isModifiedEvent = (event: React.MouseEvent<Element, MouseEvent>) =>
!!(event.metaKey || event.altKey || event.ctrlKey || event.shiftKey);
const isLeftClickEvent = (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) =>
event.button === 0;
const isLeftClickEvent = (event: React.MouseEvent<Element, MouseEvent>) => event.button === 0;
/**
*
@ -36,7 +35,7 @@ const isLeftClickEvent = (event: React.MouseEvent<HTMLAnchorElement | HTMLButton
* manage behaviours such as leftClickEvent and event with modifiers (Ctrl, Shift, etc)
*/
export const getRouterLinkProps = ({ href, onClick }: GetRouterLinkPropsDeps): RouterLinkProps => {
const guardedClickHandler = (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) => {
const guardedClickHandler = (event: React.MouseEvent<Element, MouseEvent>) => {
if (event.defaultPrevented) {
return;
}

View file

@ -16,3 +16,4 @@ export * from './src/alerts_as_data_status';
export * from './src/alerts_as_data_cases';
export * from './src/routes/stack_rule_paths';
export * from './src/rule_types';
export * from './src/rule_constants';

View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export const ALERT_KEY_JOINER = ',';

View file

@ -11,6 +11,8 @@ export const STACK_ALERTS_FEATURE_ID = 'stackAlerts';
export const ES_QUERY_ID = '.es-query';
export const ML_ANOMALY_DETECTION_RULE_TYPE_ID = 'xpack.ml.anomaly_detection_alert';
export const DEGRADED_DOCS_RULE_TYPE_ID = 'datasetQuality.degradedDocs';
/**
* These rule types are not the only stack rules. There are more.
* The variable holds all stack rule types that support multiple
@ -19,4 +21,5 @@ export const ML_ANOMALY_DETECTION_RULE_TYPE_ID = 'xpack.ml.anomaly_detection_ale
export const STACK_RULE_TYPE_IDS_SUPPORTED_BY_OBSERVABILITY = [
ES_QUERY_ID,
ML_ANOMALY_DETECTION_RULE_TYPE_ID,
DEGRADED_DOCS_RULE_TYPE_ID,
];

View file

@ -0,0 +1,12 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export { degradedDocsParamsSchema } from './latest';
export type { DegradedDocsRuleParams } from './latest';

View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export * from './v1';

View file

@ -0,0 +1,30 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { COMPARATORS } from '@kbn/alerting-comparators';
import type { TypeOf } from '@kbn/config-schema';
import { schema } from '@kbn/config-schema';
import { oneOfLiterals } from '../common/utils';
const comparators = Object.values(COMPARATORS);
const searchConfigSchema = schema.object({
index: schema.string(),
});
export const degradedDocsParamsSchema = schema.object({
timeUnit: schema.string(),
timeSize: schema.number(),
threshold: schema.arrayOf(schema.number()),
comparator: oneOfLiterals(comparators),
groupBy: schema.maybe(schema.arrayOf(schema.string())),
searchConfiguration: searchConfigSchema,
});
export type DegradedDocsRuleParams = TypeOf<typeof degradedDocsParamsSchema>;

View file

@ -473,6 +473,46 @@ Object {
}
`;
exports[`Alert as data fields checks detect AAD fields changes for: datasetQuality.degradedDocs 1`] = `
Object {
"dynamicTemplates": Array [
Object {
"strings_as_keywords": Object {
"mapping": Object {
"ignore_above": 1024,
"type": "keyword",
},
"match_mapping_type": "string",
"path_match": "kibana.alert.grouping.*",
},
},
],
"fieldMap": Object {
"kibana.alert.evaluation.threshold": Object {
"required": false,
"scaling_factor": 100,
"type": "scaled_float",
},
"kibana.alert.evaluation.value": Object {
"array": false,
"required": false,
"type": "keyword",
},
"kibana.alert.grouping": Object {
"array": false,
"dynamic": true,
"required": false,
"type": "object",
},
"kibana.alert.reason": Object {
"array": false,
"required": false,
"type": "keyword",
},
},
}
`;
exports[`Alert as data fields checks detect AAD fields changes for: logs.alert.document.count 1`] = `
Object {
"fieldMap": Object {

View file

@ -64,6 +64,7 @@ const ruleTypes: string[] = [
'siem.thresholdRule',
'siem.newTermsRule',
'siem.notifications',
'datasetQuality.degradedDocs',
];
describe('Alert as data fields checks', () => {

View file

@ -9,7 +9,7 @@ import { setStateToKbnUrl } from '@kbn/kibana-utils-plugin/common';
import { ManagementAppLocatorParams } from '@kbn/management-plugin/common/locator';
import { LocatorPublic } from '@kbn/share-plugin/common';
import { DataQualityDetailsLocatorParams } from '@kbn/deeplinks-observability';
import { datasetQualityDetailsUrlSchemaV1, DATA_QUALITY_URL_STATE_KEY } from '../url_schema';
import { datasetQualityDetailsUrlSchemaV2, DATA_QUALITY_URL_STATE_KEY } from '../url_schema';
import { deepCompactObject } from '../utils/deep_compact_object';
interface LocatorPathConstructionParams {
@ -23,9 +23,9 @@ export const constructDatasetQualityDetailsLocatorPath = async (
) => {
const { locatorParams, useHash, managementLocator } = params;
const pageState = datasetQualityDetailsUrlSchemaV1.urlSchemaRT.encode(
const pageState = datasetQualityDetailsUrlSchemaV2.urlSchemaRT.encode(
deepCompactObject({
v: 1,
v: 2,
...locatorParams,
})
);

View file

@ -10,15 +10,74 @@ import {
KibanaFeatureConfig,
KibanaFeatureScope,
ElasticsearchFeatureConfig,
SubFeaturePrivilegeGroupConfig,
SubFeaturePrivilegeGroupType,
} from '@kbn/features-plugin/common';
import { AlertConsumers, DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
import { i18n } from '@kbn/i18n';
import { PLUGIN_FEATURE_ID, PLUGIN_ID, PLUGIN_NAME } from '../common';
const degradedDocsAlertingFeatures = {
ruleTypeId: DEGRADED_DOCS_RULE_TYPE_ID,
consumers: [AlertConsumers.ALERTS],
};
const canManageRules: SubFeaturePrivilegeGroupConfig = {
groupType: 'independent' as SubFeaturePrivilegeGroupType,
privileges: [
{
id: 'manage_rules',
name: i18n.translate('xpack.dataQuality.features.canManageRules', {
defaultMessage: 'Manage rules',
}),
includeIn: 'all',
alerting: {
rule: {
all: [degradedDocsAlertingFeatures],
},
},
savedObject: {
all: [],
read: [],
},
ui: ['alerting:save'],
},
],
};
const canManageAlerts: SubFeaturePrivilegeGroupConfig = {
groupType: 'independent' as SubFeaturePrivilegeGroupType,
privileges: [
{
id: 'manage_alerts',
name: i18n.translate('xpack.dataQuality.features.canManageAlerts', {
defaultMessage: 'Manage alerts',
}),
includeIn: 'all',
alerting: {
alert: {
all: [degradedDocsAlertingFeatures],
},
},
savedObject: {
all: [],
read: [],
},
ui: ['alerting:save'],
},
],
};
export const KIBANA_FEATURE: KibanaFeatureConfig = {
id: PLUGIN_FEATURE_ID,
name: PLUGIN_NAME,
category: DEFAULT_APP_CATEGORIES.management,
scope: [KibanaFeatureScope.Spaces, KibanaFeatureScope.Security],
app: [PLUGIN_ID],
alerting: [degradedDocsAlertingFeatures],
management: {
insightsAndAlerting: ['triggersActions'],
},
privileges: {
all: {
app: [PLUGIN_ID],
@ -27,6 +86,17 @@ export const KIBANA_FEATURE: KibanaFeatureConfig = {
read: [],
},
ui: ['show'],
alerting: {
rule: {
read: [degradedDocsAlertingFeatures],
},
alert: {
all: [degradedDocsAlertingFeatures],
},
},
management: {
insightsAndAlerting: ['triggersActions'],
},
},
read: {
disabled: true,
@ -35,8 +105,36 @@ export const KIBANA_FEATURE: KibanaFeatureConfig = {
read: [],
},
ui: ['show'],
alerting: {
rule: {
read: [degradedDocsAlertingFeatures],
},
alert: {
read: [degradedDocsAlertingFeatures],
},
},
},
},
subFeatures: [
{
name: i18n.translate('xpack.dataQuality.features.app.manageRules', {
defaultMessage: 'Manage rules',
}),
description: i18n.translate('xpack.dataQuality.features.app.manageRulesDescription', {
defaultMessage: 'This feature enables users to manage dataset quality rules.',
}),
privilegeGroups: [canManageRules],
},
{
name: i18n.translate('xpack.dataQuality.features.app.manageAlerts', {
defaultMessage: 'Manage alerts',
}),
description: i18n.translate('xpack.dataQuality.features.app.manageAlertsDescription', {
defaultMessage: 'This feature enables users to manage dataset quality alerts.',
}),
privilegeGroups: [canManageAlerts],
},
],
};
export const ELASTICSEARCH_FEATURE: ElasticsearchFeatureConfig = {

View file

@ -7,13 +7,37 @@
import { CoreSetup, Plugin } from '@kbn/core/server';
import { ManagementAppLocatorParams } from '@kbn/management-plugin/common/locator';
import { MANAGEMENT_APP_LOCATOR } from '@kbn/deeplinks-management/constants';
import { Dependencies } from './types';
import { ELASTICSEARCH_FEATURE, KIBANA_FEATURE } from './features';
import {
DatasetQualityDetailsLocatorDefinition,
DatasetQualityLocatorDefinition,
} from '../common/locators';
export class DataQualityPlugin implements Plugin<void, void, any, any> {
public setup(_coreSetup: CoreSetup, { features }: Dependencies) {
public setup(_coreSetup: CoreSetup, { features, share }: Dependencies) {
features.registerKibanaFeature(KIBANA_FEATURE);
features.registerElasticsearchFeature(ELASTICSEARCH_FEATURE);
const managementLocator =
share.url.locators.get<ManagementAppLocatorParams>(MANAGEMENT_APP_LOCATOR);
if (managementLocator) {
share.url.locators.create(
new DatasetQualityLocatorDefinition({
useHash: false,
managementLocator,
})
);
share.url.locators.create(
new DatasetQualityDetailsLocatorDefinition({
useHash: false,
managementLocator,
})
);
}
}
public start() {}

View file

@ -6,7 +6,11 @@
*/
import { FeaturesPluginSetup } from '@kbn/features-plugin/server';
import type { ManagementSetup } from '@kbn/management-plugin/public/types';
import { SharePluginSetup } from '@kbn/share-plugin/server';
export interface Dependencies {
features: FeaturesPluginSetup;
management: ManagementSetup;
share: SharePluginSetup;
}

View file

@ -29,6 +29,7 @@
"@kbn/deeplinks-observability",
"@kbn/ebt-tools",
"@kbn/core-application-common",
"@kbn/rule-data-utils",
],
"exclude": ["target/**/*"]
}

View file

@ -275,3 +275,20 @@ export const getNonAggregatableDatasetsRt = rt.exact(
);
export type NonAggregatableDatasets = rt.TypeOf<typeof getNonAggregatableDatasetsRt>;
export const getPreviewChartResponseRt = rt.type({
series: rt.array(
rt.type({
name: rt.string,
data: rt.array(
rt.type({
x: rt.number,
y: rt.union([rt.number, rt.null]),
})
),
})
),
totalGroups: rt.number,
});
export type PreviewChartResponse = rt.TypeOf<typeof getPreviewChartResponseRt>;

View file

@ -715,3 +715,7 @@ export const readLess = i18n.translate(
defaultMessage: 'Read less',
}
);
export const createAlertText = i18n.translate('xpack.datasetQuality.createAlert', {
defaultMessage: 'Create rule',
});

View file

@ -16,25 +16,30 @@
"datasetQuality"
],
"requiredPlugins": [
"controls",
"data",
"dataViews",
"dataViewEditor",
"embeddable",
"fieldFormats",
"fieldsMetadata",
"fleet",
"kibanaReact",
"kibanaUtils",
"controls",
"embeddable",
"share",
"fleet",
"fieldFormats",
"dataViews",
"lens",
"fieldsMetadata",
"share",
"taskManager",
"usageCollection"
"triggersActionsUi",
],
"optionalPlugins": [
"telemetry"
"alerting",
"telemetry",
"usageCollection",
],
"requiredBundles": [
"discover"
"charts",
"discover",
"stackAlerts",
],
"extraPublicDirs": [
"common"

View file

@ -0,0 +1,63 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RuleFormFlyout } from '@kbn/response-ops-rule-form/flyout';
import React, { useCallback, useEffect, useState } from 'react';
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
import { useKibanaContextForPlugin } from '../utils/use_kibana';
interface Props {
addFlyoutVisible: boolean;
setAddFlyoutVisibility: React.Dispatch<React.SetStateAction<boolean>>;
dataStream?: string;
}
export function AlertFlyout(props: Props) {
const { addFlyoutVisible, setAddFlyoutVisibility, dataStream } = props;
const {
services: {
triggersActionsUi: { actionTypeRegistry, ruleTypeRegistry },
...services
},
} = useKibanaContextForPlugin();
const [initialValues, setInitialValues] = useState<{ searchConfiguration?: { index: string } }>(
{}
);
useEffect(() => {
if (dataStream) {
setInitialValues({ searchConfiguration: { index: dataStream } });
}
}, [dataStream]);
const onCloseAddFlyout = useCallback(
() => setAddFlyoutVisibility(false),
[setAddFlyoutVisibility]
);
return (
<>
{addFlyoutVisible && (
<RuleFormFlyout
plugins={{
...services,
ruleTypeRegistry,
actionTypeRegistry,
}}
onCancel={onCloseAddFlyout}
onSubmit={onCloseAddFlyout}
ruleTypeId={DEGRADED_DOCS_RULE_TYPE_ID}
initialValues={{
params: initialValues,
}}
/>
)}
</>
);
}

View file

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { Capabilities } from '@kbn/core/public';
import type { PluginStartContract as AlertingPublicStart } from '@kbn/alerting-plugin/public/plugin';
export const getAlertingCapabilities = (
alerting: AlertingPublicStart | undefined,
capabilities: Capabilities
) => {
const canSaveAlerts = !!capabilities.dataQuality['alerting:save'];
const isAlertingPluginEnabled = !!alerting;
const isAlertingAvailable = isAlertingPluginEnabled && canSaveAlerts;
return {
isAlertingAvailable,
};
};

View file

@ -5,18 +5,30 @@
* 2.0.
*/
import React, { useMemo } from 'react';
import { EuiBetaBadge, EuiLink, EuiPageHeader, EuiCode } from '@elastic/eui';
import { EuiBetaBadge, EuiButton, EuiCode, EuiLink, EuiPageHeader } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import { DEFAULT_DATASET_TYPE, KNOWN_TYPES } from '../../../common/constants';
import { datasetQualityAppTitle } from '../../../common/translations';
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
import { default as React, useMemo, useState } from 'react';
import { KNOWN_TYPES } from '../../../common/constants';
import { createAlertText, datasetQualityAppTitle } from '../../../common/translations';
import { AlertFlyout } from '../../alerts/alert_flyout';
import { getAlertingCapabilities } from '../../alerts/get_alerting_capabilities';
import { useKibanaContextForPlugin } from '../../utils';
import { DEFAULT_DATASET_TYPE } from '../../../common/constants';
import { useDatasetQualityFilters } from '../../hooks/use_dataset_quality_filters';
// Allow for lazy loading
// eslint-disable-next-line import/no-default-export
export default function Header() {
const {
services: { application, alerting },
} = useKibanaContextForPlugin();
const { capabilities } = application;
const [ruleType, setRuleType] = useState<typeof DEGRADED_DOCS_RULE_TYPE_ID | null>(null);
const { isAlertingAvailable } = getAlertingCapabilities(alerting, capabilities);
const { isDatasetQualityAllSignalsAvailable } = useDatasetQualityFilters();
const validTypes = useMemo(
() => (isDatasetQualityAllSignalsAvailable ? KNOWN_TYPES : [DEFAULT_DATASET_TYPE]),
@ -66,6 +78,31 @@ export default function Header() {
}}
/>
}
rightSideItems={
isAlertingAvailable
? [
<>
<EuiButton
data-test-subj="datasetQualityDetailsHeaderButton"
onClick={() => {
setRuleType(DEGRADED_DOCS_RULE_TYPE_ID);
}}
iconType="bell"
>
{createAlertText}
</EuiButton>
<AlertFlyout
addFlyoutVisible={!!ruleType}
setAddFlyoutVisibility={(visible) => {
if (!visible) {
setRuleType(null);
}
}}
/>
</>,
]
: undefined
}
/>
);
}

View file

@ -7,8 +7,10 @@
import {
EuiButton,
EuiContextMenu,
EuiFlexGroup,
EuiFlexItem,
EuiPopover,
EuiSkeletonTitle,
EuiTextColor,
EuiTitle,
@ -16,20 +18,30 @@ import {
useEuiTheme,
} from '@elastic/eui';
import { css } from '@emotion/react';
import React from 'react';
import { openInDiscoverText } from '../../../common/translations';
import { i18n } from '@kbn/i18n';
import React, { useState } from 'react';
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
import { createAlertText, openInDiscoverText } from '../../../common/translations';
import { AlertFlyout } from '../../alerts/alert_flyout';
import { getAlertingCapabilities } from '../../alerts/get_alerting_capabilities';
import {
useDatasetDetailsRedirectLinkTelemetry,
useDatasetDetailsTelemetry,
useDatasetQualityDetailsState,
useRedirectLink,
} from '../../hooks';
import { useKibanaContextForPlugin } from '../../utils';
import { IntegrationIcon } from '../common';
export function Header() {
const { datasetDetails, timeRange, integrationDetails, loadingState } =
useDatasetQualityDetailsState();
const {
services: { application, alerting },
} = useKibanaContextForPlugin();
const { capabilities } = application;
const { navigationSources } = useDatasetDetailsTelemetry();
const { rawName, name: title } = datasetDetails;
@ -44,9 +56,65 @@ export function Header() {
sendTelemetry,
});
const { isAlertingAvailable } = getAlertingCapabilities(alerting, capabilities);
const [showPopover, setShowPopover] = useState<boolean>(false);
const [ruleType, setRuleType] = useState<typeof DEGRADED_DOCS_RULE_TYPE_ID | null>(null);
const pageTitle =
integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ?? title;
const createMenuItems = [
{
name: createAlertText,
icon: 'bell',
onClick: () => {
setShowPopover(false);
setRuleType(DEGRADED_DOCS_RULE_TYPE_ID);
},
'data-test-subj': `createAlert`,
},
{
name: openInDiscoverText,
icon: 'discoverApp',
...redirectLinkProps.linkProps,
'data-test-subj': `openInDiscover`,
},
];
const titleActionButtons = [
<EuiPopover
key="actionsPopover"
isOpen={showPopover}
closePopover={() => setShowPopover(false)}
button={
<EuiButton
iconSide="right"
iconType="arrowDown"
data-test-subj="datasetQualityDetailsActionsDropdown"
key="actionsDropdown"
onClick={() => setShowPopover((prev) => !prev)}
>
{i18n.translate('xpack.datasetQuality.ActionsLabel', {
defaultMessage: 'Actions',
})}
</EuiButton>
}
panelPaddingSize="none"
repositionOnScroll
>
<EuiContextMenu
initialPanelId={0}
data-test-subj="autoFollowPatternActionContextMenu"
panels={[
{
id: 0,
items: createMenuItems,
},
]}
/>
</EuiPopover>,
];
return !loadingState.integrationDetailsLoaded ? (
<EuiSkeletonTitle
size="s"
@ -85,16 +153,29 @@ export function Header() {
justifyContent="flexEnd"
alignItems="center"
>
<EuiButton
data-test-subj="datasetQualityDetailsHeaderButton"
size="s"
{...redirectLinkProps.linkProps}
iconType="discoverApp"
>
{openInDiscoverText}
</EuiButton>
{isAlertingAvailable ? (
titleActionButtons
) : (
<EuiButton
data-test-subj="datasetQualityDetailsHeaderButton"
size="s"
{...redirectLinkProps.linkProps}
iconType="discoverApp"
>
{openInDiscoverText}
</EuiButton>
)}
</EuiFlexGroup>
</EuiFlexItem>
<AlertFlyout
dataStream={rawName}
addFlyoutVisible={!!ruleType}
setAddFlyoutVisibility={(visible) => {
if (!visible) {
setRuleType(null);
}
}}
/>
</EuiFlexGroup>
);
}

View file

@ -11,8 +11,9 @@ import { createDatasetQuality } from './components/dataset_quality';
import { createDatasetQualityDetails } from './components/dataset_quality_details';
import { createDatasetQualityControllerLazyFactory } from './controller/dataset_quality/lazy_create_controller';
import { createDatasetQualityDetailsControllerLazyFactory } from './controller/dataset_quality_details/lazy_create_controller';
import { DataStreamsStatsService } from './services/data_streams_stats';
import { registerRuleTypes } from './rule_types';
import { DataStreamDetailsService } from './services/data_stream_details';
import { DataStreamsStatsService } from './services/data_streams_stats';
import {
DatasetQualityPluginSetup,
DatasetQualityPluginStart,
@ -25,9 +26,13 @@ export class DatasetQualityPlugin
{
private telemetry = new TelemetryService();
public setup(core: CoreSetup, _plugins: DatasetQualitySetupDeps) {
public setup(core: CoreSetup, plugins: DatasetQualitySetupDeps) {
this.telemetry.setup({ analytics: core.analytics });
registerRuleTypes({
ruleTypeRegistry: plugins.triggersActionsUi.ruleTypeRegistry,
});
return {};
}

View file

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { lazy } from 'react';
import { i18n } from '@kbn/i18n';
import { RuleTypeModel } from '@kbn/triggers-actions-ui-plugin/public';
import { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
import { validate } from './rule_form/validate';
export function getRuleType(): RuleTypeModel<DegradedDocsRuleParams> {
return {
id: DEGRADED_DOCS_RULE_TYPE_ID,
description: i18n.translate('xpack.datasetQuality.alert.degradedDocs.descriptionText', {
defaultMessage: 'Alert when degraded docs percentage exceeds a threshold.',
}),
iconClass: 'bell',
documentationUrl: null,
ruleParamsExpression: lazy(() => import('./rule_form')),
validate,
requiresAppContext: false,
};
}

View file

@ -0,0 +1,176 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { EuiFlexGroup, EuiIcon, EuiLoadingChart, EuiText, EuiToolTip } from '@elastic/eui';
import numeral from '@elastic/numeral';
import { IconChartBar } from '@kbn/chart-icons';
import { EmptyPlaceholder } from '@kbn/charts-plugin/public';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import type { FC, PropsWithChildren } from 'react';
import React from 'react';
export type Maybe<T> = T | null | undefined;
function isFiniteNumber(value: any): value is number {
return isFinite(value);
}
export function asPercent(
numerator: Maybe<number>,
denominator: number | undefined,
fallbackResult = 'N/A'
) {
if (!denominator || !isFiniteNumber(numerator)) {
return fallbackResult;
}
const decimal = numerator / denominator;
if (Math.abs(decimal) >= 0.1 || decimal === 0) {
return numeral(decimal).format('0.000%');
}
return numeral(decimal).format('0.000%');
}
export const TIME_LABELS = {
s: i18n.translate('xpack.datasetQuality.alerts.timeLabels.seconds', {
defaultMessage: 'seconds',
}),
m: i18n.translate('xpack.datasetQuality.alerts.timeLabels.minutes', {
defaultMessage: 'minutes',
}),
h: i18n.translate('xpack.datasetQuality.alerts.timeLabels.hours', {
defaultMessage: 'hours',
}),
d: i18n.translate('xpack.datasetQuality.alerts.timeLabels.days', {
defaultMessage: 'days',
}),
};
export const getDomain = (series: Array<{ name?: string; data: any[] }>) => {
const xValues = series.flatMap((item) => item.data.map((d) => d.x));
const yValues = series.flatMap((item) => item.data.map((d) => d.y || 0));
return {
xMax: Math.max(...xValues),
xMin: Math.min(...xValues),
yMax: Math.max(...yValues),
yMin: Math.min(...yValues),
};
};
const EmptyContainer: FC<PropsWithChildren<unknown>> = ({ children }) => (
<div
style={{
width: '100%',
height: 150,
display: 'flex',
justifyContent: 'center',
alignItems: 'center',
}}
>
{children}
</div>
);
export function NoDataState() {
return (
<EmptyContainer>
<EmptyPlaceholder
icon={IconChartBar}
message={
<FormattedMessage
id="xpack.datasetQuality.chartPreview.noDataMessage"
defaultMessage="No results found"
/>
}
/>
</EmptyContainer>
);
}
export function LoadingState() {
return (
<EmptyContainer>
<EuiText color="subdued" data-test-subj="loadingData">
<EuiLoadingChart size="m" />
</EuiText>
</EmptyContainer>
);
}
export function ErrorState() {
return (
<EmptyContainer>
<EuiText color="subdued" data-test-subj="chartErrorState">
<FormattedMessage
id="xpack.datasetQuality.alerts.charts.errorMessage"
defaultMessage="Uh oh, something went wrong"
/>
</EuiText>
</EmptyContainer>
);
}
interface PreviewChartLabel {
field: string;
timeSize: number;
timeUnit: string;
series: number;
totalGroups: number;
}
export function TimeLabelForData({
field,
timeSize,
timeUnit,
series,
totalGroups,
}: PreviewChartLabel) {
const totalGroupsTooltip = i18n.translate(
'xpack.datasetQuality.chartPreview.TimeLabelForData.totalGroupsTooltip',
{
defaultMessage: 'Showing {series} out of {totalGroups} groups',
values: {
series,
totalGroups,
},
}
);
const xAxisInfo = (
<EuiText size="xs">
<strong>
<FormattedMessage
id="xpack.datasetQuality.chartPreview.timeLabelForData.xAxis"
defaultMessage="{field} per {timeSize} {timeUnit}"
values={{
field,
timeSize,
timeUnit: TIME_LABELS[timeUnit as keyof typeof TIME_LABELS],
}}
/>
</strong>
</EuiText>
);
return (
<div style={{ textAlign: 'center' }}>
{totalGroups > series ? (
<EuiToolTip content={totalGroupsTooltip} position="top">
<EuiFlexGroup gutterSize="xs">
{xAxisInfo}
<EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
</EuiFlexGroup>
</EuiToolTip>
) : (
xAxisInfo
)}
</div>
);
}

View file

@ -0,0 +1,239 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { RectAnnotationDatum, TickFormatter } from '@elastic/charts';
import {
AnnotationDomainType,
Axis,
BarSeries,
Chart,
LineAnnotation,
Position,
RectAnnotation,
ScaleType,
Settings,
Tooltip,
niceTimeFormatter,
} from '@elastic/charts';
import { EuiSpacer, useEuiTheme } from '@elastic/eui';
import { COMPARATORS } from '@kbn/alerting-comparators';
import { useElasticChartsTheme } from '@kbn/charts-theme';
import type { IUiSettingsClient } from '@kbn/core/public';
import { UI_SETTINGS } from '@kbn/data-plugin/public';
import { i18n } from '@kbn/i18n';
import moment from 'moment';
import React, { useEffect, useMemo, useState } from 'react';
import { TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
import { Maybe, TimeLabelForData, getDomain } from './chart_preview_helper';
function getTimeZone(uiSettings?: IUiSettingsClient) {
const kibanaTimeZone = uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ);
if (!kibanaTimeZone || kibanaTimeZone === 'Browser') {
return moment.tz.guess();
}
return kibanaTimeZone;
}
interface ChartPreviewProps {
yTickFormat?: TickFormatter;
threshold: number[];
comparator: COMPARATORS;
uiSettings?: IUiSettingsClient;
series: Array<{ name: string; data: Array<{ x: number; y: Maybe<number> }> }>;
timeSize?: number;
timeUnit?: TimeUnitChar;
totalGroups: number;
}
export function ChartPreview({
yTickFormat,
threshold,
comparator,
uiSettings,
series,
timeSize = 5,
timeUnit = 'm',
totalGroups,
}: ChartPreviewProps) {
const baseTheme = useElasticChartsTheme();
const DEFAULT_DATE_FORMAT = 'Y-MM-DD HH:mm:ss';
const barSeries = useMemo(() => {
return series.flatMap((serie) =>
serie.data.map((point) => ({ ...point, groupBy: serie.name }))
);
}, [series]);
const timeZone = getTimeZone(uiSettings);
const chartSize = 120;
const { yMax, xMin, xMax } = getDomain(series);
const chartDomain = {
max: Math.max(yMax === 0 ? 1 : yMax, Math.max(...threshold)) * 1.1, // Add 10% headroom.
min: 0,
};
const dateFormatter = useMemo(() => niceTimeFormatter([xMin, xMax]), [xMin, xMax]);
const theme = useEuiTheme();
const thresholdOpacity = 0.1;
const [sortedThreshold, setSortedThreshold] = useState(threshold);
useEffect(() => {
setSortedThreshold([...threshold].sort((a, b) => a - b));
}, [threshold]);
const style = {
fill: theme.euiTheme.colors.danger,
line: {
strokeWidth: 1,
stroke: theme.euiTheme.colors.danger,
opacity: 1,
},
opacity: thresholdOpacity,
};
const rectThresholdToMax = (value: number): RectAnnotationDatum[] => [
{
coordinates: {
x0: xMin,
x1: xMax,
y0: value,
y1: chartDomain.max,
},
},
];
const rectThresholdToMin = (value: number): RectAnnotationDatum[] => [
{
coordinates: {
x0: xMin,
x1: xMax,
y0: chartDomain.min,
y1: value,
},
},
];
const rectThresholdToThreshold: RectAnnotationDatum[] = [
{
coordinates: {
x0: xMin,
x1: xMax,
y0: sortedThreshold[0],
y1: sortedThreshold[1],
},
},
];
return (
<>
<EuiSpacer size="m" />
<Chart
size={{
height: chartSize,
}}
data-test-subj="ChartPreview"
>
<Tooltip
headerFormatter={({ value }) => {
const dateFormat =
(uiSettings && uiSettings.get(UI_SETTINGS.DATE_FORMAT)) || DEFAULT_DATE_FORMAT;
return moment(value).format(dateFormat);
}}
/>
<Settings showLegend={false} locale={i18n.getLocale()} baseTheme={baseTheme} />
<LineAnnotation
dataValues={[{ dataValue: sortedThreshold[0] }]}
domainType={AnnotationDomainType.YDomain}
id="chart_preview_line_annotation"
markerPosition="left"
style={style}
/>
{comparator !== COMPARATORS.BETWEEN && (
<RectAnnotation
dataValues={
comparator === COMPARATORS.GREATER_THAN ||
comparator === COMPARATORS.GREATER_THAN_OR_EQUALS
? rectThresholdToMax(sortedThreshold[0])
: rectThresholdToMin(sortedThreshold[0])
}
hideTooltips={true}
id="chart_preview_rect_annotation"
style={style}
/>
)}
{threshold.length > 1 &&
[COMPARATORS.NOT_BETWEEN, COMPARATORS.BETWEEN].includes(comparator) && (
<>
<LineAnnotation
dataValues={[{ dataValue: sortedThreshold[1] }]}
domainType={AnnotationDomainType.YDomain}
id="chart_preview_line_annotation_2"
markerPosition="left"
style={style}
/>
<RectAnnotation
dataValues={
comparator === COMPARATORS.NOT_BETWEEN
? rectThresholdToMax(sortedThreshold[1])
: rectThresholdToThreshold
}
hideTooltips={true}
id="chart_preview_rect_annotation_2"
style={style}
/>
</>
)}
<Axis
id="chart_preview_x_axis"
position={Position.Bottom}
showOverlappingTicks={true}
tickFormat={dateFormatter}
/>
<Axis
id="chart_preview_y_axis"
position={Position.Left}
tickFormat={yTickFormat}
ticks={5}
domain={chartDomain}
/>
<BarSeries
id="chart_preview_bar_series"
xScaleType={ScaleType.Time}
yScaleType={ScaleType.Linear}
xAccessor="x"
yAccessors={['y']}
splitSeriesAccessors={['groupBy']}
data={barSeries}
barSeriesStyle={{
rectBorder: {
strokeWidth: 1,
visible: true,
},
rect: {
opacity: 1,
},
}}
timeZone={timeZone}
/>
</Chart>
{series.length > 0 && (
<TimeLabelForData
field={'@timestamp'}
timeSize={timeSize}
timeUnit={timeUnit}
series={series.length}
totalGroups={totalGroups}
/>
)}
</>
);
}

View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RuleForm } from './rule_form';
// eslint-disable-next-line import/no-default-export
export default RuleForm;

View file

@ -0,0 +1,99 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { COMPARATORS } from '@kbn/alerting-comparators';
import { DataView } from '@kbn/data-views-plugin/common';
import { TimeRange } from '@kbn/es-query';
import { TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
import rison from '@kbn/rison';
import React from 'react';
import { useAbortableAsync } from '@kbn/react-hooks';
import { PreviewChartResponse } from '../../../../common/api_types';
import { useKibanaContextForPlugin } from '../../../utils';
import { ChartPreview } from './chart_preview';
import {
ErrorState,
LoadingState,
NoDataState,
asPercent,
} from './chart_preview/chart_preview_helper';
interface ChartOptions {
interval?: string;
}
export interface RuleConditionChartProps {
threshold: number[];
comparator: COMPARATORS;
timeSize?: number;
timeUnit?: TimeUnitChar;
dataView?: DataView;
groupBy?: string | string[];
timeRange: TimeRange;
chartOptions?: ChartOptions;
}
export function RuleConditionChart({
threshold,
comparator,
timeSize,
timeUnit,
dataView,
groupBy,
timeRange,
chartOptions: { interval } = {},
}: RuleConditionChartProps) {
const {
services: { http, uiSettings },
} = useKibanaContextForPlugin();
const { loading, value, error } = useAbortableAsync(
async ({ signal }) => {
if (dataView && timeRange.from && timeRange.to) {
return http.get<PreviewChartResponse>(
'/internal/dataset_quality/rule_types/degraded_docs/chart_preview',
{
query: {
index: dataView?.getIndexPattern(),
start: timeRange?.from,
end: timeRange?.to,
interval: interval || `${timeSize}${timeUnit}`,
groupBy: rison.encodeArray(Array.isArray(groupBy) ? groupBy : [groupBy]),
},
}
);
}
},
[http, dataView, groupBy, interval, timeRange?.from, timeRange?.to, timeSize, timeUnit]
);
return (
<div>
{loading ? (
<LoadingState />
) : !value?.series || value.series.length === 0 ? (
<NoDataState />
) : error ? (
<ErrorState />
) : (
<ChartPreview
series={value.series}
threshold={threshold}
uiSettings={uiSettings}
comparator={comparator}
yTickFormat={(d: number | null) => asPercent(d, 100)}
timeSize={timeSize}
timeUnit={timeUnit}
totalGroups={value.totalGroups}
/>
)}
</div>
);
}
// eslint-disable-next-line import/no-default-export
export default RuleConditionChart;

View file

@ -0,0 +1,316 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
EuiComboBox,
EuiExpression,
EuiFormRow,
EuiSpacer,
EuiTitle,
EuiFormErrorText,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
import { DataViewSelectPopover } from '@kbn/stack-alerts-plugin/public';
import {
ForLastExpression,
type RuleTypeParamsExpressionProps,
} from '@kbn/triggers-actions-ui-plugin/public';
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { DataView } from '@kbn/data-views-plugin/common';
import { ThresholdExpression } from '@kbn/triggers-actions-ui-plugin/public';
import { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
import { COMPARATORS } from '@kbn/alerting-comparators';
import { i18n } from '@kbn/i18n';
import { isArray } from 'lodash';
import { DataViewBase, DataViewFieldBase } from '@kbn/es-query';
import { TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
import { INDEX } from '../../../../common/es_fields';
import { useKibanaContextForPlugin } from '../../../utils';
import { RuleConditionChart } from './rule_condition_chart';
const degradedDocsLabel = i18n.translate('xpack.datasetQuality.rule.degradedDocsLabel', {
defaultMessage: 'degraded docs',
});
export const defaultRuleParams: Partial<DegradedDocsRuleParams> = {
comparator: COMPARATORS.GREATER_THAN,
threshold: [3],
timeSize: 5,
timeUnit: 'm',
groupBy: [INDEX],
};
export type DataStreamGroupByFields = Array<DataViewFieldBase & { aggregatable: boolean }>;
export const RuleForm: React.FunctionComponent<
RuleTypeParamsExpressionProps<DegradedDocsRuleParams, { adHocDataViewList: DataView[] }>
> = (props) => {
const {
services: { dataViews, dataViewEditor },
} = useKibanaContextForPlugin();
const { setRuleParams, ruleParams, errors, metadata, onChangeMetaData } = props;
const { searchConfiguration, comparator, threshold, timeSize, timeUnit, groupBy } = ruleParams;
const [dataView, setDataView] = useState<DataView>();
const [dataViewError, setDataViewError] = useState<string>();
const [preselectedOptions, _] = useState<string[]>(defaultRuleParams.groupBy ?? []);
const [adHocDataViews, setAdHocDataViews] = useState<DataView[]>(
metadata?.adHocDataViewList ?? []
);
const preFillProperty = useCallback(
(property: keyof DegradedDocsRuleParams) => {
setRuleParams(property, defaultRuleParams[property]);
},
[setRuleParams]
);
const updateProperty = useCallback(
(property: keyof DegradedDocsRuleParams, value?: any) => {
setRuleParams(property, value);
},
[setRuleParams]
);
useEffect(() => {
const initDataView = async () => {
if (!searchConfiguration?.index) {
setDataViewError(
i18n.translate('xpack.datasetQuality.rule.dataViewErrorNoTimestamp', {
defaultMessage: 'A data view is required.',
})
);
return;
}
if (searchConfiguration?.index && !dataView) {
const savedDataViews = await dataViews.getIdsWithTitle();
const savedDataViewId = savedDataViews.find(
(dv) => dv.title === searchConfiguration?.index
)?.id;
if (savedDataViewId) {
setDataView(await dataViews.get(savedDataViewId));
return;
}
let currentDataView: DataView;
const adHocDataView = adHocDataViews.find((dv) => dv.title === searchConfiguration?.index);
if (adHocDataView) {
currentDataView = adHocDataView;
} else {
currentDataView = await dataViews.create({
title: searchConfiguration?.index,
timeFieldName: '@timestamp',
});
setAdHocDataViews((prev) => [...prev, currentDataView]);
}
setDataView(currentDataView);
}
};
initDataView();
}, [adHocDataViews, dataView, dataViews, searchConfiguration?.index]);
useEffect(() => {
if (!comparator) {
preFillProperty('comparator');
}
if (!threshold) {
preFillProperty('threshold');
}
if (!timeSize) {
preFillProperty('timeSize');
}
if (!timeUnit) {
preFillProperty('timeUnit');
}
if (!groupBy) {
preFillProperty('groupBy');
}
}, [preFillProperty, comparator, groupBy, threshold, timeSize, timeUnit]);
const onGroupByChange = useCallback(
(group: string | null | string[]) => {
const gb = group ? (isArray(group) ? group : [group]) : [];
setRuleParams('groupBy', gb);
},
[setRuleParams]
);
const derivedIndexPattern = useMemo<DataViewBase>(
() => ({
fields: dataView?.fields || [],
title: dataView?.getIndexPattern() || 'unknown-index',
}),
[dataView]
);
const onSelectDataView = useCallback(
(newDataView: DataView) => {
setDataViewError(undefined);
updateProperty('searchConfiguration', { index: newDataView.getIndexPattern() });
setDataView(newDataView);
},
[updateProperty]
);
const handleChange = useCallback(
(selectedOptions: Array<{ label: string }>) => {
const gb = selectedOptions.map((option) => option.label);
onGroupByChange([...new Set(preselectedOptions.concat(gb))]);
},
[onGroupByChange, preselectedOptions]
);
const getPreSelectedOptions = () => {
return preselectedOptions.map((field) => ({
label: field,
color: 'lightgray',
disabled: true,
}));
};
const getUserSelectedOptions = (group: string[] | undefined) => {
return (group ?? [])
.filter((g) => !preselectedOptions.includes(g))
.map((field) => ({
label: field,
}));
};
const selectedOptions = [...getPreSelectedOptions(), ...getUserSelectedOptions(groupBy)];
return (
<>
<EuiTitle size="xs">
<h5>
<FormattedMessage
id="xpack.datasetQuality.rule.dataView"
defaultMessage="Select a data view"
/>
</h5>
</EuiTitle>
<EuiSpacer size="s" />
<DataViewSelectPopover
dependencies={{ dataViews, dataViewEditor }}
dataView={dataView}
metadata={{ adHocDataViewList: adHocDataViews }}
onSelectDataView={onSelectDataView}
onChangeMetaData={({ adHocDataViewList }) => {
onChangeMetaData({ ...metadata, adHocDataViewList });
}}
/>
{dataViewError && (
<>
<EuiFormErrorText data-test-subj="datasetQualityRuleDataViewError">
{dataViewError}
</EuiFormErrorText>
<EuiSpacer size="s" />
</>
)}
<EuiSpacer size="m" />
<EuiTitle size="xs">
<h5>
<FormattedMessage
id="xpack.datasetQuality.rule.alertCondition"
defaultMessage="Set rule conditions"
/>
</h5>
</EuiTitle>
<EuiSpacer size="s" />
<EuiExpression
data-test-subj="datasetQualityRuleCountExpression"
description={'PERCENTAGE'}
value={degradedDocsLabel}
display="columns"
onClick={() => {}}
disabled={true}
/>
<ThresholdExpression
thresholdComparator={comparator ?? defaultRuleParams.comparator}
threshold={threshold}
onChangeSelectedThresholdComparator={(value) => updateProperty('comparator', value)}
onChangeSelectedThreshold={(value) => updateProperty('threshold', value)}
errors={errors}
display="fullWidth"
unit={'%'}
/>
<RuleConditionChart
threshold={threshold}
comparator={comparator as COMPARATORS}
timeSize={timeSize}
timeUnit={timeUnit as TimeUnitChar}
dataView={dataView}
groupBy={groupBy}
timeRange={{ from: `now-${(timeSize ?? 1) * 20}${timeUnit}`, to: 'now' }}
/>
<EuiSpacer size="l" />
<ForLastExpression
timeWindowSize={timeSize}
timeWindowUnit={timeUnit}
errors={{
timeSize: [],
timeUnit: [],
}}
onChangeWindowSize={(value) => updateProperty('timeSize', value)}
onChangeWindowUnit={(value) => updateProperty('timeUnit', value)}
display="fullWidth"
/>
<EuiSpacer size="l" />
<EuiFormRow
label={i18n.translate('xpack.datasetQuality.rule.createAlertPerText', {
defaultMessage: 'Group alerts by (optional)',
})}
helpText={i18n.translate('xpack.datasetQuality.rule.createAlertPerHelpText', {
defaultMessage:
'Create an alert for every unique value. For example: "host.id" or "cloud.region".',
})}
fullWidth
display="rowCompressed"
>
<EuiComboBox
data-test-subj="datasetQualityRuleGroupBy"
placeholder={i18n.translate('xpack.datasetQuality.rule.groupByLabel', {
defaultMessage: 'Everything',
})}
aria-label={i18n.translate('xpack.datasetQuality.rule.groupByAriaLabel', {
defaultMessage: 'Graph per',
})}
fullWidth
singleSelection={false}
selectedOptions={selectedOptions}
options={((derivedIndexPattern.fields as DataStreamGroupByFields) ?? [])
.filter((f) => f.aggregatable && f.type === 'string')
.map((f) => ({ label: f.name }))}
onChange={handleChange}
isClearable={true}
/>
</EuiFormRow>
</>
);
};

View file

@ -0,0 +1,67 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { COMPARATORS } from '@kbn/alerting-comparators';
import { i18n } from '@kbn/i18n';
import { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
import { ValidationResult } from '@kbn/triggers-actions-ui-plugin/public/types';
const invalidThresholdValue = (value?: number) =>
!value || (value && (isNaN(value) || value < 0 || value > 100));
export function validate(ruleParams: DegradedDocsRuleParams): ValidationResult {
const errors: { [key: string]: string[] } = {};
if (!ruleParams.searchConfiguration) {
return {
errors: {
searchConfiguration: [
i18n.translate(
'xpack.datasetQuality.alerts.validation.error.requiredSearchConfiguration',
{
defaultMessage: 'Search source configuration is required.',
}
),
],
},
};
}
if (!ruleParams.searchConfiguration.index) {
return {
errors: {
searchConfiguration: [
i18n.translate('xpack.datasetQuality.alerts.validation.error.requiredDataViewText', {
defaultMessage: 'Data view is required.',
}),
],
},
};
}
if (!ruleParams.threshold || invalidThresholdValue(ruleParams.threshold?.[0])) {
errors.threshold0 = [
i18n.translate('xpack.datasetQuality.alerts.validation.threshold', {
defaultMessage: 'A valid percentage threshold is required (0-100).',
}),
];
}
if (
(ruleParams.comparator === COMPARATORS.BETWEEN ||
ruleParams.comparator === COMPARATORS.NOT_BETWEEN) &&
invalidThresholdValue(ruleParams.threshold?.[1])
) {
errors.threshold1 = [
i18n.translate('xpack.datasetQuality.alerts.validation.threshold', {
defaultMessage: 'A valid percentage threshold is required (0-100).',
}),
];
}
return { errors };
}

View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { TriggersAndActionsUIPublicPluginSetup } from '@kbn/triggers-actions-ui-plugin/public';
import { getRuleType as getDegradedDocsRuleType } from './degraded_docs';
export function registerRuleTypes({
ruleTypeRegistry,
}: {
ruleTypeRegistry: TriggersAndActionsUIPublicPluginSetup['ruleTypeRegistry'];
}) {
ruleTypeRegistry.register(getDegradedDocsRuleType());
}

View file

@ -5,14 +5,22 @@
* 2.0.
*/
import { ChartsPluginStart } from '@kbn/charts-plugin/public';
import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
import { DataViewEditorStart } from '@kbn/data-view-editor-plugin/public';
import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
import type { LensPublicStart } from '@kbn/lens-plugin/public';
import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
import {
TriggersAndActionsUIPublicPluginSetup,
TriggersAndActionsUIPublicPluginStart,
} from '@kbn/triggers-actions-ui-plugin/public';
import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
import type { ComponentType } from 'react';
import type { PluginSetupContract as AlertingPublicSetup } from '@kbn/alerting-plugin/public/plugin';
import type { PluginStartContract as AlertingPublicStart } from '@kbn/alerting-plugin/public/plugin';
import type { DatasetQualityProps } from './components/dataset_quality';
import type { DatasetQualityDetailsProps } from './components/dataset_quality_details';
import type { CreateDatasetQualityController } from './controller/dataset_quality';
@ -29,15 +37,21 @@ export interface DatasetQualityPluginStart {
}
export interface DatasetQualityStartDeps {
alerting: AlertingPublicStart;
charts: ChartsPluginStart;
data: DataPublicPluginStart;
share: SharePluginStart;
fieldFormats: FieldFormatsStart;
unifiedSearch: UnifiedSearchPublicPluginStart;
lens: LensPublicStart;
dataViewEditor: DataViewEditorStart;
dataViews: DataViewsPublicPluginStart;
fieldFormats: FieldFormatsStart;
fieldsMetadata: FieldsMetadataPublicStart;
lens: LensPublicStart;
share: SharePluginStart;
triggersActionsUi: TriggersAndActionsUIPublicPluginStart;
unifiedSearch: UnifiedSearchPublicPluginStart;
}
export interface DatasetQualitySetupDeps {
alerting?: AlertingPublicSetup;
share: SharePluginSetup;
triggersActionsUi: TriggersAndActionsUIPublicPluginSetup;
}

View file

@ -7,10 +7,11 @@
import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server';
import { mapValues } from 'lodash';
import { DataTelemetryService } from './services';
import { getDatasetQualityServerRouteRepository } from './routes';
import { registerRoutes } from './routes/register_routes';
import { DatasetQualityRouteHandlerResources } from './routes/types';
import { registerBuiltInRuleTypes } from './rule_types';
import { DataTelemetryService } from './services';
import {
DatasetQualityPluginSetup,
DatasetQualityPluginSetupDependencies,
@ -78,6 +79,10 @@ export class DatasetQualityServerPlugin
// Setup Data Telemetry Service
this.dataTelemetryService.setup(plugins.taskManager, plugins.usageCollection);
if (plugins.alerting) {
registerBuiltInRuleTypes(plugins.alerting, plugins.share?.url.locators);
}
return {};
}

View file

@ -7,11 +7,13 @@
import type { EndpointOf, ServerRouteRepository } from '@kbn/server-route-repository';
import { dataStreamsRouteRepository } from './data_streams/routes';
import { integrationsRouteRepository } from './integrations/routes';
import { ruleTypesRouteRepository } from './rule_types/routes';
function getTypedDatasetQualityServerRouteRepository() {
const repository = {
...dataStreamsRouteRepository,
...integrationsRouteRepository,
...ruleTypesRouteRepository,
};
return repository;

View file

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import * as t from 'io-ts';
import { PreviewChartResponse } from '../../../common/api_types';
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
import { getChartPreview } from '../../rule_types/degraded_docs/get_chart_preview';
import { groupByRt } from '../../types/default_api_types';
const degradedDocsChartPreviewRoute = createDatasetQualityServerRoute({
endpoint: 'GET /internal/dataset_quality/rule_types/degraded_docs/chart_preview',
params: t.type({
query: t.type({
index: t.string,
groupBy: groupByRt,
start: t.string,
end: t.string,
interval: t.string,
}),
}),
options: {
tags: [],
},
security: {
authz: {
enabled: false,
reason:
'This API delegates security to the currently logged in user and their Elasticsearch permissions.',
},
},
async handler(resources): Promise<PreviewChartResponse> {
const { context, params } = resources;
const coreContext = await context.core;
const esClient = coreContext.elasticsearch.client.asCurrentUser;
const degradedDocsChartPreview = await getChartPreview({
esClient,
...params.query,
});
return { ...degradedDocsChartPreview };
},
});
export const ruleTypesRouteRepository = {
...degradedDocsChartPreviewRoute,
};

View file

@ -0,0 +1,157 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
DATA_QUALITY_DETAILS_LOCATOR_ID,
DATA_QUALITY_LOCATOR_ID,
} from '@kbn/deeplinks-observability';
import { i18n } from '@kbn/i18n';
import { getHumanReadableComparator } from '@kbn/stack-alerts-plugin/common';
import type { Comparator } from '@kbn/stack-alerts-plugin/common/comparator_types';
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
import moment from 'moment';
import { TimeRangeConfig } from '../../common/types';
import { AdditionalContext, DatasetQualityRuleParams } from './types';
export const getPaddedAlertTimeRange = (
alertStart: string,
alertEnd?: string,
lookBackWindow?: {
size: number;
unit: 's' | 'm' | 'h' | 'd';
}
): TimeRangeConfig => {
const alertDuration = moment.duration(moment(alertEnd).diff(moment(alertStart)));
const now = moment().toISOString();
// If alert duration is less than 160 min, we use 20 minute buffer
// Otherwise, we use 8 times alert duration
const defaultDurationMs =
alertDuration.asMinutes() < 160
? moment.duration(20, 'minutes').asMilliseconds()
: alertDuration.asMilliseconds() / 8;
// To ensure the alert time range at least covers 20 times lookback window,
// we compare lookBackDurationMs and defaultDurationMs to use any of those that is longer
const lookBackDurationMs =
lookBackWindow &&
moment.duration(lookBackWindow.size * 20, lookBackWindow.unit).asMilliseconds();
const durationMs =
lookBackDurationMs && lookBackDurationMs - defaultDurationMs > 0
? lookBackDurationMs
: defaultDurationMs;
const from = moment(alertStart).subtract(durationMs, 'millisecond').toISOString();
const to =
alertEnd && moment(alertEnd).add(durationMs, 'millisecond').isBefore(now)
? moment(alertEnd).add(durationMs, 'millisecond').toISOString()
: now;
return {
from,
to,
refresh: {
pause: true,
value: 60000,
},
};
};
const getDataQualityViewInAppUrl = ({
index,
from,
to,
locatorsClient,
}: {
index: string;
from: string;
to: string;
locatorsClient?: LocatorClient;
}) => {
const timeRange: TimeRangeConfig | undefined = getPaddedAlertTimeRange(from, to);
timeRange.to = to ? timeRange.to : 'now';
// If index is a wildcard or multiple indices, redirect to the data quality overview page
if (index.includes('*') || index.includes(',')) {
return locatorsClient?.get(DATA_QUALITY_LOCATOR_ID)?.getRedirectUrl({
timeRange: {
from,
to,
refresh: {
pause: true,
value: 60000,
},
},
});
}
return locatorsClient?.get(DATA_QUALITY_DETAILS_LOCATOR_ID)?.getRedirectUrl({
dataStream: index,
timeRange: {
from,
to,
refresh: {
pause: true,
value: 60000,
},
},
});
};
const generateReason = (
value: string,
group: string,
timeSize: number,
timeUnit: string,
comparator: Comparator,
threshold: number[]
) =>
i18n.translate('xpack.datasetQuality.rule.alertTypeContextReasonDescription', {
defaultMessage: `Percentage of degraded documents is {value}% in the last {window} for {group}. Alert when {comparator} {threshold}%.`,
values: {
value,
window: `${timeSize}${timeUnit}`,
group,
comparator: getHumanReadableComparator(comparator),
threshold: threshold.join(' and '),
},
});
export const generateContext = ({
group,
dateStart,
dateEnd,
value,
params,
grouping,
locatorsClient,
}: {
group: string;
dateStart: string;
dateEnd: string;
value: string;
params: DatasetQualityRuleParams;
grouping?: AdditionalContext;
locatorsClient?: LocatorClient;
}) => ({
value,
reason: generateReason(
value,
group,
params.timeSize,
params.timeUnit,
params.comparator as Comparator,
params.threshold
),
grouping,
timestamp: dateEnd,
viewInAppUrl: getDataQualityViewInAppUrl({
index: params.searchConfiguration.index,
from: dateStart,
to: dateEnd,
locatorsClient,
}),
});

View file

@ -0,0 +1,184 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { AlertsClientError, ExecutorType, RuleExecutorOptions } from '@kbn/alerting-plugin/server';
import { i18n } from '@kbn/i18n';
import { ComparatorFns, TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
import {
ALERT_EVALUATION_THRESHOLD,
ALERT_EVALUATION_VALUE,
ALERT_GROUPING,
ALERT_KEY_JOINER,
ALERT_REASON,
} from '@kbn/rule-data-utils';
import { Comparator } from '@kbn/stack-alerts-plugin/common/comparator_types';
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
import { _IGNORED } from '../../../common/es_fields';
import { generateContext } from '../context';
import { getDocsStats } from '../get_docs_stats';
import {
AdditionalContext,
THRESHOLD_MET_GROUP,
type DatasetQualityAlert,
type DatasetQualityAlertContext,
type DatasetQualityAlertState,
type DatasetQualityAllowedActionGroups,
type DatasetQualityRuleParams,
type DatasetQualityRuleTypeState,
} from '../types';
export const formatDurationFromTimeUnitChar = (time: number, unit: TimeUnitChar): string => {
const sForPlural = time !== 0 && time > 1 ? 's' : '';
switch (unit) {
case 's':
return `${time} sec${sForPlural}`;
case 'm':
return `${time} min${sForPlural}`;
case 'h':
return `${time} hr${sForPlural}`;
case 'd':
return `${time} day${sForPlural}`;
default:
return `${time} ${unit}`;
}
};
export const getRuleExecutor = (locatorsClient?: LocatorClient) =>
async function executor(
options: RuleExecutorOptions<
DatasetQualityRuleParams,
DatasetQualityRuleTypeState,
DatasetQualityAlertState,
DatasetQualityAlertContext,
DatasetQualityAllowedActionGroups,
DatasetQualityAlert
>
): ReturnType<
ExecutorType<
DatasetQualityRuleParams,
DatasetQualityRuleTypeState,
DatasetQualityAlertState,
DatasetQualityAlertContext,
DatasetQualityAllowedActionGroups
>
> {
const { services, params, logger, getTimeRange } = options;
const { alertsClient, scopedClusterClient } = services;
if (!alertsClient) {
throw new AlertsClientError();
}
const alertLimit = alertsClient.getAlertLimitValue();
const { dateStart, dateEnd } = getTimeRange(`${params.timeSize}${params.timeUnit}`);
const index = params.searchConfiguration.index;
const datasetQualityDegradedResults = await getDocsStats({
index,
dateStart,
groupBy: params.groupBy ?? [],
query: {
must: { exists: { field: _IGNORED } },
},
scopedClusterClient,
});
const unmetGroupValues: Record<string, string> = {};
const compareFn = ComparatorFns.get(params.comparator as Comparator);
if (compareFn == null) {
throw new Error(
i18n.translate('xpack.datasetQuality.rule.invalidComparatorErrorMessage', {
defaultMessage: 'invalid thresholdComparator specified: {comparator}',
values: {
comparator: params.comparator,
},
})
);
}
let generatedAlerts = 0;
for (const groupResult of datasetQualityDegradedResults) {
const { bucketKey, percentage } = groupResult;
const alertId = bucketKey.join(ALERT_KEY_JOINER);
const met = compareFn(percentage, params.threshold);
if (!met) {
unmetGroupValues[alertId] = percentage.toFixed(2);
continue;
}
const groupByFields: AdditionalContext = {};
const groupBy = params.groupBy ?? [];
for (let i = 0; i < bucketKey.length; i++) {
const fieldName = groupBy[i];
groupByFields[fieldName] = bucketKey[i];
}
if (generatedAlerts < alertLimit) {
const context = generateContext({
group: alertId,
dateStart,
dateEnd,
value: percentage.toFixed(2),
params,
grouping: groupByFields,
locatorsClient,
});
alertsClient.report({
id: alertId,
actionGroup: THRESHOLD_MET_GROUP.id,
state: {},
context,
payload: {
[ALERT_REASON]: context.reason,
[ALERT_EVALUATION_VALUE]: `${context.value}`,
[ALERT_EVALUATION_THRESHOLD]:
params.threshold?.length === 1 ? params.threshold[0] : null,
[ALERT_GROUPING]: groupByFields,
},
});
}
generatedAlerts++;
}
alertsClient.setAlertLimitReached(generatedAlerts >= alertLimit);
// Handle recovered alerts context
const { getRecoveredAlerts } = alertsClient;
for (const recoveredAlert of getRecoveredAlerts()) {
const alertId = recoveredAlert.alert.getId();
logger.debug(`setting context for recovered alert ${alertId}`);
const grouping = recoveredAlert.hit?.[ALERT_GROUPING];
const percentage = unmetGroupValues[alertId] ?? '0';
const recoveryContext = generateContext({
group: alertId,
dateStart,
dateEnd,
value: percentage,
params,
grouping,
locatorsClient,
});
alertsClient.setAlertData({
id: alertId,
context: recoveryContext,
payload: {
[ALERT_REASON]: recoveryContext.reason,
[ALERT_EVALUATION_VALUE]: `${recoveryContext.value}`,
[ALERT_EVALUATION_THRESHOLD]: params.threshold?.length === 1 ? params.threshold[0] : null,
[ALERT_GROUPING]: grouping,
},
});
}
return { state: {} };
};

View file

@ -0,0 +1,177 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { estypes } from '@elastic/elasticsearch';
import { ElasticsearchClient } from '@kbn/core/server';
import { PreviewChartResponse } from '../../../common/api_types';
import { Coordinate } from '../../../common/types';
import { extractKey } from '../extract_key';
import { MISSING_VALUE } from '../types';
interface DataStreamTotals {
x: Coordinate['x'];
totalCount: number;
ignoredCount: number;
}
const NUM_SERIES = 5;
const DEFAULT_GROUPS = 1000;
export const getFilteredBarSeries = (barSeries: PreviewChartResponse['series']) => {
const sortedSeries = barSeries.sort((a, b) => {
const aMax = Math.max(...a.data.map((point) => point.y as number));
const bMax = Math.max(...b.data.map((point) => point.y as number));
return bMax - aMax;
});
return sortedSeries.slice(0, NUM_SERIES);
};
export async function getChartPreview({
esClient,
index,
groupBy,
start,
end,
interval,
}: {
esClient: ElasticsearchClient;
index: string;
groupBy: string[];
start: string;
end: string;
interval: string;
}): Promise<PreviewChartResponse> {
const bool = {
filter: [
{
range: {
'@timestamp': {
gte: start,
lte: end,
format: 'epoch_millis',
},
},
},
],
};
const aggs = {
...(groupBy.length > 0
? {
series: {
...(groupBy.length === 1
? {
terms: {
field: groupBy[0],
size: DEFAULT_GROUPS,
order: { _count: 'desc' as const },
},
}
: {
multi_terms: {
terms: groupBy.map((field) => ({ field, missing: MISSING_VALUE })),
size: DEFAULT_GROUPS,
order: { _count: 'desc' as const },
},
}),
aggs: {
timeseries: {
date_histogram: {
field: '@timestamp',
fixed_interval: interval,
extended_bounds: {
min: start,
max: end,
},
},
aggs: {
ignored_fields: {
filter: {
exists: {
field: '_ignored',
},
},
},
},
},
},
},
}
: {}),
};
const esResult = await esClient.search<
unknown,
{ series: estypes.AggregationsStringTermsAggregate }
>({
index,
track_total_hits: false,
size: 0,
query: {
bool,
},
aggs,
});
if (!esResult.aggregations) {
return { series: [], totalGroups: 0 };
}
const seriesBuckets = (esResult.aggregations?.series?.buckets ??
[]) as estypes.AggregationsStringTermsBucket[];
const seriesDataMap: Record<string, DataStreamTotals[]> = seriesBuckets.reduce((acc, bucket) => {
const bucketKey = extractKey({
groupBy,
bucketKey: Array.isArray(bucket.key) ? bucket.key : [bucket.key],
});
const timeSeriesBuckets = bucket.timeseries as estypes.AggregationsTimeSeriesAggregate;
const timeseries = timeSeriesBuckets.buckets as estypes.AggregationsTimeSeriesBucket[];
timeseries.forEach((timeseriesBucket: estypes.AggregationsTimeSeriesBucket) => {
const x = (timeseriesBucket as Record<string, estypes.FieldValue>).key as number;
const totalCount = timeseriesBucket.doc_count ?? 0;
const ignoredCount =
(timeseriesBucket.ignored_fields as estypes.AggregationsMultiBucketBase)?.doc_count ?? 0;
if (acc[bucketKey.join(',')]) {
acc[bucketKey.join(',')].push({ x, totalCount, ignoredCount });
} else {
acc[bucketKey.join(',')] = [{ x, totalCount, ignoredCount }];
}
});
return acc;
}, {} as Record<string, DataStreamTotals[]>);
const series = Object.keys(seriesDataMap).map((key) => ({
name: key,
data: Array.from(
seriesDataMap[key]
.reduce((map, curr: DataStreamTotals) => {
if (!map.has(curr.x)) map.set(curr.x, { ...curr });
else {
map.get(curr.x).totalCount += curr.totalCount;
map.get(curr.x).ignoredCount += curr.ignoredCount;
}
return map;
}, new Map())
.values()
).map((item: DataStreamTotals) => ({
x: item.x,
y: item.totalCount ? (item.ignoredCount / item.totalCount) * 100 : 0,
})),
}));
const filteredSeries = getFilteredBarSeries(series);
return {
series: filteredSeries,
totalGroups: series.length,
};
}

View file

@ -0,0 +1,151 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { RuleType } from '@kbn/alerting-plugin/server';
import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
import { i18n } from '@kbn/i18n';
import {
DegradedDocsRuleParams,
degradedDocsParamsSchema,
} from '@kbn/response-ops-rule-params/degraded_docs';
import {
ALERT_EVALUATION_THRESHOLD,
ALERT_EVALUATION_VALUE,
ALERT_GROUPING,
ALERT_REASON,
DEGRADED_DOCS_RULE_TYPE_ID,
STACK_ALERTS_FEATURE_ID,
} from '@kbn/rule-data-utils';
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
import {
DATASET_QUALITY_REGISTRATION_CONTEXT,
DatasetQualityAlert,
DatasetQualityAlertContext,
DatasetQualityAllowedActionGroups,
THRESHOLD_MET_GROUP,
} from '../types';
import { getRuleExecutor } from './executor';
export function DegradedDocsRuleType(
locatorsClient?: LocatorClient
): RuleType<
DegradedDocsRuleParams,
never,
{},
{},
DatasetQualityAlertContext,
DatasetQualityAllowedActionGroups,
never,
DatasetQualityAlert
> {
return {
id: DEGRADED_DOCS_RULE_TYPE_ID,
name: i18n.translate('xpack.datasetQuality.rule.degradedDocs.name', {
defaultMessage: 'Degraded docs',
}),
solution: 'stack',
validate: {
params: degradedDocsParamsSchema,
},
schemas: {
params: {
type: 'config-schema',
schema: degradedDocsParamsSchema,
},
},
defaultActionGroupId: THRESHOLD_MET_GROUP.id,
actionGroups: [THRESHOLD_MET_GROUP],
category: DEFAULT_APP_CATEGORIES.management.id,
producer: STACK_ALERTS_FEATURE_ID,
minimumLicenseRequired: 'basic',
isExportable: true,
executor: getRuleExecutor(locatorsClient),
doesSetRecoveryContext: true,
actionVariables: {
context: [
{ name: 'reason', description: actionVariableContextReasonLabel },
{ name: 'value', description: actionVariableContextValueLabel },
{ name: 'grouping', description: actionVariableContextGroupingLabel },
{
name: 'threshold',
description: actionVariableContextThresholdLabel,
},
{ name: 'viewInAppUrl', description: viewInAppUrlActionVariableDescription },
],
},
alerts: {
context: DATASET_QUALITY_REGISTRATION_CONTEXT,
mappings: {
fieldMap: {
[ALERT_REASON]: { type: 'keyword', array: false, required: false },
[ALERT_EVALUATION_VALUE]: { type: 'keyword', array: false, required: false },
[ALERT_EVALUATION_THRESHOLD]: {
type: 'scaled_float',
scaling_factor: 100,
required: false,
},
[ALERT_GROUPING]: {
type: 'object',
dynamic: true,
array: false,
required: false,
},
},
dynamicTemplates: [
{
strings_as_keywords: {
path_match: 'kibana.alert.grouping.*',
match_mapping_type: 'string',
mapping: {
type: 'keyword',
ignore_above: 1024,
},
},
},
],
},
shouldWrite: true,
useEcs: true,
},
};
}
export const actionVariableContextReasonLabel = i18n.translate(
'xpack.datasetQuality.alerting.actionVariableContextReasonLabel',
{
defaultMessage: 'A concise description of the reason for the alert.',
}
);
const actionVariableContextValueLabel = i18n.translate(
'xpack.datasetQuality.alerting.actionVariableContextValueLabel',
{
defaultMessage: 'The value that met the threshold condition.',
}
);
const actionVariableContextThresholdLabel = i18n.translate(
'xpack.datasetQuality.alerting.actionVariableContextThresholdLabel',
{
defaultMessage:
'An array of rule threshold values. For between and notBetween thresholds, there are two values.',
}
);
const actionVariableContextGroupingLabel = i18n.translate(
'xpack.datasetQuality.alerting.actionVariableContextGrouping',
{
defaultMessage: 'The object containing groups that are reporting data',
}
);
export const viewInAppUrlActionVariableDescription = i18n.translate(
'xpack.datasetQuality.alerting.actionVariableContextViewInAppUrl',
{
defaultMessage: 'Link to the alert source',
}
);

View file

@ -0,0 +1,66 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { extractKey } from './extract_key';
describe('extractKey', () => {
it('returns empty if "_index" is not part of the groupBy', async () => {
const result = extractKey({
groupBy: ['host.name', 'source.ip'],
bucketKey: ['.ds-logs-custom-default-2025.04.08-000001', '127.0.0.1'],
});
expect(result).toEqual(['.ds-logs-custom-default-2025.04.08-000001', '127.0.0.1']);
});
describe('when "_index" is part of the groupBy', () => {
it('and is not a backing index name', async () => {
const result = extractKey({
groupBy: ['_index'],
bucketKey: ['logs-custom-default'],
});
expect(result).toEqual(['logs-custom-default']);
});
it('and is the only element in groupBy', async () => {
const result = extractKey({
groupBy: ['_index'],
bucketKey: ['.ds-logs-custom-default-2025.04.08-000001'],
});
expect(result).toEqual(['logs-custom-default']);
});
it('and is at the beginning of the groupBy', async () => {
const result = extractKey({
groupBy: ['_index', 'cloud.provider'],
bucketKey: ['.ds-logs-custom-default-2025.04.08-000001', 'aws'],
});
expect(result).toEqual(['logs-custom-default', 'aws']);
});
it('and is at the end of the groupBy', async () => {
const result = extractKey({
groupBy: ['cloud.provider', '_index'],
bucketKey: ['aws', '.ds-logs-custom-default-2025.04.08-000001'],
});
expect(result).toEqual(['aws', 'logs-custom-default']);
});
it('and is at the middle of the groupBy', async () => {
const result = extractKey({
groupBy: ['cloud.region', '_index', 'cloud.provider'],
bucketKey: ['eu-central-1', '.ds-logs-custom-default-2025.04.08-000001', 'aws'],
});
expect(result).toEqual(['eu-central-1', 'logs-custom-default', 'aws']);
});
});
});

View file

@ -0,0 +1,45 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FieldValue } from '@elastic/elasticsearch/lib/api/types';
import { INDEX } from '../../common/es_fields';
import { extractIndexNameFromBackingIndex } from '../../common/utils';
/**
* Extracts the key from the bucket key based on the groupBy fields.
* If "_index" is not part of the groupBy, bucketKey is returned.
* Otherwise, it replaces the "_index" value in the bucketKey with the actual dataStream name.
*
* @param {Object} params - The parameters object.
* @param {string[]} params.groupBy - The groupBy fields.
* @param {string[]} params.bucketKey - The bucket key values.
* @returns {string[]} The extracted key.
* @example
* // returns ['logs-dataset-namespace', 'aws']
* extractKey({ groupBy: ['_index', 'cloud.provider'], bucketKey: ['.ds-logs-dataset-namespace-2025.04.08-000001', 'aws'] });
*/
export const extractKey = ({
groupBy,
bucketKey,
}: {
groupBy: string[];
bucketKey: FieldValue[];
}): string[] => {
if (!groupBy.includes(INDEX)) {
return bucketKey as string[];
}
const dataStreamIndex = groupBy.findIndex((group) => group === INDEX);
const dataStreamName = extractIndexNameFromBackingIndex(bucketKey[dataStreamIndex] as string);
const key = [
...bucketKey.slice(0, dataStreamIndex),
dataStreamName,
...bucketKey.slice(dataStreamIndex + 1),
];
return key as string[];
};

View file

@ -0,0 +1,120 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { estypes } from '@elastic/elasticsearch';
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import { IScopedClusterClient } from '@kbn/core/server';
import { extractKey } from './extract_key';
import { MISSING_VALUE } from './types';
const DEFAULT_GROUPS = 1000;
export interface FetchEsQueryOpts {
index: string;
groupBy: string[];
query?: {
must?: QueryDslQueryContainer | QueryDslQueryContainer[];
};
services: {
scopedClusterClient: IScopedClusterClient;
};
dateStart: string;
}
export async function fetchEsQuery({
index,
groupBy,
query = {},
services,
dateStart,
}: FetchEsQueryOpts) {
const { scopedClusterClient } = services;
const esClient = scopedClusterClient.asCurrentUser;
const { must } = query;
const bool = {
filter: [
{
range: {
'@timestamp': {
gte: new Date(dateStart).getTime(),
},
},
},
],
};
const aggs = {
...(groupBy.length > 0
? {
dataStreams: {
...(groupBy.length === 1
? {
terms: {
field: groupBy[0],
size: DEFAULT_GROUPS,
order: { _count: 'desc' as const },
},
}
: {
multi_terms: {
terms: groupBy.map((field) => ({ field, missing: MISSING_VALUE })),
size: DEFAULT_GROUPS,
order: { _count: 'desc' as const },
},
}),
},
}
: {}),
};
const esResult = await esClient.search<
unknown,
{ dataStreams: estypes.AggregationsStringTermsAggregate }
>({
index,
track_total_hits: false,
size: 0,
query: {
bool: {
...bool,
...(must ? { must } : {}),
},
},
aggs,
});
const dataStreamBuckets =
(esResult.aggregations?.dataStreams?.buckets as estypes.AggregationsStringTermsBucketKeys[]) ||
[];
// Group values by dataStream name instead of backing index name
const groupedDataStreams = dataStreamBuckets.reduce(
(acc: Record<string, { bucketKey: string[]; docCount: number }>, bucket) => {
const key = extractKey({
groupBy,
bucketKey: Array.isArray(bucket.key) ? bucket.key : [bucket.key],
});
return {
...acc,
[key.join(',')]: {
bucketKey: key,
docCount: (acc[key.join(',')]?.docCount ?? 0) + bucket.doc_count,
},
};
},
{} as Record<string, { bucketKey: string[]; docCount: number }>
);
return Object.keys(groupedDataStreams).reduce((obj, bucket) => {
obj[groupedDataStreams[bucket].bucketKey.join(',')] = {
docCount: groupedDataStreams[bucket].docCount,
bucketKey: groupedDataStreams[bucket].bucketKey,
};
return obj;
}, {} as Record<string, { bucketKey: string[]; docCount: number }>);
}

View file

@ -0,0 +1,62 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { IScopedClusterClient } from '@kbn/core/server';
import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import { fetchEsQuery } from './fetch_es_query';
export interface DocStat {
bucketKey: string[];
percentage: number;
}
export const getDocsStats = async ({
index,
dateStart,
groupBy,
query,
scopedClusterClient,
}: {
index: string;
dateStart: string;
groupBy: string[];
query?: {
must: QueryDslQueryContainer | QueryDslQueryContainer[];
};
scopedClusterClient: IScopedClusterClient;
}): Promise<DocStat[]> => {
const dataStreamTotalDocsResults = await fetchEsQuery({
index,
dateStart,
groupBy,
services: {
scopedClusterClient,
},
});
const dataStreamQueryDocsResults = await fetchEsQuery({
index,
dateStart,
groupBy,
query,
services: {
scopedClusterClient,
},
});
return Object.keys(dataStreamTotalDocsResults).map((key) => {
const totalDocs = dataStreamTotalDocsResults[key].docCount;
const queryDocs = dataStreamQueryDocsResults[key]?.docCount ?? 0;
const bucketKey =
dataStreamTotalDocsResults[key]?.bucketKey ?? dataStreamQueryDocsResults[key]?.bucketKey;
return {
bucketKey,
percentage: totalDocs ? (queryDocs / totalDocs) * 100 : 0,
};
});
};

View file

@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { AlertingServerSetup } from '@kbn/alerting-plugin/server';
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
import { DegradedDocsRuleType } from './degraded_docs/register';
export function registerBuiltInRuleTypes(
alertingPlugin: AlertingServerSetup,
locatorsClient?: LocatorClient
) {
alertingPlugin.registerType(DegradedDocsRuleType(locatorsClient));
}

View file

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import {
ActionGroupIdsOf,
AlertInstanceContext as AlertContext,
AlertInstanceState as AlertState,
} from '@kbn/alerting-plugin/common';
import { RuleTypeState } from '@kbn/alerting-plugin/server';
import { StackAlert } from '@kbn/alerts-as-data-utils';
import { i18n } from '@kbn/i18n';
import type { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
export type DatasetQualityRuleParams = DegradedDocsRuleParams;
export type DatasetQualityRuleTypeState = RuleTypeState;
export type DatasetQualityAlertState = AlertState;
export type DatasetQualityAlertContext = AlertContext;
export type DatasetQualityAllowedActionGroups = ActionGroupIdsOf<typeof THRESHOLD_MET_GROUP>;
export type DatasetQualityAlert = Omit<StackAlert, 'kibana.alert.evaluation.threshold'> & {
'kibana.alert.evaluation.threshold'?: string | number | null;
'kibana.alert.grouping'?: Record<string, string>;
};
export interface AdditionalContext {
[x: string]: any;
}
export const DATASET_QUALITY_REGISTRATION_CONTEXT = 'dataset.quality';
const THRESHOLD_MET_GROUP_ID = 'threshold_met';
export const THRESHOLD_MET_GROUP = {
id: THRESHOLD_MET_GROUP_ID,
name: i18n.translate('xpack.datasetQuality.alerting.action.thresholdMet', {
defaultMessage: 'Threshold met',
}),
};
export const MISSING_VALUE = i18n.translate('xpack.datasetQuality.alerting.missingValue', {
defaultMessage: 'N/A',
});

View file

@ -5,8 +5,10 @@
* 2.0.
*/
import { AlertingServerSetup, AlertingServerStart } from '@kbn/alerting-plugin/server';
import { CustomRequestHandlerContext } from '@kbn/core/server';
import type { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/server';
import {
TaskManagerSetupContract,
TaskManagerStartContract,
@ -15,17 +17,21 @@ import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-
import { UsageCollectionSetup, UsageCollectionStart } from '@kbn/usage-collection-plugin/server';
export interface DatasetQualityPluginSetupDependencies {
alerting?: AlertingServerSetup;
fleet: FleetSetupContract;
telemetry: TelemetryPluginSetup;
taskManager: TaskManagerSetupContract;
telemetry: TelemetryPluginSetup;
usageCollection?: UsageCollectionSetup;
share?: SharePluginSetup;
}
export interface DatasetQualityPluginStartDependencies {
alerting?: AlertingServerStart;
fleet: FleetStartContract;
telemetry: TelemetryPluginStart;
taskManager: TaskManagerStartContract;
telemetry: TelemetryPluginStart;
usageCollection?: UsageCollectionStart;
share?: SharePluginStart;
}
// eslint-disable-next-line @typescript-eslint/no-empty-interface

View file

@ -29,3 +29,14 @@ export const rangeRt = t.type({
start: isoToEpochRt,
end: isoToEpochRt,
});
export const groupByRt = new t.Type<string[], string, unknown>(
'groupByRt',
(input: unknown): input is string[] =>
Array.isArray(input) && input.every((value) => typeof value === 'string'),
(input: unknown, context: t.Context) =>
typeof input === 'string' && input.split(',').every((value) => typeof value === 'string')
? t.success(input.split(',') as string[])
: t.failure(input, context),
(output: string[]) => output.join(',')
);

View file

@ -10,55 +10,66 @@
"../../../../../typings/**/*"
],
"kbn_references": [
"@kbn/core",
"@kbn/core-plugins-server",
"@kbn/core-elasticsearch-server-mocks",
"@kbn/fleet-plugin",
"@kbn/server-route-repository",
"@kbn/share-plugin",
"@kbn/std",
"@kbn/i18n",
"@kbn/kibana-react-plugin",
"@kbn/i18n-react",
"@kbn/field-formats-plugin",
"@kbn/field-types",
"@kbn/io-ts-utils",
"@kbn/es-types",
"@kbn/deeplinks-observability",
"@kbn/router-utils",
"@kbn/xstate-utils",
"@kbn/shared-ux-utility",
"@kbn/data-plugin",
"@kbn/unified-search-plugin",
"@kbn/timerange",
"@kbn/lens-plugin",
"@kbn/data-views-plugin",
"@kbn/shared-ux-error-boundary",
"@kbn/es-query",
"@kbn/core-saved-objects-api-server",
"@kbn/deeplinks-management",
"@kbn/deeplinks-analytics",
"@kbn/core-elasticsearch-server",
"@kbn/ui-actions-plugin",
"@kbn/alerting-comparators",
"@kbn/alerting-plugin",
"@kbn/alerts-as-data-utils",
"@kbn/calculate-auto",
"@kbn/discover-plugin",
"@kbn/shared-ux-prompt-no-data-views-types",
"@kbn/ebt-tools",
"@kbn/fields-metadata-plugin",
"@kbn/server-route-repository-utils",
"@kbn/chart-icons",
"@kbn/charts-plugin",
"@kbn/charts-theme",
"@kbn/core-analytics-browser",
"@kbn/core-elasticsearch-server-mocks",
"@kbn/core-elasticsearch-server",
"@kbn/core-lifecycle-browser",
"@kbn/core-notifications-browser",
"@kbn/telemetry-plugin",
"@kbn/usage-collection-plugin",
"@kbn/rison",
"@kbn/task-manager-plugin",
"@kbn/core-plugins-server",
"@kbn/core-saved-objects-api-server",
"@kbn/core",
"@kbn/data-plugin",
"@kbn/data-view-editor-plugin",
"@kbn/data-views-plugin",
"@kbn/deeplinks-analytics",
"@kbn/deeplinks-management",
"@kbn/deeplinks-observability",
"@kbn/discover-plugin",
"@kbn/ebt-tools",
"@kbn/es-query",
"@kbn/es-types",
"@kbn/field-formats-plugin",
"@kbn/field-types",
"@kbn/field-utils",
"@kbn/fields-metadata-plugin",
"@kbn/fleet-plugin",
"@kbn/i18n-react",
"@kbn/i18n",
"@kbn/io-ts-utils",
"@kbn/kibana-react-plugin",
"@kbn/lens-plugin",
"@kbn/logging",
"@kbn/ui-theme",
"@kbn/react-hooks",
"@kbn/charts-theme",
"@kbn/unified-histogram"
"@kbn/response-ops-rule-form",
"@kbn/response-ops-rule-params",
"@kbn/rison",
"@kbn/router-utils",
"@kbn/rule-data-utils",
"@kbn/server-route-repository-utils",
"@kbn/server-route-repository",
"@kbn/share-plugin",
"@kbn/shared-ux-error-boundary",
"@kbn/shared-ux-prompt-no-data-views-types",
"@kbn/shared-ux-utility",
"@kbn/stack-alerts-plugin",
"@kbn/std",
"@kbn/task-manager-plugin",
"@kbn/telemetry-plugin",
"@kbn/timerange",
"@kbn/triggers-actions-ui-plugin",
"@kbn/ui-actions-plugin",
"@kbn/ui-theme",
"@kbn/unified-histogram",
"@kbn/unified-search-plugin",
"@kbn/usage-collection-plugin",
"@kbn/xstate-utils",
],
"exclude": [
"target/**/*"

View file

@ -68,6 +68,7 @@ export default function createRegisteredRuleTypeTests({ getService }: FtrProvide
'apm.anomaly',
'apm.error_rate',
'apm.transaction_error_rate',
'datasetQuality.degradedDocs',
];
expect(

View file

@ -36,8 +36,8 @@ export default function ({ getService }: FtrProviderContext) {
return fieldStat.name === 'geo_point';
}
);
expect(geoPointFieldStats.count).to.be(55);
expect(geoPointFieldStats.index_count).to.be(12);
expect(geoPointFieldStats.count).to.be(63);
expect(geoPointFieldStats.index_count).to.be(13);
const geoShapeFieldStats = apiResponse.cluster_stats.indices.mappings.field_types.find(
(fieldStat: estypes.ClusterStatsFieldTypes) => {

View file

@ -207,7 +207,7 @@ export default function ({ getService }: FtrProviderContext) {
securitySolutionSiemMigrations: ['all', 'read', 'minimal_all', 'minimal_read'],
infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],
logs: ['all', 'read', 'minimal_all', 'minimal_read'],
dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
dataQuality: ['all', 'read', 'minimal_all', 'minimal_read', 'manage_rules', 'manage_alerts'],
manageReporting: ['all', 'read', 'minimal_all', 'minimal_read'],
apm: ['all', 'read', 'minimal_all', 'minimal_read', 'settings_save'],
discover: [

View file

@ -312,7 +312,14 @@ export default function ({ getService }: FtrProviderContext) {
securitySolutionNotes: ['all', 'read', 'minimal_all', 'minimal_read'],
infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],
logs: ['all', 'read', 'minimal_all', 'minimal_read'],
dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
dataQuality: [
'all',
'read',
'minimal_all',
'minimal_read',
'manage_rules',
'manage_alerts',
],
manageReporting: ['all', 'read', 'minimal_all', 'minimal_read'],
apm: ['all', 'read', 'minimal_all', 'minimal_read', 'settings_save'],
discover: [

View file

@ -100,6 +100,7 @@ export default function ({ getService }: FtrProviderContext) {
'alerting:apm.error_rate',
'alerting:apm.transaction_duration',
'alerting:apm.transaction_error_rate',
'alerting:datasetQuality.degradedDocs',
'alerting:logs.alert.document.count',
'alerting:metrics.alert.inventory.threshold',
'alerting:metrics.alert.threshold',

View file

@ -5,37 +5,37 @@
* 2.0.
*/
import { COMPARATORS } from '@kbn/alerting-comparators';
import { i18n } from '@kbn/i18n';
import {
ALERT_EVALUATION_THRESHOLD,
ALERT_EVALUATION_VALUE,
ALERT_EVALUATION_VALUES,
ALERT_RULE_PARAMETERS,
ALERT_RULE_TYPE_ID,
METRIC_INVENTORY_THRESHOLD_ALERT_TYPE_ID,
OBSERVABILITY_THRESHOLD_RULE_TYPE_ID,
LOG_THRESHOLD_ALERT_TYPE_ID,
ALERT_EVALUATION_THRESHOLD,
ApmRuleType,
SLO_BURN_RATE_RULE_TYPE_ID,
DEGRADED_DOCS_RULE_TYPE_ID,
LOG_THRESHOLD_ALERT_TYPE_ID,
METRIC_INVENTORY_THRESHOLD_ALERT_TYPE_ID,
METRIC_THRESHOLD_ALERT_TYPE_ID,
OBSERVABILITY_THRESHOLD_RULE_TYPE_ID,
SLO_BURN_RATE_RULE_TYPE_ID,
} from '@kbn/rule-data-utils';
import { EsQueryRuleParams } from '@kbn/stack-alerts-plugin/public/rule_types/es_query/types';
import { i18n } from '@kbn/i18n';
import { COMPARATORS } from '@kbn/alerting-comparators';
import { asDuration, asPercent, convertToBuiltInComparators } from '../../../../common';
import { createFormatter } from '../../../../common/custom_threshold_rule/formatters';
import { METRIC_FORMATTERS } from '../../../../common/custom_threshold_rule/formatters/snapshot_metric_formats';
import { metricValueFormatter } from '../../../../common/custom_threshold_rule/metric_value_formatter';
import {
BaseMetricExpressionParams,
CustomMetricExpressionParams,
} from '../../../../common/custom_threshold_rule/types';
import {
ABOVE_OR_EQ_TEXT,
ABOVE_TEXT,
BELOW_OR_EQ_TEXT,
BELOW_TEXT,
} from '../../../../common/i18n';
import { asDuration, asPercent, convertToBuiltInComparators } from '../../../../common';
import { createFormatter } from '../../../../common/custom_threshold_rule/formatters';
import { metricValueFormatter } from '../../../../common/custom_threshold_rule/metric_value_formatter';
import { METRIC_FORMATTERS } from '../../../../common/custom_threshold_rule/formatters/snapshot_metric_formats';
import {
BaseMetricExpressionParams,
CustomMetricExpressionParams,
} from '../../../../common/custom_threshold_rule/types';
import { TopAlert } from '../../../typings/alerts';
import { isFieldsSameType } from './is_fields_same_type';
export interface FlyoutThresholdData {
@ -308,6 +308,15 @@ export const mapRuleParamsWithFlyout = (alert: TopAlert): FlyoutThresholdData[]
),
} as unknown as FlyoutThresholdData;
return [SLOBurnRateFlyoutMap];
case DEGRADED_DOCS_RULE_TYPE_ID:
const DegradedDocsFlyoutMap = {
observedValue: [asPercent(alert.fields[ALERT_EVALUATION_VALUE], 100)],
threshold: [asPercent(alert.fields[ALERT_EVALUATION_THRESHOLD], 100)],
comparator: ruleParams.comparator,
} as unknown as FlyoutThresholdData;
return [DegradedDocsFlyoutMap];
default:
return [];
}

View file

@ -5,8 +5,12 @@
* 2.0.
*/
import {
DEGRADED_DOCS_RULE_TYPE_ID,
ES_QUERY_ID,
ML_ANOMALY_DETECTION_RULE_TYPE_ID,
} from '@kbn/rule-data-utils';
import { useMemo } from 'react';
import { ES_QUERY_ID, ML_ANOMALY_DETECTION_RULE_TYPE_ID } from '@kbn/rule-data-utils';
import { usePluginContext } from './use_plugin_context';
export function useGetFilteredRuleTypes() {
@ -15,6 +19,7 @@ export function useGetFilteredRuleTypes() {
return useMemo(() => {
return [
ES_QUERY_ID,
DEGRADED_DOCS_RULE_TYPE_ID,
ML_ANOMALY_DETECTION_RULE_TYPE_ID,
...observabilityRuleTypeRegistry.list(),
];

View file

@ -0,0 +1,184 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
import { log, timerange } from '@kbn/apm-synthtrace-client';
import expect from '@kbn/expect';
import rison from '@kbn/rison';
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
import { SupertestWithRoleScopeType } from '../../../services';
const MORE_THAN_1024_CHARS =
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
const roleScopedSupertest = getService('roleScopedSupertest');
const synthtrace = getService('synthtrace');
const start = '2025-04-11T08:00:00.000Z';
const end = '2025-04-11T08:05:30.000Z';
const type = 'logs';
const dataset = 'nginx.access';
const namespace = 'default';
const serviceName = 'my-service';
const otherServiceName = 'my-other-service';
async function callApiAs(
roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType,
dataStream: string,
groupBy = ['_index'],
interval = '1m'
) {
return roleScopedSupertestWithCookieCredentials
.get(`/internal/dataset_quality/rule_types/degraded_docs/chart_preview`)
.query({
index: dataStream,
groupBy: rison.encodeArray(groupBy),
start: new Date(start).getTime(),
end: new Date(end).getTime(),
interval,
});
}
describe('Chart preview per DataStream', function () {
let synthtraceLogsEsClient: LogsSynthtraceEsClient;
let supertestEditorWithCookieCredentials: SupertestWithRoleScopeType;
before(async () => {
synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient();
supertestEditorWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
'editor',
{
useCookieHeader: true,
withInternalHeaders: true,
}
);
});
describe('gets the degraded fields timeseries per data stream', () => {
before(async () => {
await synthtraceLogsEsClient.index(
Array.from({ length: 6 }, (_, i) =>
timerange(start, end)
.interval('1m')
.rate(1)
.generator((timestamp, j) =>
log
.create()
.message('This is a log message')
.timestamp(timestamp)
.dataset(`${dataset}.${i}`)
.namespace(namespace)
.logLevel(i && j % i === 0 ? MORE_THAN_1024_CHARS : 'error')
.defaults({
'log.file.path': '/error.log',
'service.name': j % 2 ? serviceName + i : otherServiceName + 1,
'trace.id': i && j % i === 0 ? MORE_THAN_1024_CHARS : 'trace-id',
})
)
)
);
});
after(async () => {
await synthtraceLogsEsClient.clean();
});
it('returns proper timeSeries data for degraded fields when querying a single dataStream', async () => {
const logsTimeSeriesData = [
{
name: 'logs-nginx.access.5-default',
data: [
{ x: 1744358400000, y: 100 },
{ x: 1744358460000, y: 0 },
{ x: 1744358520000, y: 0 },
{ x: 1744358580000, y: 0 },
{ x: 1744358640000, y: 0 },
{ x: 1744358700000, y: 100 },
],
},
];
const resp = await callApiAs(
supertestEditorWithCookieCredentials,
`${type}-${dataset}.5-*`
);
expect(resp.body.series).to.eql(logsTimeSeriesData);
});
it('returns proper timeSeries data when querying at a specific interval', async () => {
const logsTimeSeriesData = [
{
name: 'logs-nginx.access.1-default',
data: [
{ x: 1744358400000, y: 100 },
{ x: 1744358700000, y: 100 },
],
},
];
const resp = await callApiAs(
supertestEditorWithCookieCredentials,
`${type}-${dataset}.1-*`,
['_index'],
'5m'
);
expect(resp.body.series).to.eql(logsTimeSeriesData);
});
it('returns proper timeSeries data grouped using multiple keys', async () => {
const logsTimeSeriesData = [
{
name: 'logs-nginx.access.1-default,my-other-service1',
data: [
{ x: 1744358400000, y: 100 },
{ x: 1744358460000, y: 0 },
{ x: 1744358520000, y: 100 },
{ x: 1744358580000, y: 0 },
{ x: 1744358640000, y: 100 },
{ x: 1744358700000, y: 0 },
],
},
{
name: 'logs-nginx.access.1-default,my-service1',
data: [
{ x: 1744358400000, y: 0 },
{ x: 1744358460000, y: 100 },
{ x: 1744358520000, y: 0 },
{ x: 1744358580000, y: 100 },
{ x: 1744358640000, y: 0 },
{ x: 1744358700000, y: 100 },
],
},
];
const resp = await callApiAs(
supertestEditorWithCookieCredentials,
`${type}-${dataset}.1-*`,
['_index', 'service.name']
);
expect(resp.body.series).to.eql(logsTimeSeriesData);
});
it('returns maximum 5 proper timeseries but the totalGroups indicates that there were more', async () => {
const resp = await callApiAs(
supertestEditorWithCookieCredentials,
`${type}-${dataset}.*-*`
);
expect(resp.body.series.length).to.be(5);
expect(resp.body.totalGroups).to.be(6);
});
it('returns empty when dataStream does not exists or does not have data reported', async () => {
const resp = await callApiAs(supertestEditorWithCookieCredentials, `${type}-other-*`);
expect(resp.body.series.length).to.be(0);
expect(resp.body.totalGroups).to.be(0);
});
});
});
}

View file

@ -9,20 +9,21 @@ import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_cont
export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
describe('Dataset quality', () => {
loadTestFile(require.resolve('./integrations'));
loadTestFile(require.resolve('./integration_dashboards'));
loadTestFile(require.resolve('./degraded_field_analyze'));
loadTestFile(require.resolve('./data_stream_settings'));
loadTestFile(require.resolve('./data_stream_rollover'));
loadTestFile(require.resolve('./update_field_limit'));
loadTestFile(require.resolve('./chart_preview'));
loadTestFile(require.resolve('./check_and_load_integration'));
loadTestFile(require.resolve('./data_stream_details'));
loadTestFile(require.resolve('./data_stream_rollover'));
loadTestFile(require.resolve('./data_stream_settings'));
loadTestFile(require.resolve('./data_stream_total_docs'));
loadTestFile(require.resolve('./degraded_docs'));
loadTestFile(require.resolve('./degraded_fields'));
loadTestFile(require.resolve('./data_stream_details'));
loadTestFile(require.resolve('./degraded_field_analyze'));
loadTestFile(require.resolve('./degraded_field_values'));
loadTestFile(require.resolve('./failed_docs'));
loadTestFile(require.resolve('./degraded_fields'));
loadTestFile(require.resolve('./failed_docs_errors'));
loadTestFile(require.resolve('./failed_docs_stats'));
loadTestFile(require.resolve('./failed_docs'));
loadTestFile(require.resolve('./integration_dashboards'));
loadTestFile(require.resolve('./integrations'));
loadTestFile(require.resolve('./update_field_limit'));
});
}

View file

@ -418,8 +418,9 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
dataStream: regularDataStreamName,
});
const discoverButton =
await PageObjects.datasetQuality.getDatasetQualityDetailsHeaderButton();
await PageObjects.datasetQuality.openDatasetQualityDetailsActionsButton();
const discoverButton = await PageObjects.datasetQuality.getOpenInDiscoverButton();
await discoverButton.click();

View file

@ -156,7 +156,6 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
datasetQualityInsufficientPrivileges: 'datasetQualityInsufficientPrivileges',
datasetQualityNoDataEmptyState: 'datasetQualityTableNoData',
datasetQualityNoPrivilegesEmptyState: 'datasetQualityNoPrivilegesEmptyState',
superDatePickerToggleQuickMenuButton: 'superDatePickerToggleQuickMenuButton',
superDatePickerApplyTimeButton: 'superDatePickerApplyTimeButton',
superDatePickerQuickMenu: 'superDatePickerQuickMenu',
@ -170,6 +169,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist',
datasetQualityDetailsOverviewDegradedFieldToggleSwitch:
'datasetQualityDetailsOverviewDegradedFieldToggleSwitch',
datasetQualityDetailsActionsDropdown: 'datasetQualityDetailsActionsDropdown',
openInDiscover: 'openInDiscover',
};
return {
@ -437,6 +438,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
return testSubjects.find(testSubjectSelectors.datasetQualityDetailsHeaderButton);
},
openDatasetQualityDetailsActionsButton() {
return testSubjects.click(testSubjectSelectors.datasetQualityDetailsActionsDropdown);
},
getOpenInDiscoverButton() {
return testSubjects.find(testSubjectSelectors.openInDiscover);
},
openIntegrationActionsMenu() {
return testSubjects.click(testSubjectSelectors.datasetQualityDetailsIntegrationActionsButton);
},