mirror of
https://github.com/elastic/kibana.git
synced 2025-06-27 18:51:07 -04:00
[Dataset quality] degradedDocs rule (#216026)
Closes https://github.com/elastic/kibana/issues/179173. ## Rule type A dedicated stack rule type was created `datasetQuality.degradedDocs`. <img width="1759" alt="image" src="https://github.com/user-attachments/assets/5004a08d-6f12-4f5e-b27f-5f4db242dcf0" /> <img width="2318" alt="image" src="https://github.com/user-attachments/assets/f8b2664a-f1c6-48c5-a617-c6f1b79bf0f7" /> This new rule is aggregated by default using `_index` and could be further configured by the user (e.g. user can also aggregate by `cloud.provider`). A new rule type was needed to be created since there is no actual way to aggregate all documents in a dataStream if we use a DataView like `logs-*-*`. Inside datasStream documents there is no indication about the dataStream where they belong to, instead we just have `_index` which contains backingIndexName instead of actual index. It's important to note, that this rule type is also visible from `Observability > Alerts`, which is useful specially for serverless. https://github.com/user-attachments/assets/000aee51-4895-4f4c-9484-924ace4325c5 ## Role Based Access-Control (RBAC) RBAC for dataset quality alerts is defined within dataQuality kibana feature. We have three privileges defined: 1. `all`: This privilege now contains a subFeature `manage_rules` that will allow for more granularity on alerting level. It's by default assigned to `all` but can be disabled. 2. `read`: This privilege is only related to serverless (when we don't have yet custom roles). https://github.com/user-attachments/assets/70ed5bde-bf45-4024-b448-228799fcaf71 3. `none`: This privilege is only relevant for stateful (in serverless we don't have custom roles). ## 🎥 Demo ### Serverless #### `all` privileges https://github.com/user-attachments/assets/8dad6e30-a261-4a69-979f-6dfc2a41c888 #### `read` privileges https://github.com/user-attachments/assets/e1cb108d-22a0-4e7f-b252-9cc12d1e9d65 ### Stateful #### `all` privileges https://github.com/user-attachments/assets/d96f3b70-35b2-466b-aa59-a07190d24d93 #### `all` privileges with subFeature disabled https://github.com/user-attachments/assets/808ab811-9320-43e4-b2a6-06d530a78b82 #### `none` privileges (Stateful) https://github.com/user-attachments/assets/18f2a2d6-d825-4713-acea-0d72f451e9ab ## How to test? 1. run synthrace scenario `degraded_logs` in live mode ``` node scripts/synthtrace degraded_logs --live ``` 2. Open dataset quality page (/app/management/data/data_quality) 3. Select `synth.3` dataset (/app/management/data/data_quality/details?pageState=(dataStream:logs-synth.3-default) 4. Click on `Actions` and then select `Create rule` 5. Fill out the alert form 6. Go to `Observability > Alerts` or `Stack management > Alerts` (/app/observability/alerts) ## Release note Adds the Create alert rule action to dataset quality page and dataset quality details. This allows you to generate an alert when the percentage of degraded docs on the chart crosses a certain threshold. --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Aleh Zasypkin <aleh.zasypkin@elastic.co> Co-authored-by: Faisal Kanout <faisal.kanout@elastic.co>
This commit is contained in:
parent
2c8ec79abf
commit
64df229998
59 changed files with 2964 additions and 119 deletions
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
// ---------------------------------- WARNING ----------------------------------
|
||||
// this file was generated, and should not be edited by hand
|
||||
// ---------------------------------- WARNING ----------------------------------
|
||||
import * as rt from 'io-ts';
|
||||
import type { Either } from 'fp-ts/Either';
|
||||
import { AlertSchema } from './alert_schema';
|
||||
import { EcsSchema } from './ecs_schema';
|
||||
const ISO_DATE_PATTERN = /^d{4}-d{2}-d{2}Td{2}:d{2}:d{2}.d{3}Z$/;
|
||||
export const IsoDateString = new rt.Type<string, string, unknown>(
|
||||
'IsoDateString',
|
||||
rt.string.is,
|
||||
(input, context): Either<rt.Errors, string> => {
|
||||
if (typeof input === 'string' && ISO_DATE_PATTERN.test(input)) {
|
||||
return rt.success(input);
|
||||
} else {
|
||||
return rt.failure(input, context);
|
||||
}
|
||||
},
|
||||
rt.identity
|
||||
);
|
||||
export type IsoDateStringC = typeof IsoDateString;
|
||||
export const schemaUnknown = rt.unknown;
|
||||
export const schemaUnknownArray = rt.array(rt.unknown);
|
||||
export const schemaString = rt.string;
|
||||
export const schemaStringArray = rt.array(schemaString);
|
||||
export const schemaNumber = rt.number;
|
||||
export const schemaNumberArray = rt.array(schemaNumber);
|
||||
export const schemaDate = rt.union([IsoDateString, schemaNumber]);
|
||||
export const schemaDateArray = rt.array(schemaDate);
|
||||
export const schemaDateRange = rt.partial({
|
||||
gte: schemaDate,
|
||||
lte: schemaDate,
|
||||
});
|
||||
export const schemaDateRangeArray = rt.array(schemaDateRange);
|
||||
export const schemaStringOrNumber = rt.union([schemaString, schemaNumber]);
|
||||
export const schemaStringOrNumberArray = rt.array(schemaStringOrNumber);
|
||||
export const schemaBoolean = rt.boolean;
|
||||
export const schemaBooleanArray = rt.array(schemaBoolean);
|
||||
const schemaGeoPointCoords = rt.type({
|
||||
type: schemaString,
|
||||
coordinates: schemaNumberArray,
|
||||
});
|
||||
const schemaGeoPointString = schemaString;
|
||||
const schemaGeoPointLatLon = rt.type({
|
||||
lat: schemaNumber,
|
||||
lon: schemaNumber,
|
||||
});
|
||||
const schemaGeoPointLocation = rt.type({
|
||||
location: schemaNumberArray,
|
||||
});
|
||||
const schemaGeoPointLocationString = rt.type({
|
||||
location: schemaString,
|
||||
});
|
||||
export const schemaGeoPoint = rt.union([
|
||||
schemaGeoPointCoords,
|
||||
schemaGeoPointString,
|
||||
schemaGeoPointLatLon,
|
||||
schemaGeoPointLocation,
|
||||
schemaGeoPointLocationString,
|
||||
]);
|
||||
export const schemaGeoPointArray = rt.array(schemaGeoPoint);
|
||||
// prettier-ignore
|
||||
const DatasetQualityAlertRequired = rt.type({
|
||||
});
|
||||
// prettier-ignore
|
||||
const DatasetQualityAlertOptional = rt.partial({
|
||||
'kibana.alert.evaluation.threshold': schemaStringOrNumber,
|
||||
'kibana.alert.evaluation.value': schemaString,
|
||||
'kibana.alert.grouping': schemaUnknown,
|
||||
'kibana.alert.reason': schemaString,
|
||||
});
|
||||
|
||||
// prettier-ignore
|
||||
export const DatasetQualityAlertSchema = rt.intersection([DatasetQualityAlertRequired, DatasetQualityAlertOptional, AlertSchema, EcsSchema]);
|
||||
// prettier-ignore
|
||||
export type DatasetQualityAlert = rt.TypeOf<typeof DatasetQualityAlertSchema>;
|
|
@ -9,19 +9,18 @@
|
|||
|
||||
export interface RouterLinkProps {
|
||||
href: string | undefined;
|
||||
onClick: (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement, MouseEvent>) => void;
|
||||
onClick: (event: React.MouseEvent<Element, MouseEvent>) => void;
|
||||
}
|
||||
|
||||
interface GetRouterLinkPropsDeps {
|
||||
href?: string;
|
||||
onClick(event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>): void;
|
||||
onClick(event: React.MouseEvent<Element, MouseEvent>): void;
|
||||
}
|
||||
|
||||
const isModifiedEvent = (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) =>
|
||||
const isModifiedEvent = (event: React.MouseEvent<Element, MouseEvent>) =>
|
||||
!!(event.metaKey || event.altKey || event.ctrlKey || event.shiftKey);
|
||||
|
||||
const isLeftClickEvent = (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) =>
|
||||
event.button === 0;
|
||||
const isLeftClickEvent = (event: React.MouseEvent<Element, MouseEvent>) => event.button === 0;
|
||||
|
||||
/**
|
||||
*
|
||||
|
@ -36,7 +35,7 @@ const isLeftClickEvent = (event: React.MouseEvent<HTMLAnchorElement | HTMLButton
|
|||
* manage behaviours such as leftClickEvent and event with modifiers (Ctrl, Shift, etc)
|
||||
*/
|
||||
export const getRouterLinkProps = ({ href, onClick }: GetRouterLinkPropsDeps): RouterLinkProps => {
|
||||
const guardedClickHandler = (event: React.MouseEvent<HTMLAnchorElement | HTMLButtonElement>) => {
|
||||
const guardedClickHandler = (event: React.MouseEvent<Element, MouseEvent>) => {
|
||||
if (event.defaultPrevented) {
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -16,3 +16,4 @@ export * from './src/alerts_as_data_status';
|
|||
export * from './src/alerts_as_data_cases';
|
||||
export * from './src/routes/stack_rule_paths';
|
||||
export * from './src/rule_types';
|
||||
export * from './src/rule_constants';
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
export const ALERT_KEY_JOINER = ',';
|
|
@ -11,6 +11,8 @@ export const STACK_ALERTS_FEATURE_ID = 'stackAlerts';
|
|||
export const ES_QUERY_ID = '.es-query';
|
||||
export const ML_ANOMALY_DETECTION_RULE_TYPE_ID = 'xpack.ml.anomaly_detection_alert';
|
||||
|
||||
export const DEGRADED_DOCS_RULE_TYPE_ID = 'datasetQuality.degradedDocs';
|
||||
|
||||
/**
|
||||
* These rule types are not the only stack rules. There are more.
|
||||
* The variable holds all stack rule types that support multiple
|
||||
|
@ -19,4 +21,5 @@ export const ML_ANOMALY_DETECTION_RULE_TYPE_ID = 'xpack.ml.anomaly_detection_ale
|
|||
export const STACK_RULE_TYPE_IDS_SUPPORTED_BY_OBSERVABILITY = [
|
||||
ES_QUERY_ID,
|
||||
ML_ANOMALY_DETECTION_RULE_TYPE_ID,
|
||||
DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
];
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
export { degradedDocsParamsSchema } from './latest';
|
||||
|
||||
export type { DegradedDocsRuleParams } from './latest';
|
|
@ -0,0 +1,10 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
export * from './v1';
|
|
@ -0,0 +1,30 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the "Elastic License
|
||||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
|
||||
* Public License v 1"; you may not use this file except in compliance with, at
|
||||
* your election, the "Elastic License 2.0", the "GNU Affero General Public
|
||||
* License v3.0 only", or the "Server Side Public License, v 1".
|
||||
*/
|
||||
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
import type { TypeOf } from '@kbn/config-schema';
|
||||
import { schema } from '@kbn/config-schema';
|
||||
import { oneOfLiterals } from '../common/utils';
|
||||
|
||||
const comparators = Object.values(COMPARATORS);
|
||||
|
||||
const searchConfigSchema = schema.object({
|
||||
index: schema.string(),
|
||||
});
|
||||
|
||||
export const degradedDocsParamsSchema = schema.object({
|
||||
timeUnit: schema.string(),
|
||||
timeSize: schema.number(),
|
||||
threshold: schema.arrayOf(schema.number()),
|
||||
comparator: oneOfLiterals(comparators),
|
||||
groupBy: schema.maybe(schema.arrayOf(schema.string())),
|
||||
searchConfiguration: searchConfigSchema,
|
||||
});
|
||||
|
||||
export type DegradedDocsRuleParams = TypeOf<typeof degradedDocsParamsSchema>;
|
|
@ -473,6 +473,46 @@ Object {
|
|||
}
|
||||
`;
|
||||
|
||||
exports[`Alert as data fields checks detect AAD fields changes for: datasetQuality.degradedDocs 1`] = `
|
||||
Object {
|
||||
"dynamicTemplates": Array [
|
||||
Object {
|
||||
"strings_as_keywords": Object {
|
||||
"mapping": Object {
|
||||
"ignore_above": 1024,
|
||||
"type": "keyword",
|
||||
},
|
||||
"match_mapping_type": "string",
|
||||
"path_match": "kibana.alert.grouping.*",
|
||||
},
|
||||
},
|
||||
],
|
||||
"fieldMap": Object {
|
||||
"kibana.alert.evaluation.threshold": Object {
|
||||
"required": false,
|
||||
"scaling_factor": 100,
|
||||
"type": "scaled_float",
|
||||
},
|
||||
"kibana.alert.evaluation.value": Object {
|
||||
"array": false,
|
||||
"required": false,
|
||||
"type": "keyword",
|
||||
},
|
||||
"kibana.alert.grouping": Object {
|
||||
"array": false,
|
||||
"dynamic": true,
|
||||
"required": false,
|
||||
"type": "object",
|
||||
},
|
||||
"kibana.alert.reason": Object {
|
||||
"array": false,
|
||||
"required": false,
|
||||
"type": "keyword",
|
||||
},
|
||||
},
|
||||
}
|
||||
`;
|
||||
|
||||
exports[`Alert as data fields checks detect AAD fields changes for: logs.alert.document.count 1`] = `
|
||||
Object {
|
||||
"fieldMap": Object {
|
||||
|
|
|
@ -64,6 +64,7 @@ const ruleTypes: string[] = [
|
|||
'siem.thresholdRule',
|
||||
'siem.newTermsRule',
|
||||
'siem.notifications',
|
||||
'datasetQuality.degradedDocs',
|
||||
];
|
||||
|
||||
describe('Alert as data fields checks', () => {
|
||||
|
|
|
@ -9,7 +9,7 @@ import { setStateToKbnUrl } from '@kbn/kibana-utils-plugin/common';
|
|||
import { ManagementAppLocatorParams } from '@kbn/management-plugin/common/locator';
|
||||
import { LocatorPublic } from '@kbn/share-plugin/common';
|
||||
import { DataQualityDetailsLocatorParams } from '@kbn/deeplinks-observability';
|
||||
import { datasetQualityDetailsUrlSchemaV1, DATA_QUALITY_URL_STATE_KEY } from '../url_schema';
|
||||
import { datasetQualityDetailsUrlSchemaV2, DATA_QUALITY_URL_STATE_KEY } from '../url_schema';
|
||||
import { deepCompactObject } from '../utils/deep_compact_object';
|
||||
|
||||
interface LocatorPathConstructionParams {
|
||||
|
@ -23,9 +23,9 @@ export const constructDatasetQualityDetailsLocatorPath = async (
|
|||
) => {
|
||||
const { locatorParams, useHash, managementLocator } = params;
|
||||
|
||||
const pageState = datasetQualityDetailsUrlSchemaV1.urlSchemaRT.encode(
|
||||
const pageState = datasetQualityDetailsUrlSchemaV2.urlSchemaRT.encode(
|
||||
deepCompactObject({
|
||||
v: 1,
|
||||
v: 2,
|
||||
...locatorParams,
|
||||
})
|
||||
);
|
||||
|
|
|
@ -10,15 +10,74 @@ import {
|
|||
KibanaFeatureConfig,
|
||||
KibanaFeatureScope,
|
||||
ElasticsearchFeatureConfig,
|
||||
SubFeaturePrivilegeGroupConfig,
|
||||
SubFeaturePrivilegeGroupType,
|
||||
} from '@kbn/features-plugin/common';
|
||||
import { AlertConsumers, DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { PLUGIN_FEATURE_ID, PLUGIN_ID, PLUGIN_NAME } from '../common';
|
||||
|
||||
const degradedDocsAlertingFeatures = {
|
||||
ruleTypeId: DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
consumers: [AlertConsumers.ALERTS],
|
||||
};
|
||||
|
||||
const canManageRules: SubFeaturePrivilegeGroupConfig = {
|
||||
groupType: 'independent' as SubFeaturePrivilegeGroupType,
|
||||
privileges: [
|
||||
{
|
||||
id: 'manage_rules',
|
||||
name: i18n.translate('xpack.dataQuality.features.canManageRules', {
|
||||
defaultMessage: 'Manage rules',
|
||||
}),
|
||||
includeIn: 'all',
|
||||
alerting: {
|
||||
rule: {
|
||||
all: [degradedDocsAlertingFeatures],
|
||||
},
|
||||
},
|
||||
savedObject: {
|
||||
all: [],
|
||||
read: [],
|
||||
},
|
||||
ui: ['alerting:save'],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const canManageAlerts: SubFeaturePrivilegeGroupConfig = {
|
||||
groupType: 'independent' as SubFeaturePrivilegeGroupType,
|
||||
privileges: [
|
||||
{
|
||||
id: 'manage_alerts',
|
||||
name: i18n.translate('xpack.dataQuality.features.canManageAlerts', {
|
||||
defaultMessage: 'Manage alerts',
|
||||
}),
|
||||
includeIn: 'all',
|
||||
alerting: {
|
||||
alert: {
|
||||
all: [degradedDocsAlertingFeatures],
|
||||
},
|
||||
},
|
||||
savedObject: {
|
||||
all: [],
|
||||
read: [],
|
||||
},
|
||||
ui: ['alerting:save'],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const KIBANA_FEATURE: KibanaFeatureConfig = {
|
||||
id: PLUGIN_FEATURE_ID,
|
||||
name: PLUGIN_NAME,
|
||||
category: DEFAULT_APP_CATEGORIES.management,
|
||||
scope: [KibanaFeatureScope.Spaces, KibanaFeatureScope.Security],
|
||||
app: [PLUGIN_ID],
|
||||
alerting: [degradedDocsAlertingFeatures],
|
||||
management: {
|
||||
insightsAndAlerting: ['triggersActions'],
|
||||
},
|
||||
privileges: {
|
||||
all: {
|
||||
app: [PLUGIN_ID],
|
||||
|
@ -27,6 +86,17 @@ export const KIBANA_FEATURE: KibanaFeatureConfig = {
|
|||
read: [],
|
||||
},
|
||||
ui: ['show'],
|
||||
alerting: {
|
||||
rule: {
|
||||
read: [degradedDocsAlertingFeatures],
|
||||
},
|
||||
alert: {
|
||||
all: [degradedDocsAlertingFeatures],
|
||||
},
|
||||
},
|
||||
management: {
|
||||
insightsAndAlerting: ['triggersActions'],
|
||||
},
|
||||
},
|
||||
read: {
|
||||
disabled: true,
|
||||
|
@ -35,8 +105,36 @@ export const KIBANA_FEATURE: KibanaFeatureConfig = {
|
|||
read: [],
|
||||
},
|
||||
ui: ['show'],
|
||||
alerting: {
|
||||
rule: {
|
||||
read: [degradedDocsAlertingFeatures],
|
||||
},
|
||||
alert: {
|
||||
read: [degradedDocsAlertingFeatures],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
subFeatures: [
|
||||
{
|
||||
name: i18n.translate('xpack.dataQuality.features.app.manageRules', {
|
||||
defaultMessage: 'Manage rules',
|
||||
}),
|
||||
description: i18n.translate('xpack.dataQuality.features.app.manageRulesDescription', {
|
||||
defaultMessage: 'This feature enables users to manage dataset quality rules.',
|
||||
}),
|
||||
privilegeGroups: [canManageRules],
|
||||
},
|
||||
{
|
||||
name: i18n.translate('xpack.dataQuality.features.app.manageAlerts', {
|
||||
defaultMessage: 'Manage alerts',
|
||||
}),
|
||||
description: i18n.translate('xpack.dataQuality.features.app.manageAlertsDescription', {
|
||||
defaultMessage: 'This feature enables users to manage dataset quality alerts.',
|
||||
}),
|
||||
privilegeGroups: [canManageAlerts],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const ELASTICSEARCH_FEATURE: ElasticsearchFeatureConfig = {
|
||||
|
|
|
@ -7,13 +7,37 @@
|
|||
|
||||
import { CoreSetup, Plugin } from '@kbn/core/server';
|
||||
|
||||
import { ManagementAppLocatorParams } from '@kbn/management-plugin/common/locator';
|
||||
import { MANAGEMENT_APP_LOCATOR } from '@kbn/deeplinks-management/constants';
|
||||
import { Dependencies } from './types';
|
||||
import { ELASTICSEARCH_FEATURE, KIBANA_FEATURE } from './features';
|
||||
import {
|
||||
DatasetQualityDetailsLocatorDefinition,
|
||||
DatasetQualityLocatorDefinition,
|
||||
} from '../common/locators';
|
||||
|
||||
export class DataQualityPlugin implements Plugin<void, void, any, any> {
|
||||
public setup(_coreSetup: CoreSetup, { features }: Dependencies) {
|
||||
public setup(_coreSetup: CoreSetup, { features, share }: Dependencies) {
|
||||
features.registerKibanaFeature(KIBANA_FEATURE);
|
||||
features.registerElasticsearchFeature(ELASTICSEARCH_FEATURE);
|
||||
|
||||
const managementLocator =
|
||||
share.url.locators.get<ManagementAppLocatorParams>(MANAGEMENT_APP_LOCATOR);
|
||||
|
||||
if (managementLocator) {
|
||||
share.url.locators.create(
|
||||
new DatasetQualityLocatorDefinition({
|
||||
useHash: false,
|
||||
managementLocator,
|
||||
})
|
||||
);
|
||||
share.url.locators.create(
|
||||
new DatasetQualityDetailsLocatorDefinition({
|
||||
useHash: false,
|
||||
managementLocator,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public start() {}
|
||||
|
|
|
@ -6,7 +6,11 @@
|
|||
*/
|
||||
|
||||
import { FeaturesPluginSetup } from '@kbn/features-plugin/server';
|
||||
import type { ManagementSetup } from '@kbn/management-plugin/public/types';
|
||||
import { SharePluginSetup } from '@kbn/share-plugin/server';
|
||||
|
||||
export interface Dependencies {
|
||||
features: FeaturesPluginSetup;
|
||||
management: ManagementSetup;
|
||||
share: SharePluginSetup;
|
||||
}
|
||||
|
|
|
@ -29,6 +29,7 @@
|
|||
"@kbn/deeplinks-observability",
|
||||
"@kbn/ebt-tools",
|
||||
"@kbn/core-application-common",
|
||||
"@kbn/rule-data-utils",
|
||||
],
|
||||
"exclude": ["target/**/*"]
|
||||
}
|
||||
|
|
|
@ -275,3 +275,20 @@ export const getNonAggregatableDatasetsRt = rt.exact(
|
|||
);
|
||||
|
||||
export type NonAggregatableDatasets = rt.TypeOf<typeof getNonAggregatableDatasetsRt>;
|
||||
|
||||
export const getPreviewChartResponseRt = rt.type({
|
||||
series: rt.array(
|
||||
rt.type({
|
||||
name: rt.string,
|
||||
data: rt.array(
|
||||
rt.type({
|
||||
x: rt.number,
|
||||
y: rt.union([rt.number, rt.null]),
|
||||
})
|
||||
),
|
||||
})
|
||||
),
|
||||
totalGroups: rt.number,
|
||||
});
|
||||
|
||||
export type PreviewChartResponse = rt.TypeOf<typeof getPreviewChartResponseRt>;
|
||||
|
|
|
@ -715,3 +715,7 @@ export const readLess = i18n.translate(
|
|||
defaultMessage: 'Read less',
|
||||
}
|
||||
);
|
||||
|
||||
export const createAlertText = i18n.translate('xpack.datasetQuality.createAlert', {
|
||||
defaultMessage: 'Create rule',
|
||||
});
|
||||
|
|
|
@ -16,25 +16,30 @@
|
|||
"datasetQuality"
|
||||
],
|
||||
"requiredPlugins": [
|
||||
"controls",
|
||||
"data",
|
||||
"dataViews",
|
||||
"dataViewEditor",
|
||||
"embeddable",
|
||||
"fieldFormats",
|
||||
"fieldsMetadata",
|
||||
"fleet",
|
||||
"kibanaReact",
|
||||
"kibanaUtils",
|
||||
"controls",
|
||||
"embeddable",
|
||||
"share",
|
||||
"fleet",
|
||||
"fieldFormats",
|
||||
"dataViews",
|
||||
"lens",
|
||||
"fieldsMetadata",
|
||||
"share",
|
||||
"taskManager",
|
||||
"usageCollection"
|
||||
"triggersActionsUi",
|
||||
],
|
||||
"optionalPlugins": [
|
||||
"telemetry"
|
||||
"alerting",
|
||||
"telemetry",
|
||||
"usageCollection",
|
||||
],
|
||||
"requiredBundles": [
|
||||
"discover"
|
||||
"charts",
|
||||
"discover",
|
||||
"stackAlerts",
|
||||
],
|
||||
"extraPublicDirs": [
|
||||
"common"
|
||||
|
|
|
@ -0,0 +1,63 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { RuleFormFlyout } from '@kbn/response-ops-rule-form/flyout';
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
|
||||
import { useKibanaContextForPlugin } from '../utils/use_kibana';
|
||||
|
||||
interface Props {
|
||||
addFlyoutVisible: boolean;
|
||||
setAddFlyoutVisibility: React.Dispatch<React.SetStateAction<boolean>>;
|
||||
dataStream?: string;
|
||||
}
|
||||
|
||||
export function AlertFlyout(props: Props) {
|
||||
const { addFlyoutVisible, setAddFlyoutVisibility, dataStream } = props;
|
||||
|
||||
const {
|
||||
services: {
|
||||
triggersActionsUi: { actionTypeRegistry, ruleTypeRegistry },
|
||||
...services
|
||||
},
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const [initialValues, setInitialValues] = useState<{ searchConfiguration?: { index: string } }>(
|
||||
{}
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (dataStream) {
|
||||
setInitialValues({ searchConfiguration: { index: dataStream } });
|
||||
}
|
||||
}, [dataStream]);
|
||||
|
||||
const onCloseAddFlyout = useCallback(
|
||||
() => setAddFlyoutVisibility(false),
|
||||
[setAddFlyoutVisibility]
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
{addFlyoutVisible && (
|
||||
<RuleFormFlyout
|
||||
plugins={{
|
||||
...services,
|
||||
ruleTypeRegistry,
|
||||
actionTypeRegistry,
|
||||
}}
|
||||
onCancel={onCloseAddFlyout}
|
||||
onSubmit={onCloseAddFlyout}
|
||||
ruleTypeId={DEGRADED_DOCS_RULE_TYPE_ID}
|
||||
initialValues={{
|
||||
params: initialValues,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
|
@ -0,0 +1,22 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { Capabilities } from '@kbn/core/public';
|
||||
import type { PluginStartContract as AlertingPublicStart } from '@kbn/alerting-plugin/public/plugin';
|
||||
|
||||
export const getAlertingCapabilities = (
|
||||
alerting: AlertingPublicStart | undefined,
|
||||
capabilities: Capabilities
|
||||
) => {
|
||||
const canSaveAlerts = !!capabilities.dataQuality['alerting:save'];
|
||||
const isAlertingPluginEnabled = !!alerting;
|
||||
const isAlertingAvailable = isAlertingPluginEnabled && canSaveAlerts;
|
||||
|
||||
return {
|
||||
isAlertingAvailable,
|
||||
};
|
||||
};
|
|
@ -5,18 +5,30 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import React, { useMemo } from 'react';
|
||||
import { EuiBetaBadge, EuiLink, EuiPageHeader, EuiCode } from '@elastic/eui';
|
||||
import { EuiBetaBadge, EuiButton, EuiCode, EuiLink, EuiPageHeader } from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
|
||||
import { DEFAULT_DATASET_TYPE, KNOWN_TYPES } from '../../../common/constants';
|
||||
import { datasetQualityAppTitle } from '../../../common/translations';
|
||||
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
|
||||
import { default as React, useMemo, useState } from 'react';
|
||||
import { KNOWN_TYPES } from '../../../common/constants';
|
||||
import { createAlertText, datasetQualityAppTitle } from '../../../common/translations';
|
||||
import { AlertFlyout } from '../../alerts/alert_flyout';
|
||||
import { getAlertingCapabilities } from '../../alerts/get_alerting_capabilities';
|
||||
import { useKibanaContextForPlugin } from '../../utils';
|
||||
import { DEFAULT_DATASET_TYPE } from '../../../common/constants';
|
||||
import { useDatasetQualityFilters } from '../../hooks/use_dataset_quality_filters';
|
||||
|
||||
// Allow for lazy loading
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default function Header() {
|
||||
const {
|
||||
services: { application, alerting },
|
||||
} = useKibanaContextForPlugin();
|
||||
const { capabilities } = application;
|
||||
|
||||
const [ruleType, setRuleType] = useState<typeof DEGRADED_DOCS_RULE_TYPE_ID | null>(null);
|
||||
|
||||
const { isAlertingAvailable } = getAlertingCapabilities(alerting, capabilities);
|
||||
const { isDatasetQualityAllSignalsAvailable } = useDatasetQualityFilters();
|
||||
const validTypes = useMemo(
|
||||
() => (isDatasetQualityAllSignalsAvailable ? KNOWN_TYPES : [DEFAULT_DATASET_TYPE]),
|
||||
|
@ -66,6 +78,31 @@ export default function Header() {
|
|||
}}
|
||||
/>
|
||||
}
|
||||
rightSideItems={
|
||||
isAlertingAvailable
|
||||
? [
|
||||
<>
|
||||
<EuiButton
|
||||
data-test-subj="datasetQualityDetailsHeaderButton"
|
||||
onClick={() => {
|
||||
setRuleType(DEGRADED_DOCS_RULE_TYPE_ID);
|
||||
}}
|
||||
iconType="bell"
|
||||
>
|
||||
{createAlertText}
|
||||
</EuiButton>
|
||||
<AlertFlyout
|
||||
addFlyoutVisible={!!ruleType}
|
||||
setAddFlyoutVisibility={(visible) => {
|
||||
if (!visible) {
|
||||
setRuleType(null);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</>,
|
||||
]
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
|
@ -7,8 +7,10 @@
|
|||
|
||||
import {
|
||||
EuiButton,
|
||||
EuiContextMenu,
|
||||
EuiFlexGroup,
|
||||
EuiFlexItem,
|
||||
EuiPopover,
|
||||
EuiSkeletonTitle,
|
||||
EuiTextColor,
|
||||
EuiTitle,
|
||||
|
@ -16,20 +18,30 @@ import {
|
|||
useEuiTheme,
|
||||
} from '@elastic/eui';
|
||||
import { css } from '@emotion/react';
|
||||
import React from 'react';
|
||||
import { openInDiscoverText } from '../../../common/translations';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import React, { useState } from 'react';
|
||||
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
|
||||
import { createAlertText, openInDiscoverText } from '../../../common/translations';
|
||||
import { AlertFlyout } from '../../alerts/alert_flyout';
|
||||
import { getAlertingCapabilities } from '../../alerts/get_alerting_capabilities';
|
||||
import {
|
||||
useDatasetDetailsRedirectLinkTelemetry,
|
||||
useDatasetDetailsTelemetry,
|
||||
useDatasetQualityDetailsState,
|
||||
useRedirectLink,
|
||||
} from '../../hooks';
|
||||
import { useKibanaContextForPlugin } from '../../utils';
|
||||
import { IntegrationIcon } from '../common';
|
||||
|
||||
export function Header() {
|
||||
const { datasetDetails, timeRange, integrationDetails, loadingState } =
|
||||
useDatasetQualityDetailsState();
|
||||
|
||||
const {
|
||||
services: { application, alerting },
|
||||
} = useKibanaContextForPlugin();
|
||||
const { capabilities } = application;
|
||||
|
||||
const { navigationSources } = useDatasetDetailsTelemetry();
|
||||
|
||||
const { rawName, name: title } = datasetDetails;
|
||||
|
@ -44,9 +56,65 @@ export function Header() {
|
|||
sendTelemetry,
|
||||
});
|
||||
|
||||
const { isAlertingAvailable } = getAlertingCapabilities(alerting, capabilities);
|
||||
|
||||
const [showPopover, setShowPopover] = useState<boolean>(false);
|
||||
const [ruleType, setRuleType] = useState<typeof DEGRADED_DOCS_RULE_TYPE_ID | null>(null);
|
||||
|
||||
const pageTitle =
|
||||
integrationDetails?.integration?.integration?.datasets?.[datasetDetails.name] ?? title;
|
||||
|
||||
const createMenuItems = [
|
||||
{
|
||||
name: createAlertText,
|
||||
icon: 'bell',
|
||||
onClick: () => {
|
||||
setShowPopover(false);
|
||||
setRuleType(DEGRADED_DOCS_RULE_TYPE_ID);
|
||||
},
|
||||
'data-test-subj': `createAlert`,
|
||||
},
|
||||
{
|
||||
name: openInDiscoverText,
|
||||
icon: 'discoverApp',
|
||||
...redirectLinkProps.linkProps,
|
||||
'data-test-subj': `openInDiscover`,
|
||||
},
|
||||
];
|
||||
const titleActionButtons = [
|
||||
<EuiPopover
|
||||
key="actionsPopover"
|
||||
isOpen={showPopover}
|
||||
closePopover={() => setShowPopover(false)}
|
||||
button={
|
||||
<EuiButton
|
||||
iconSide="right"
|
||||
iconType="arrowDown"
|
||||
data-test-subj="datasetQualityDetailsActionsDropdown"
|
||||
key="actionsDropdown"
|
||||
onClick={() => setShowPopover((prev) => !prev)}
|
||||
>
|
||||
{i18n.translate('xpack.datasetQuality.ActionsLabel', {
|
||||
defaultMessage: 'Actions',
|
||||
})}
|
||||
</EuiButton>
|
||||
}
|
||||
panelPaddingSize="none"
|
||||
repositionOnScroll
|
||||
>
|
||||
<EuiContextMenu
|
||||
initialPanelId={0}
|
||||
data-test-subj="autoFollowPatternActionContextMenu"
|
||||
panels={[
|
||||
{
|
||||
id: 0,
|
||||
items: createMenuItems,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</EuiPopover>,
|
||||
];
|
||||
|
||||
return !loadingState.integrationDetailsLoaded ? (
|
||||
<EuiSkeletonTitle
|
||||
size="s"
|
||||
|
@ -85,16 +153,29 @@ export function Header() {
|
|||
justifyContent="flexEnd"
|
||||
alignItems="center"
|
||||
>
|
||||
<EuiButton
|
||||
data-test-subj="datasetQualityDetailsHeaderButton"
|
||||
size="s"
|
||||
{...redirectLinkProps.linkProps}
|
||||
iconType="discoverApp"
|
||||
>
|
||||
{openInDiscoverText}
|
||||
</EuiButton>
|
||||
{isAlertingAvailable ? (
|
||||
titleActionButtons
|
||||
) : (
|
||||
<EuiButton
|
||||
data-test-subj="datasetQualityDetailsHeaderButton"
|
||||
size="s"
|
||||
{...redirectLinkProps.linkProps}
|
||||
iconType="discoverApp"
|
||||
>
|
||||
{openInDiscoverText}
|
||||
</EuiButton>
|
||||
)}
|
||||
</EuiFlexGroup>
|
||||
</EuiFlexItem>
|
||||
<AlertFlyout
|
||||
dataStream={rawName}
|
||||
addFlyoutVisible={!!ruleType}
|
||||
setAddFlyoutVisibility={(visible) => {
|
||||
if (!visible) {
|
||||
setRuleType(null);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</EuiFlexGroup>
|
||||
);
|
||||
}
|
||||
|
|
|
@ -11,8 +11,9 @@ import { createDatasetQuality } from './components/dataset_quality';
|
|||
import { createDatasetQualityDetails } from './components/dataset_quality_details';
|
||||
import { createDatasetQualityControllerLazyFactory } from './controller/dataset_quality/lazy_create_controller';
|
||||
import { createDatasetQualityDetailsControllerLazyFactory } from './controller/dataset_quality_details/lazy_create_controller';
|
||||
import { DataStreamsStatsService } from './services/data_streams_stats';
|
||||
import { registerRuleTypes } from './rule_types';
|
||||
import { DataStreamDetailsService } from './services/data_stream_details';
|
||||
import { DataStreamsStatsService } from './services/data_streams_stats';
|
||||
import {
|
||||
DatasetQualityPluginSetup,
|
||||
DatasetQualityPluginStart,
|
||||
|
@ -25,9 +26,13 @@ export class DatasetQualityPlugin
|
|||
{
|
||||
private telemetry = new TelemetryService();
|
||||
|
||||
public setup(core: CoreSetup, _plugins: DatasetQualitySetupDeps) {
|
||||
public setup(core: CoreSetup, plugins: DatasetQualitySetupDeps) {
|
||||
this.telemetry.setup({ analytics: core.analytics });
|
||||
|
||||
registerRuleTypes({
|
||||
ruleTypeRegistry: plugins.triggersActionsUi.ruleTypeRegistry,
|
||||
});
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { lazy } from 'react';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { RuleTypeModel } from '@kbn/triggers-actions-ui-plugin/public';
|
||||
import { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
|
||||
import { DEGRADED_DOCS_RULE_TYPE_ID } from '@kbn/rule-data-utils';
|
||||
import { validate } from './rule_form/validate';
|
||||
|
||||
export function getRuleType(): RuleTypeModel<DegradedDocsRuleParams> {
|
||||
return {
|
||||
id: DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
description: i18n.translate('xpack.datasetQuality.alert.degradedDocs.descriptionText', {
|
||||
defaultMessage: 'Alert when degraded docs percentage exceeds a threshold.',
|
||||
}),
|
||||
iconClass: 'bell',
|
||||
documentationUrl: null,
|
||||
ruleParamsExpression: lazy(() => import('./rule_form')),
|
||||
validate,
|
||||
requiresAppContext: false,
|
||||
};
|
||||
}
|
|
@ -0,0 +1,176 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { EuiFlexGroup, EuiIcon, EuiLoadingChart, EuiText, EuiToolTip } from '@elastic/eui';
|
||||
import numeral from '@elastic/numeral';
|
||||
import { IconChartBar } from '@kbn/chart-icons';
|
||||
import { EmptyPlaceholder } from '@kbn/charts-plugin/public';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import type { FC, PropsWithChildren } from 'react';
|
||||
import React from 'react';
|
||||
|
||||
export type Maybe<T> = T | null | undefined;
|
||||
|
||||
function isFiniteNumber(value: any): value is number {
|
||||
return isFinite(value);
|
||||
}
|
||||
|
||||
export function asPercent(
|
||||
numerator: Maybe<number>,
|
||||
denominator: number | undefined,
|
||||
fallbackResult = 'N/A'
|
||||
) {
|
||||
if (!denominator || !isFiniteNumber(numerator)) {
|
||||
return fallbackResult;
|
||||
}
|
||||
|
||||
const decimal = numerator / denominator;
|
||||
|
||||
if (Math.abs(decimal) >= 0.1 || decimal === 0) {
|
||||
return numeral(decimal).format('0.000%');
|
||||
}
|
||||
|
||||
return numeral(decimal).format('0.000%');
|
||||
}
|
||||
|
||||
export const TIME_LABELS = {
|
||||
s: i18n.translate('xpack.datasetQuality.alerts.timeLabels.seconds', {
|
||||
defaultMessage: 'seconds',
|
||||
}),
|
||||
m: i18n.translate('xpack.datasetQuality.alerts.timeLabels.minutes', {
|
||||
defaultMessage: 'minutes',
|
||||
}),
|
||||
h: i18n.translate('xpack.datasetQuality.alerts.timeLabels.hours', {
|
||||
defaultMessage: 'hours',
|
||||
}),
|
||||
d: i18n.translate('xpack.datasetQuality.alerts.timeLabels.days', {
|
||||
defaultMessage: 'days',
|
||||
}),
|
||||
};
|
||||
|
||||
export const getDomain = (series: Array<{ name?: string; data: any[] }>) => {
|
||||
const xValues = series.flatMap((item) => item.data.map((d) => d.x));
|
||||
const yValues = series.flatMap((item) => item.data.map((d) => d.y || 0));
|
||||
return {
|
||||
xMax: Math.max(...xValues),
|
||||
xMin: Math.min(...xValues),
|
||||
yMax: Math.max(...yValues),
|
||||
yMin: Math.min(...yValues),
|
||||
};
|
||||
};
|
||||
|
||||
const EmptyContainer: FC<PropsWithChildren<unknown>> = ({ children }) => (
|
||||
<div
|
||||
style={{
|
||||
width: '100%',
|
||||
height: 150,
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
|
||||
export function NoDataState() {
|
||||
return (
|
||||
<EmptyContainer>
|
||||
<EmptyPlaceholder
|
||||
icon={IconChartBar}
|
||||
message={
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.chartPreview.noDataMessage"
|
||||
defaultMessage="No results found"
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</EmptyContainer>
|
||||
);
|
||||
}
|
||||
|
||||
export function LoadingState() {
|
||||
return (
|
||||
<EmptyContainer>
|
||||
<EuiText color="subdued" data-test-subj="loadingData">
|
||||
<EuiLoadingChart size="m" />
|
||||
</EuiText>
|
||||
</EmptyContainer>
|
||||
);
|
||||
}
|
||||
|
||||
export function ErrorState() {
|
||||
return (
|
||||
<EmptyContainer>
|
||||
<EuiText color="subdued" data-test-subj="chartErrorState">
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.alerts.charts.errorMessage"
|
||||
defaultMessage="Uh oh, something went wrong"
|
||||
/>
|
||||
</EuiText>
|
||||
</EmptyContainer>
|
||||
);
|
||||
}
|
||||
|
||||
interface PreviewChartLabel {
|
||||
field: string;
|
||||
timeSize: number;
|
||||
timeUnit: string;
|
||||
series: number;
|
||||
totalGroups: number;
|
||||
}
|
||||
|
||||
export function TimeLabelForData({
|
||||
field,
|
||||
timeSize,
|
||||
timeUnit,
|
||||
series,
|
||||
totalGroups,
|
||||
}: PreviewChartLabel) {
|
||||
const totalGroupsTooltip = i18n.translate(
|
||||
'xpack.datasetQuality.chartPreview.TimeLabelForData.totalGroupsTooltip',
|
||||
{
|
||||
defaultMessage: 'Showing {series} out of {totalGroups} groups',
|
||||
values: {
|
||||
series,
|
||||
totalGroups,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
const xAxisInfo = (
|
||||
<EuiText size="xs">
|
||||
<strong>
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.chartPreview.timeLabelForData.xAxis"
|
||||
defaultMessage="{field} per {timeSize} {timeUnit}"
|
||||
values={{
|
||||
field,
|
||||
timeSize,
|
||||
timeUnit: TIME_LABELS[timeUnit as keyof typeof TIME_LABELS],
|
||||
}}
|
||||
/>
|
||||
</strong>
|
||||
</EuiText>
|
||||
);
|
||||
|
||||
return (
|
||||
<div style={{ textAlign: 'center' }}>
|
||||
{totalGroups > series ? (
|
||||
<EuiToolTip content={totalGroupsTooltip} position="top">
|
||||
<EuiFlexGroup gutterSize="xs">
|
||||
{xAxisInfo}
|
||||
<EuiIcon size="s" color="subdued" type="questionInCircle" className="eui-alignTop" />
|
||||
</EuiFlexGroup>
|
||||
</EuiToolTip>
|
||||
) : (
|
||||
xAxisInfo
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
|
@ -0,0 +1,239 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { RectAnnotationDatum, TickFormatter } from '@elastic/charts';
|
||||
import {
|
||||
AnnotationDomainType,
|
||||
Axis,
|
||||
BarSeries,
|
||||
Chart,
|
||||
LineAnnotation,
|
||||
Position,
|
||||
RectAnnotation,
|
||||
ScaleType,
|
||||
Settings,
|
||||
Tooltip,
|
||||
niceTimeFormatter,
|
||||
} from '@elastic/charts';
|
||||
import { EuiSpacer, useEuiTheme } from '@elastic/eui';
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
import { useElasticChartsTheme } from '@kbn/charts-theme';
|
||||
import type { IUiSettingsClient } from '@kbn/core/public';
|
||||
import { UI_SETTINGS } from '@kbn/data-plugin/public';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import moment from 'moment';
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import { TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
|
||||
import { Maybe, TimeLabelForData, getDomain } from './chart_preview_helper';
|
||||
|
||||
function getTimeZone(uiSettings?: IUiSettingsClient) {
|
||||
const kibanaTimeZone = uiSettings?.get<'Browser' | string>(UI_SETTINGS.DATEFORMAT_TZ);
|
||||
if (!kibanaTimeZone || kibanaTimeZone === 'Browser') {
|
||||
return moment.tz.guess();
|
||||
}
|
||||
|
||||
return kibanaTimeZone;
|
||||
}
|
||||
|
||||
interface ChartPreviewProps {
|
||||
yTickFormat?: TickFormatter;
|
||||
threshold: number[];
|
||||
comparator: COMPARATORS;
|
||||
uiSettings?: IUiSettingsClient;
|
||||
series: Array<{ name: string; data: Array<{ x: number; y: Maybe<number> }> }>;
|
||||
timeSize?: number;
|
||||
timeUnit?: TimeUnitChar;
|
||||
totalGroups: number;
|
||||
}
|
||||
|
||||
export function ChartPreview({
|
||||
yTickFormat,
|
||||
threshold,
|
||||
comparator,
|
||||
uiSettings,
|
||||
series,
|
||||
timeSize = 5,
|
||||
timeUnit = 'm',
|
||||
totalGroups,
|
||||
}: ChartPreviewProps) {
|
||||
const baseTheme = useElasticChartsTheme();
|
||||
const DEFAULT_DATE_FORMAT = 'Y-MM-DD HH:mm:ss';
|
||||
|
||||
const barSeries = useMemo(() => {
|
||||
return series.flatMap((serie) =>
|
||||
serie.data.map((point) => ({ ...point, groupBy: serie.name }))
|
||||
);
|
||||
}, [series]);
|
||||
|
||||
const timeZone = getTimeZone(uiSettings);
|
||||
|
||||
const chartSize = 120;
|
||||
|
||||
const { yMax, xMin, xMax } = getDomain(series);
|
||||
|
||||
const chartDomain = {
|
||||
max: Math.max(yMax === 0 ? 1 : yMax, Math.max(...threshold)) * 1.1, // Add 10% headroom.
|
||||
min: 0,
|
||||
};
|
||||
|
||||
const dateFormatter = useMemo(() => niceTimeFormatter([xMin, xMax]), [xMin, xMax]);
|
||||
const theme = useEuiTheme();
|
||||
const thresholdOpacity = 0.1;
|
||||
|
||||
const [sortedThreshold, setSortedThreshold] = useState(threshold);
|
||||
|
||||
useEffect(() => {
|
||||
setSortedThreshold([...threshold].sort((a, b) => a - b));
|
||||
}, [threshold]);
|
||||
|
||||
const style = {
|
||||
fill: theme.euiTheme.colors.danger,
|
||||
line: {
|
||||
strokeWidth: 1,
|
||||
stroke: theme.euiTheme.colors.danger,
|
||||
opacity: 1,
|
||||
},
|
||||
opacity: thresholdOpacity,
|
||||
};
|
||||
|
||||
const rectThresholdToMax = (value: number): RectAnnotationDatum[] => [
|
||||
{
|
||||
coordinates: {
|
||||
x0: xMin,
|
||||
x1: xMax,
|
||||
y0: value,
|
||||
y1: chartDomain.max,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const rectThresholdToMin = (value: number): RectAnnotationDatum[] => [
|
||||
{
|
||||
coordinates: {
|
||||
x0: xMin,
|
||||
x1: xMax,
|
||||
y0: chartDomain.min,
|
||||
y1: value,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const rectThresholdToThreshold: RectAnnotationDatum[] = [
|
||||
{
|
||||
coordinates: {
|
||||
x0: xMin,
|
||||
x1: xMax,
|
||||
y0: sortedThreshold[0],
|
||||
y1: sortedThreshold[1],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<>
|
||||
<EuiSpacer size="m" />
|
||||
<Chart
|
||||
size={{
|
||||
height: chartSize,
|
||||
}}
|
||||
data-test-subj="ChartPreview"
|
||||
>
|
||||
<Tooltip
|
||||
headerFormatter={({ value }) => {
|
||||
const dateFormat =
|
||||
(uiSettings && uiSettings.get(UI_SETTINGS.DATE_FORMAT)) || DEFAULT_DATE_FORMAT;
|
||||
return moment(value).format(dateFormat);
|
||||
}}
|
||||
/>
|
||||
<Settings showLegend={false} locale={i18n.getLocale()} baseTheme={baseTheme} />
|
||||
<LineAnnotation
|
||||
dataValues={[{ dataValue: sortedThreshold[0] }]}
|
||||
domainType={AnnotationDomainType.YDomain}
|
||||
id="chart_preview_line_annotation"
|
||||
markerPosition="left"
|
||||
style={style}
|
||||
/>
|
||||
{comparator !== COMPARATORS.BETWEEN && (
|
||||
<RectAnnotation
|
||||
dataValues={
|
||||
comparator === COMPARATORS.GREATER_THAN ||
|
||||
comparator === COMPARATORS.GREATER_THAN_OR_EQUALS
|
||||
? rectThresholdToMax(sortedThreshold[0])
|
||||
: rectThresholdToMin(sortedThreshold[0])
|
||||
}
|
||||
hideTooltips={true}
|
||||
id="chart_preview_rect_annotation"
|
||||
style={style}
|
||||
/>
|
||||
)}
|
||||
{threshold.length > 1 &&
|
||||
[COMPARATORS.NOT_BETWEEN, COMPARATORS.BETWEEN].includes(comparator) && (
|
||||
<>
|
||||
<LineAnnotation
|
||||
dataValues={[{ dataValue: sortedThreshold[1] }]}
|
||||
domainType={AnnotationDomainType.YDomain}
|
||||
id="chart_preview_line_annotation_2"
|
||||
markerPosition="left"
|
||||
style={style}
|
||||
/>
|
||||
<RectAnnotation
|
||||
dataValues={
|
||||
comparator === COMPARATORS.NOT_BETWEEN
|
||||
? rectThresholdToMax(sortedThreshold[1])
|
||||
: rectThresholdToThreshold
|
||||
}
|
||||
hideTooltips={true}
|
||||
id="chart_preview_rect_annotation_2"
|
||||
style={style}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
<Axis
|
||||
id="chart_preview_x_axis"
|
||||
position={Position.Bottom}
|
||||
showOverlappingTicks={true}
|
||||
tickFormat={dateFormatter}
|
||||
/>
|
||||
<Axis
|
||||
id="chart_preview_y_axis"
|
||||
position={Position.Left}
|
||||
tickFormat={yTickFormat}
|
||||
ticks={5}
|
||||
domain={chartDomain}
|
||||
/>
|
||||
<BarSeries
|
||||
id="chart_preview_bar_series"
|
||||
xScaleType={ScaleType.Time}
|
||||
yScaleType={ScaleType.Linear}
|
||||
xAccessor="x"
|
||||
yAccessors={['y']}
|
||||
splitSeriesAccessors={['groupBy']}
|
||||
data={barSeries}
|
||||
barSeriesStyle={{
|
||||
rectBorder: {
|
||||
strokeWidth: 1,
|
||||
visible: true,
|
||||
},
|
||||
rect: {
|
||||
opacity: 1,
|
||||
},
|
||||
}}
|
||||
timeZone={timeZone}
|
||||
/>
|
||||
</Chart>
|
||||
{series.length > 0 && (
|
||||
<TimeLabelForData
|
||||
field={'@timestamp'}
|
||||
timeSize={timeSize}
|
||||
timeUnit={timeUnit}
|
||||
series={series.length}
|
||||
totalGroups={totalGroups}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { RuleForm } from './rule_form';
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default RuleForm;
|
|
@ -0,0 +1,99 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
import { DataView } from '@kbn/data-views-plugin/common';
|
||||
import { TimeRange } from '@kbn/es-query';
|
||||
import { TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
|
||||
import rison from '@kbn/rison';
|
||||
import React from 'react';
|
||||
import { useAbortableAsync } from '@kbn/react-hooks';
|
||||
import { PreviewChartResponse } from '../../../../common/api_types';
|
||||
import { useKibanaContextForPlugin } from '../../../utils';
|
||||
import { ChartPreview } from './chart_preview';
|
||||
import {
|
||||
ErrorState,
|
||||
LoadingState,
|
||||
NoDataState,
|
||||
asPercent,
|
||||
} from './chart_preview/chart_preview_helper';
|
||||
|
||||
interface ChartOptions {
|
||||
interval?: string;
|
||||
}
|
||||
|
||||
export interface RuleConditionChartProps {
|
||||
threshold: number[];
|
||||
comparator: COMPARATORS;
|
||||
timeSize?: number;
|
||||
timeUnit?: TimeUnitChar;
|
||||
dataView?: DataView;
|
||||
groupBy?: string | string[];
|
||||
timeRange: TimeRange;
|
||||
chartOptions?: ChartOptions;
|
||||
}
|
||||
|
||||
export function RuleConditionChart({
|
||||
threshold,
|
||||
comparator,
|
||||
timeSize,
|
||||
timeUnit,
|
||||
dataView,
|
||||
groupBy,
|
||||
timeRange,
|
||||
chartOptions: { interval } = {},
|
||||
}: RuleConditionChartProps) {
|
||||
const {
|
||||
services: { http, uiSettings },
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const { loading, value, error } = useAbortableAsync(
|
||||
async ({ signal }) => {
|
||||
if (dataView && timeRange.from && timeRange.to) {
|
||||
return http.get<PreviewChartResponse>(
|
||||
'/internal/dataset_quality/rule_types/degraded_docs/chart_preview',
|
||||
{
|
||||
query: {
|
||||
index: dataView?.getIndexPattern(),
|
||||
start: timeRange?.from,
|
||||
end: timeRange?.to,
|
||||
interval: interval || `${timeSize}${timeUnit}`,
|
||||
groupBy: rison.encodeArray(Array.isArray(groupBy) ? groupBy : [groupBy]),
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
},
|
||||
[http, dataView, groupBy, interval, timeRange?.from, timeRange?.to, timeSize, timeUnit]
|
||||
);
|
||||
|
||||
return (
|
||||
<div>
|
||||
{loading ? (
|
||||
<LoadingState />
|
||||
) : !value?.series || value.series.length === 0 ? (
|
||||
<NoDataState />
|
||||
) : error ? (
|
||||
<ErrorState />
|
||||
) : (
|
||||
<ChartPreview
|
||||
series={value.series}
|
||||
threshold={threshold}
|
||||
uiSettings={uiSettings}
|
||||
comparator={comparator}
|
||||
yTickFormat={(d: number | null) => asPercent(d, 100)}
|
||||
timeSize={timeSize}
|
||||
timeUnit={timeUnit}
|
||||
totalGroups={value.totalGroups}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line import/no-default-export
|
||||
export default RuleConditionChart;
|
|
@ -0,0 +1,316 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
EuiComboBox,
|
||||
EuiExpression,
|
||||
EuiFormRow,
|
||||
EuiSpacer,
|
||||
EuiTitle,
|
||||
EuiFormErrorText,
|
||||
} from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n-react';
|
||||
import { DataViewSelectPopover } from '@kbn/stack-alerts-plugin/public';
|
||||
import {
|
||||
ForLastExpression,
|
||||
type RuleTypeParamsExpressionProps,
|
||||
} from '@kbn/triggers-actions-ui-plugin/public';
|
||||
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { DataView } from '@kbn/data-views-plugin/common';
|
||||
import { ThresholdExpression } from '@kbn/triggers-actions-ui-plugin/public';
|
||||
import { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { isArray } from 'lodash';
|
||||
import { DataViewBase, DataViewFieldBase } from '@kbn/es-query';
|
||||
import { TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
|
||||
import { INDEX } from '../../../../common/es_fields';
|
||||
import { useKibanaContextForPlugin } from '../../../utils';
|
||||
import { RuleConditionChart } from './rule_condition_chart';
|
||||
|
||||
const degradedDocsLabel = i18n.translate('xpack.datasetQuality.rule.degradedDocsLabel', {
|
||||
defaultMessage: 'degraded docs',
|
||||
});
|
||||
|
||||
export const defaultRuleParams: Partial<DegradedDocsRuleParams> = {
|
||||
comparator: COMPARATORS.GREATER_THAN,
|
||||
threshold: [3],
|
||||
timeSize: 5,
|
||||
timeUnit: 'm',
|
||||
groupBy: [INDEX],
|
||||
};
|
||||
|
||||
export type DataStreamGroupByFields = Array<DataViewFieldBase & { aggregatable: boolean }>;
|
||||
|
||||
export const RuleForm: React.FunctionComponent<
|
||||
RuleTypeParamsExpressionProps<DegradedDocsRuleParams, { adHocDataViewList: DataView[] }>
|
||||
> = (props) => {
|
||||
const {
|
||||
services: { dataViews, dataViewEditor },
|
||||
} = useKibanaContextForPlugin();
|
||||
|
||||
const { setRuleParams, ruleParams, errors, metadata, onChangeMetaData } = props;
|
||||
const { searchConfiguration, comparator, threshold, timeSize, timeUnit, groupBy } = ruleParams;
|
||||
|
||||
const [dataView, setDataView] = useState<DataView>();
|
||||
const [dataViewError, setDataViewError] = useState<string>();
|
||||
const [preselectedOptions, _] = useState<string[]>(defaultRuleParams.groupBy ?? []);
|
||||
const [adHocDataViews, setAdHocDataViews] = useState<DataView[]>(
|
||||
metadata?.adHocDataViewList ?? []
|
||||
);
|
||||
|
||||
const preFillProperty = useCallback(
|
||||
(property: keyof DegradedDocsRuleParams) => {
|
||||
setRuleParams(property, defaultRuleParams[property]);
|
||||
},
|
||||
[setRuleParams]
|
||||
);
|
||||
|
||||
const updateProperty = useCallback(
|
||||
(property: keyof DegradedDocsRuleParams, value?: any) => {
|
||||
setRuleParams(property, value);
|
||||
},
|
||||
[setRuleParams]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const initDataView = async () => {
|
||||
if (!searchConfiguration?.index) {
|
||||
setDataViewError(
|
||||
i18n.translate('xpack.datasetQuality.rule.dataViewErrorNoTimestamp', {
|
||||
defaultMessage: 'A data view is required.',
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (searchConfiguration?.index && !dataView) {
|
||||
const savedDataViews = await dataViews.getIdsWithTitle();
|
||||
const savedDataViewId = savedDataViews.find(
|
||||
(dv) => dv.title === searchConfiguration?.index
|
||||
)?.id;
|
||||
|
||||
if (savedDataViewId) {
|
||||
setDataView(await dataViews.get(savedDataViewId));
|
||||
return;
|
||||
}
|
||||
|
||||
let currentDataView: DataView;
|
||||
const adHocDataView = adHocDataViews.find((dv) => dv.title === searchConfiguration?.index);
|
||||
|
||||
if (adHocDataView) {
|
||||
currentDataView = adHocDataView;
|
||||
} else {
|
||||
currentDataView = await dataViews.create({
|
||||
title: searchConfiguration?.index,
|
||||
timeFieldName: '@timestamp',
|
||||
});
|
||||
|
||||
setAdHocDataViews((prev) => [...prev, currentDataView]);
|
||||
}
|
||||
|
||||
setDataView(currentDataView);
|
||||
}
|
||||
};
|
||||
|
||||
initDataView();
|
||||
}, [adHocDataViews, dataView, dataViews, searchConfiguration?.index]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!comparator) {
|
||||
preFillProperty('comparator');
|
||||
}
|
||||
|
||||
if (!threshold) {
|
||||
preFillProperty('threshold');
|
||||
}
|
||||
|
||||
if (!timeSize) {
|
||||
preFillProperty('timeSize');
|
||||
}
|
||||
|
||||
if (!timeUnit) {
|
||||
preFillProperty('timeUnit');
|
||||
}
|
||||
|
||||
if (!groupBy) {
|
||||
preFillProperty('groupBy');
|
||||
}
|
||||
}, [preFillProperty, comparator, groupBy, threshold, timeSize, timeUnit]);
|
||||
|
||||
const onGroupByChange = useCallback(
|
||||
(group: string | null | string[]) => {
|
||||
const gb = group ? (isArray(group) ? group : [group]) : [];
|
||||
setRuleParams('groupBy', gb);
|
||||
},
|
||||
[setRuleParams]
|
||||
);
|
||||
|
||||
const derivedIndexPattern = useMemo<DataViewBase>(
|
||||
() => ({
|
||||
fields: dataView?.fields || [],
|
||||
title: dataView?.getIndexPattern() || 'unknown-index',
|
||||
}),
|
||||
[dataView]
|
||||
);
|
||||
|
||||
const onSelectDataView = useCallback(
|
||||
(newDataView: DataView) => {
|
||||
setDataViewError(undefined);
|
||||
updateProperty('searchConfiguration', { index: newDataView.getIndexPattern() });
|
||||
setDataView(newDataView);
|
||||
},
|
||||
[updateProperty]
|
||||
);
|
||||
|
||||
const handleChange = useCallback(
|
||||
(selectedOptions: Array<{ label: string }>) => {
|
||||
const gb = selectedOptions.map((option) => option.label);
|
||||
|
||||
onGroupByChange([...new Set(preselectedOptions.concat(gb))]);
|
||||
},
|
||||
[onGroupByChange, preselectedOptions]
|
||||
);
|
||||
|
||||
const getPreSelectedOptions = () => {
|
||||
return preselectedOptions.map((field) => ({
|
||||
label: field,
|
||||
color: 'lightgray',
|
||||
disabled: true,
|
||||
}));
|
||||
};
|
||||
|
||||
const getUserSelectedOptions = (group: string[] | undefined) => {
|
||||
return (group ?? [])
|
||||
.filter((g) => !preselectedOptions.includes(g))
|
||||
.map((field) => ({
|
||||
label: field,
|
||||
}));
|
||||
};
|
||||
|
||||
const selectedOptions = [...getPreSelectedOptions(), ...getUserSelectedOptions(groupBy)];
|
||||
|
||||
return (
|
||||
<>
|
||||
<EuiTitle size="xs">
|
||||
<h5>
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.rule.dataView"
|
||||
defaultMessage="Select a data view"
|
||||
/>
|
||||
</h5>
|
||||
</EuiTitle>
|
||||
|
||||
<EuiSpacer size="s" />
|
||||
|
||||
<DataViewSelectPopover
|
||||
dependencies={{ dataViews, dataViewEditor }}
|
||||
dataView={dataView}
|
||||
metadata={{ adHocDataViewList: adHocDataViews }}
|
||||
onSelectDataView={onSelectDataView}
|
||||
onChangeMetaData={({ adHocDataViewList }) => {
|
||||
onChangeMetaData({ ...metadata, adHocDataViewList });
|
||||
}}
|
||||
/>
|
||||
{dataViewError && (
|
||||
<>
|
||||
<EuiFormErrorText data-test-subj="datasetQualityRuleDataViewError">
|
||||
{dataViewError}
|
||||
</EuiFormErrorText>
|
||||
<EuiSpacer size="s" />
|
||||
</>
|
||||
)}
|
||||
|
||||
<EuiSpacer size="m" />
|
||||
|
||||
<EuiTitle size="xs">
|
||||
<h5>
|
||||
<FormattedMessage
|
||||
id="xpack.datasetQuality.rule.alertCondition"
|
||||
defaultMessage="Set rule conditions"
|
||||
/>
|
||||
</h5>
|
||||
</EuiTitle>
|
||||
|
||||
<EuiSpacer size="s" />
|
||||
|
||||
<EuiExpression
|
||||
data-test-subj="datasetQualityRuleCountExpression"
|
||||
description={'PERCENTAGE'}
|
||||
value={degradedDocsLabel}
|
||||
display="columns"
|
||||
onClick={() => {}}
|
||||
disabled={true}
|
||||
/>
|
||||
|
||||
<ThresholdExpression
|
||||
thresholdComparator={comparator ?? defaultRuleParams.comparator}
|
||||
threshold={threshold}
|
||||
onChangeSelectedThresholdComparator={(value) => updateProperty('comparator', value)}
|
||||
onChangeSelectedThreshold={(value) => updateProperty('threshold', value)}
|
||||
errors={errors}
|
||||
display="fullWidth"
|
||||
unit={'%'}
|
||||
/>
|
||||
|
||||
<RuleConditionChart
|
||||
threshold={threshold}
|
||||
comparator={comparator as COMPARATORS}
|
||||
timeSize={timeSize}
|
||||
timeUnit={timeUnit as TimeUnitChar}
|
||||
dataView={dataView}
|
||||
groupBy={groupBy}
|
||||
timeRange={{ from: `now-${(timeSize ?? 1) * 20}${timeUnit}`, to: 'now' }}
|
||||
/>
|
||||
|
||||
<EuiSpacer size="l" />
|
||||
|
||||
<ForLastExpression
|
||||
timeWindowSize={timeSize}
|
||||
timeWindowUnit={timeUnit}
|
||||
errors={{
|
||||
timeSize: [],
|
||||
timeUnit: [],
|
||||
}}
|
||||
onChangeWindowSize={(value) => updateProperty('timeSize', value)}
|
||||
onChangeWindowUnit={(value) => updateProperty('timeUnit', value)}
|
||||
display="fullWidth"
|
||||
/>
|
||||
|
||||
<EuiSpacer size="l" />
|
||||
|
||||
<EuiFormRow
|
||||
label={i18n.translate('xpack.datasetQuality.rule.createAlertPerText', {
|
||||
defaultMessage: 'Group alerts by (optional)',
|
||||
})}
|
||||
helpText={i18n.translate('xpack.datasetQuality.rule.createAlertPerHelpText', {
|
||||
defaultMessage:
|
||||
'Create an alert for every unique value. For example: "host.id" or "cloud.region".',
|
||||
})}
|
||||
fullWidth
|
||||
display="rowCompressed"
|
||||
>
|
||||
<EuiComboBox
|
||||
data-test-subj="datasetQualityRuleGroupBy"
|
||||
placeholder={i18n.translate('xpack.datasetQuality.rule.groupByLabel', {
|
||||
defaultMessage: 'Everything',
|
||||
})}
|
||||
aria-label={i18n.translate('xpack.datasetQuality.rule.groupByAriaLabel', {
|
||||
defaultMessage: 'Graph per',
|
||||
})}
|
||||
fullWidth
|
||||
singleSelection={false}
|
||||
selectedOptions={selectedOptions}
|
||||
options={((derivedIndexPattern.fields as DataStreamGroupByFields) ?? [])
|
||||
.filter((f) => f.aggregatable && f.type === 'string')
|
||||
.map((f) => ({ label: f.name }))}
|
||||
onChange={handleChange}
|
||||
isClearable={true}
|
||||
/>
|
||||
</EuiFormRow>
|
||||
</>
|
||||
);
|
||||
};
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
|
||||
import { ValidationResult } from '@kbn/triggers-actions-ui-plugin/public/types';
|
||||
|
||||
const invalidThresholdValue = (value?: number) =>
|
||||
!value || (value && (isNaN(value) || value < 0 || value > 100));
|
||||
|
||||
export function validate(ruleParams: DegradedDocsRuleParams): ValidationResult {
|
||||
const errors: { [key: string]: string[] } = {};
|
||||
|
||||
if (!ruleParams.searchConfiguration) {
|
||||
return {
|
||||
errors: {
|
||||
searchConfiguration: [
|
||||
i18n.translate(
|
||||
'xpack.datasetQuality.alerts.validation.error.requiredSearchConfiguration',
|
||||
{
|
||||
defaultMessage: 'Search source configuration is required.',
|
||||
}
|
||||
),
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (!ruleParams.searchConfiguration.index) {
|
||||
return {
|
||||
errors: {
|
||||
searchConfiguration: [
|
||||
i18n.translate('xpack.datasetQuality.alerts.validation.error.requiredDataViewText', {
|
||||
defaultMessage: 'Data view is required.',
|
||||
}),
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (!ruleParams.threshold || invalidThresholdValue(ruleParams.threshold?.[0])) {
|
||||
errors.threshold0 = [
|
||||
i18n.translate('xpack.datasetQuality.alerts.validation.threshold', {
|
||||
defaultMessage: 'A valid percentage threshold is required (0-100).',
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
if (
|
||||
(ruleParams.comparator === COMPARATORS.BETWEEN ||
|
||||
ruleParams.comparator === COMPARATORS.NOT_BETWEEN) &&
|
||||
invalidThresholdValue(ruleParams.threshold?.[1])
|
||||
) {
|
||||
errors.threshold1 = [
|
||||
i18n.translate('xpack.datasetQuality.alerts.validation.threshold', {
|
||||
defaultMessage: 'A valid percentage threshold is required (0-100).',
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
return { errors };
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { TriggersAndActionsUIPublicPluginSetup } from '@kbn/triggers-actions-ui-plugin/public';
|
||||
import { getRuleType as getDegradedDocsRuleType } from './degraded_docs';
|
||||
|
||||
export function registerRuleTypes({
|
||||
ruleTypeRegistry,
|
||||
}: {
|
||||
ruleTypeRegistry: TriggersAndActionsUIPublicPluginSetup['ruleTypeRegistry'];
|
||||
}) {
|
||||
ruleTypeRegistry.register(getDegradedDocsRuleType());
|
||||
}
|
|
@ -5,14 +5,22 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ChartsPluginStart } from '@kbn/charts-plugin/public';
|
||||
import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
|
||||
import { DataViewEditorStart } from '@kbn/data-view-editor-plugin/public';
|
||||
import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
|
||||
import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public';
|
||||
import type { FieldsMetadataPublicStart } from '@kbn/fields-metadata-plugin/public';
|
||||
import type { LensPublicStart } from '@kbn/lens-plugin/public';
|
||||
import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
|
||||
import {
|
||||
TriggersAndActionsUIPublicPluginSetup,
|
||||
TriggersAndActionsUIPublicPluginStart,
|
||||
} from '@kbn/triggers-actions-ui-plugin/public';
|
||||
import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
|
||||
import type { ComponentType } from 'react';
|
||||
import type { PluginSetupContract as AlertingPublicSetup } from '@kbn/alerting-plugin/public/plugin';
|
||||
import type { PluginStartContract as AlertingPublicStart } from '@kbn/alerting-plugin/public/plugin';
|
||||
import type { DatasetQualityProps } from './components/dataset_quality';
|
||||
import type { DatasetQualityDetailsProps } from './components/dataset_quality_details';
|
||||
import type { CreateDatasetQualityController } from './controller/dataset_quality';
|
||||
|
@ -29,15 +37,21 @@ export interface DatasetQualityPluginStart {
|
|||
}
|
||||
|
||||
export interface DatasetQualityStartDeps {
|
||||
alerting: AlertingPublicStart;
|
||||
charts: ChartsPluginStart;
|
||||
data: DataPublicPluginStart;
|
||||
share: SharePluginStart;
|
||||
fieldFormats: FieldFormatsStart;
|
||||
unifiedSearch: UnifiedSearchPublicPluginStart;
|
||||
lens: LensPublicStart;
|
||||
dataViewEditor: DataViewEditorStart;
|
||||
dataViews: DataViewsPublicPluginStart;
|
||||
fieldFormats: FieldFormatsStart;
|
||||
fieldsMetadata: FieldsMetadataPublicStart;
|
||||
lens: LensPublicStart;
|
||||
share: SharePluginStart;
|
||||
triggersActionsUi: TriggersAndActionsUIPublicPluginStart;
|
||||
unifiedSearch: UnifiedSearchPublicPluginStart;
|
||||
}
|
||||
|
||||
export interface DatasetQualitySetupDeps {
|
||||
alerting?: AlertingPublicSetup;
|
||||
share: SharePluginSetup;
|
||||
triggersActionsUi: TriggersAndActionsUIPublicPluginSetup;
|
||||
}
|
||||
|
|
|
@ -7,10 +7,11 @@
|
|||
|
||||
import { CoreSetup, CoreStart, Logger, Plugin, PluginInitializerContext } from '@kbn/core/server';
|
||||
import { mapValues } from 'lodash';
|
||||
import { DataTelemetryService } from './services';
|
||||
import { getDatasetQualityServerRouteRepository } from './routes';
|
||||
import { registerRoutes } from './routes/register_routes';
|
||||
import { DatasetQualityRouteHandlerResources } from './routes/types';
|
||||
import { registerBuiltInRuleTypes } from './rule_types';
|
||||
import { DataTelemetryService } from './services';
|
||||
import {
|
||||
DatasetQualityPluginSetup,
|
||||
DatasetQualityPluginSetupDependencies,
|
||||
|
@ -78,6 +79,10 @@ export class DatasetQualityServerPlugin
|
|||
// Setup Data Telemetry Service
|
||||
this.dataTelemetryService.setup(plugins.taskManager, plugins.usageCollection);
|
||||
|
||||
if (plugins.alerting) {
|
||||
registerBuiltInRuleTypes(plugins.alerting, plugins.share?.url.locators);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
|
|
|
@ -7,11 +7,13 @@
|
|||
import type { EndpointOf, ServerRouteRepository } from '@kbn/server-route-repository';
|
||||
import { dataStreamsRouteRepository } from './data_streams/routes';
|
||||
import { integrationsRouteRepository } from './integrations/routes';
|
||||
import { ruleTypesRouteRepository } from './rule_types/routes';
|
||||
|
||||
function getTypedDatasetQualityServerRouteRepository() {
|
||||
const repository = {
|
||||
...dataStreamsRouteRepository,
|
||||
...integrationsRouteRepository,
|
||||
...ruleTypesRouteRepository,
|
||||
};
|
||||
|
||||
return repository;
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import * as t from 'io-ts';
|
||||
import { PreviewChartResponse } from '../../../common/api_types';
|
||||
import { createDatasetQualityServerRoute } from '../create_datasets_quality_server_route';
|
||||
import { getChartPreview } from '../../rule_types/degraded_docs/get_chart_preview';
|
||||
import { groupByRt } from '../../types/default_api_types';
|
||||
|
||||
const degradedDocsChartPreviewRoute = createDatasetQualityServerRoute({
|
||||
endpoint: 'GET /internal/dataset_quality/rule_types/degraded_docs/chart_preview',
|
||||
params: t.type({
|
||||
query: t.type({
|
||||
index: t.string,
|
||||
groupBy: groupByRt,
|
||||
start: t.string,
|
||||
end: t.string,
|
||||
interval: t.string,
|
||||
}),
|
||||
}),
|
||||
options: {
|
||||
tags: [],
|
||||
},
|
||||
security: {
|
||||
authz: {
|
||||
enabled: false,
|
||||
reason:
|
||||
'This API delegates security to the currently logged in user and their Elasticsearch permissions.',
|
||||
},
|
||||
},
|
||||
async handler(resources): Promise<PreviewChartResponse> {
|
||||
const { context, params } = resources;
|
||||
const coreContext = await context.core;
|
||||
|
||||
const esClient = coreContext.elasticsearch.client.asCurrentUser;
|
||||
|
||||
const degradedDocsChartPreview = await getChartPreview({
|
||||
esClient,
|
||||
...params.query,
|
||||
});
|
||||
|
||||
return { ...degradedDocsChartPreview };
|
||||
},
|
||||
});
|
||||
|
||||
export const ruleTypesRouteRepository = {
|
||||
...degradedDocsChartPreviewRoute,
|
||||
};
|
|
@ -0,0 +1,157 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
DATA_QUALITY_DETAILS_LOCATOR_ID,
|
||||
DATA_QUALITY_LOCATOR_ID,
|
||||
} from '@kbn/deeplinks-observability';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { getHumanReadableComparator } from '@kbn/stack-alerts-plugin/common';
|
||||
import type { Comparator } from '@kbn/stack-alerts-plugin/common/comparator_types';
|
||||
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
|
||||
import moment from 'moment';
|
||||
import { TimeRangeConfig } from '../../common/types';
|
||||
import { AdditionalContext, DatasetQualityRuleParams } from './types';
|
||||
|
||||
export const getPaddedAlertTimeRange = (
|
||||
alertStart: string,
|
||||
alertEnd?: string,
|
||||
lookBackWindow?: {
|
||||
size: number;
|
||||
unit: 's' | 'm' | 'h' | 'd';
|
||||
}
|
||||
): TimeRangeConfig => {
|
||||
const alertDuration = moment.duration(moment(alertEnd).diff(moment(alertStart)));
|
||||
const now = moment().toISOString();
|
||||
|
||||
// If alert duration is less than 160 min, we use 20 minute buffer
|
||||
// Otherwise, we use 8 times alert duration
|
||||
const defaultDurationMs =
|
||||
alertDuration.asMinutes() < 160
|
||||
? moment.duration(20, 'minutes').asMilliseconds()
|
||||
: alertDuration.asMilliseconds() / 8;
|
||||
// To ensure the alert time range at least covers 20 times lookback window,
|
||||
// we compare lookBackDurationMs and defaultDurationMs to use any of those that is longer
|
||||
const lookBackDurationMs =
|
||||
lookBackWindow &&
|
||||
moment.duration(lookBackWindow.size * 20, lookBackWindow.unit).asMilliseconds();
|
||||
const durationMs =
|
||||
lookBackDurationMs && lookBackDurationMs - defaultDurationMs > 0
|
||||
? lookBackDurationMs
|
||||
: defaultDurationMs;
|
||||
|
||||
const from = moment(alertStart).subtract(durationMs, 'millisecond').toISOString();
|
||||
const to =
|
||||
alertEnd && moment(alertEnd).add(durationMs, 'millisecond').isBefore(now)
|
||||
? moment(alertEnd).add(durationMs, 'millisecond').toISOString()
|
||||
: now;
|
||||
|
||||
return {
|
||||
from,
|
||||
to,
|
||||
refresh: {
|
||||
pause: true,
|
||||
value: 60000,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const getDataQualityViewInAppUrl = ({
|
||||
index,
|
||||
from,
|
||||
to,
|
||||
locatorsClient,
|
||||
}: {
|
||||
index: string;
|
||||
from: string;
|
||||
to: string;
|
||||
locatorsClient?: LocatorClient;
|
||||
}) => {
|
||||
const timeRange: TimeRangeConfig | undefined = getPaddedAlertTimeRange(from, to);
|
||||
timeRange.to = to ? timeRange.to : 'now';
|
||||
|
||||
// If index is a wildcard or multiple indices, redirect to the data quality overview page
|
||||
if (index.includes('*') || index.includes(',')) {
|
||||
return locatorsClient?.get(DATA_QUALITY_LOCATOR_ID)?.getRedirectUrl({
|
||||
timeRange: {
|
||||
from,
|
||||
to,
|
||||
refresh: {
|
||||
pause: true,
|
||||
value: 60000,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return locatorsClient?.get(DATA_QUALITY_DETAILS_LOCATOR_ID)?.getRedirectUrl({
|
||||
dataStream: index,
|
||||
timeRange: {
|
||||
from,
|
||||
to,
|
||||
refresh: {
|
||||
pause: true,
|
||||
value: 60000,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const generateReason = (
|
||||
value: string,
|
||||
group: string,
|
||||
timeSize: number,
|
||||
timeUnit: string,
|
||||
comparator: Comparator,
|
||||
threshold: number[]
|
||||
) =>
|
||||
i18n.translate('xpack.datasetQuality.rule.alertTypeContextReasonDescription', {
|
||||
defaultMessage: `Percentage of degraded documents is {value}% in the last {window} for {group}. Alert when {comparator} {threshold}%.`,
|
||||
values: {
|
||||
value,
|
||||
window: `${timeSize}${timeUnit}`,
|
||||
group,
|
||||
comparator: getHumanReadableComparator(comparator),
|
||||
threshold: threshold.join(' and '),
|
||||
},
|
||||
});
|
||||
|
||||
export const generateContext = ({
|
||||
group,
|
||||
dateStart,
|
||||
dateEnd,
|
||||
value,
|
||||
params,
|
||||
grouping,
|
||||
locatorsClient,
|
||||
}: {
|
||||
group: string;
|
||||
dateStart: string;
|
||||
dateEnd: string;
|
||||
value: string;
|
||||
params: DatasetQualityRuleParams;
|
||||
grouping?: AdditionalContext;
|
||||
locatorsClient?: LocatorClient;
|
||||
}) => ({
|
||||
value,
|
||||
reason: generateReason(
|
||||
value,
|
||||
group,
|
||||
params.timeSize,
|
||||
params.timeUnit,
|
||||
params.comparator as Comparator,
|
||||
params.threshold
|
||||
),
|
||||
grouping,
|
||||
timestamp: dateEnd,
|
||||
viewInAppUrl: getDataQualityViewInAppUrl({
|
||||
index: params.searchConfiguration.index,
|
||||
from: dateStart,
|
||||
to: dateEnd,
|
||||
locatorsClient,
|
||||
}),
|
||||
});
|
|
@ -0,0 +1,184 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { AlertsClientError, ExecutorType, RuleExecutorOptions } from '@kbn/alerting-plugin/server';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { ComparatorFns, TimeUnitChar } from '@kbn/response-ops-rule-params/common/utils';
|
||||
import {
|
||||
ALERT_EVALUATION_THRESHOLD,
|
||||
ALERT_EVALUATION_VALUE,
|
||||
ALERT_GROUPING,
|
||||
ALERT_KEY_JOINER,
|
||||
ALERT_REASON,
|
||||
} from '@kbn/rule-data-utils';
|
||||
import { Comparator } from '@kbn/stack-alerts-plugin/common/comparator_types';
|
||||
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
|
||||
import { _IGNORED } from '../../../common/es_fields';
|
||||
import { generateContext } from '../context';
|
||||
import { getDocsStats } from '../get_docs_stats';
|
||||
import {
|
||||
AdditionalContext,
|
||||
THRESHOLD_MET_GROUP,
|
||||
type DatasetQualityAlert,
|
||||
type DatasetQualityAlertContext,
|
||||
type DatasetQualityAlertState,
|
||||
type DatasetQualityAllowedActionGroups,
|
||||
type DatasetQualityRuleParams,
|
||||
type DatasetQualityRuleTypeState,
|
||||
} from '../types';
|
||||
|
||||
export const formatDurationFromTimeUnitChar = (time: number, unit: TimeUnitChar): string => {
|
||||
const sForPlural = time !== 0 && time > 1 ? 's' : '';
|
||||
switch (unit) {
|
||||
case 's':
|
||||
return `${time} sec${sForPlural}`;
|
||||
case 'm':
|
||||
return `${time} min${sForPlural}`;
|
||||
case 'h':
|
||||
return `${time} hr${sForPlural}`;
|
||||
case 'd':
|
||||
return `${time} day${sForPlural}`;
|
||||
default:
|
||||
return `${time} ${unit}`;
|
||||
}
|
||||
};
|
||||
|
||||
export const getRuleExecutor = (locatorsClient?: LocatorClient) =>
|
||||
async function executor(
|
||||
options: RuleExecutorOptions<
|
||||
DatasetQualityRuleParams,
|
||||
DatasetQualityRuleTypeState,
|
||||
DatasetQualityAlertState,
|
||||
DatasetQualityAlertContext,
|
||||
DatasetQualityAllowedActionGroups,
|
||||
DatasetQualityAlert
|
||||
>
|
||||
): ReturnType<
|
||||
ExecutorType<
|
||||
DatasetQualityRuleParams,
|
||||
DatasetQualityRuleTypeState,
|
||||
DatasetQualityAlertState,
|
||||
DatasetQualityAlertContext,
|
||||
DatasetQualityAllowedActionGroups
|
||||
>
|
||||
> {
|
||||
const { services, params, logger, getTimeRange } = options;
|
||||
const { alertsClient, scopedClusterClient } = services;
|
||||
|
||||
if (!alertsClient) {
|
||||
throw new AlertsClientError();
|
||||
}
|
||||
|
||||
const alertLimit = alertsClient.getAlertLimitValue();
|
||||
|
||||
const { dateStart, dateEnd } = getTimeRange(`${params.timeSize}${params.timeUnit}`);
|
||||
const index = params.searchConfiguration.index;
|
||||
|
||||
const datasetQualityDegradedResults = await getDocsStats({
|
||||
index,
|
||||
dateStart,
|
||||
groupBy: params.groupBy ?? [],
|
||||
query: {
|
||||
must: { exists: { field: _IGNORED } },
|
||||
},
|
||||
scopedClusterClient,
|
||||
});
|
||||
|
||||
const unmetGroupValues: Record<string, string> = {};
|
||||
const compareFn = ComparatorFns.get(params.comparator as Comparator);
|
||||
if (compareFn == null) {
|
||||
throw new Error(
|
||||
i18n.translate('xpack.datasetQuality.rule.invalidComparatorErrorMessage', {
|
||||
defaultMessage: 'invalid thresholdComparator specified: {comparator}',
|
||||
values: {
|
||||
comparator: params.comparator,
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
let generatedAlerts = 0;
|
||||
for (const groupResult of datasetQualityDegradedResults) {
|
||||
const { bucketKey, percentage } = groupResult;
|
||||
const alertId = bucketKey.join(ALERT_KEY_JOINER);
|
||||
const met = compareFn(percentage, params.threshold);
|
||||
|
||||
if (!met) {
|
||||
unmetGroupValues[alertId] = percentage.toFixed(2);
|
||||
continue;
|
||||
}
|
||||
|
||||
const groupByFields: AdditionalContext = {};
|
||||
const groupBy = params.groupBy ?? [];
|
||||
|
||||
for (let i = 0; i < bucketKey.length; i++) {
|
||||
const fieldName = groupBy[i];
|
||||
groupByFields[fieldName] = bucketKey[i];
|
||||
}
|
||||
|
||||
if (generatedAlerts < alertLimit) {
|
||||
const context = generateContext({
|
||||
group: alertId,
|
||||
dateStart,
|
||||
dateEnd,
|
||||
value: percentage.toFixed(2),
|
||||
params,
|
||||
grouping: groupByFields,
|
||||
locatorsClient,
|
||||
});
|
||||
alertsClient.report({
|
||||
id: alertId,
|
||||
actionGroup: THRESHOLD_MET_GROUP.id,
|
||||
state: {},
|
||||
context,
|
||||
payload: {
|
||||
[ALERT_REASON]: context.reason,
|
||||
[ALERT_EVALUATION_VALUE]: `${context.value}`,
|
||||
[ALERT_EVALUATION_THRESHOLD]:
|
||||
params.threshold?.length === 1 ? params.threshold[0] : null,
|
||||
[ALERT_GROUPING]: groupByFields,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
generatedAlerts++;
|
||||
}
|
||||
|
||||
alertsClient.setAlertLimitReached(generatedAlerts >= alertLimit);
|
||||
|
||||
// Handle recovered alerts context
|
||||
const { getRecoveredAlerts } = alertsClient;
|
||||
for (const recoveredAlert of getRecoveredAlerts()) {
|
||||
const alertId = recoveredAlert.alert.getId();
|
||||
logger.debug(`setting context for recovered alert ${alertId}`);
|
||||
|
||||
const grouping = recoveredAlert.hit?.[ALERT_GROUPING];
|
||||
const percentage = unmetGroupValues[alertId] ?? '0';
|
||||
const recoveryContext = generateContext({
|
||||
group: alertId,
|
||||
dateStart,
|
||||
dateEnd,
|
||||
value: percentage,
|
||||
params,
|
||||
grouping,
|
||||
locatorsClient,
|
||||
});
|
||||
|
||||
alertsClient.setAlertData({
|
||||
id: alertId,
|
||||
context: recoveryContext,
|
||||
payload: {
|
||||
[ALERT_REASON]: recoveryContext.reason,
|
||||
[ALERT_EVALUATION_VALUE]: `${recoveryContext.value}`,
|
||||
[ALERT_EVALUATION_THRESHOLD]: params.threshold?.length === 1 ? params.threshold[0] : null,
|
||||
[ALERT_GROUPING]: grouping,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return { state: {} };
|
||||
};
|
|
@ -0,0 +1,177 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import type { estypes } from '@elastic/elasticsearch';
|
||||
import { ElasticsearchClient } from '@kbn/core/server';
|
||||
import { PreviewChartResponse } from '../../../common/api_types';
|
||||
import { Coordinate } from '../../../common/types';
|
||||
import { extractKey } from '../extract_key';
|
||||
import { MISSING_VALUE } from '../types';
|
||||
|
||||
interface DataStreamTotals {
|
||||
x: Coordinate['x'];
|
||||
totalCount: number;
|
||||
ignoredCount: number;
|
||||
}
|
||||
|
||||
const NUM_SERIES = 5;
|
||||
const DEFAULT_GROUPS = 1000;
|
||||
|
||||
export const getFilteredBarSeries = (barSeries: PreviewChartResponse['series']) => {
|
||||
const sortedSeries = barSeries.sort((a, b) => {
|
||||
const aMax = Math.max(...a.data.map((point) => point.y as number));
|
||||
const bMax = Math.max(...b.data.map((point) => point.y as number));
|
||||
return bMax - aMax;
|
||||
});
|
||||
|
||||
return sortedSeries.slice(0, NUM_SERIES);
|
||||
};
|
||||
|
||||
export async function getChartPreview({
|
||||
esClient,
|
||||
index,
|
||||
groupBy,
|
||||
start,
|
||||
end,
|
||||
interval,
|
||||
}: {
|
||||
esClient: ElasticsearchClient;
|
||||
index: string;
|
||||
groupBy: string[];
|
||||
start: string;
|
||||
end: string;
|
||||
interval: string;
|
||||
}): Promise<PreviewChartResponse> {
|
||||
const bool = {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: start,
|
||||
lte: end,
|
||||
format: 'epoch_millis',
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const aggs = {
|
||||
...(groupBy.length > 0
|
||||
? {
|
||||
series: {
|
||||
...(groupBy.length === 1
|
||||
? {
|
||||
terms: {
|
||||
field: groupBy[0],
|
||||
size: DEFAULT_GROUPS,
|
||||
order: { _count: 'desc' as const },
|
||||
},
|
||||
}
|
||||
: {
|
||||
multi_terms: {
|
||||
terms: groupBy.map((field) => ({ field, missing: MISSING_VALUE })),
|
||||
size: DEFAULT_GROUPS,
|
||||
order: { _count: 'desc' as const },
|
||||
},
|
||||
}),
|
||||
aggs: {
|
||||
timeseries: {
|
||||
date_histogram: {
|
||||
field: '@timestamp',
|
||||
fixed_interval: interval,
|
||||
extended_bounds: {
|
||||
min: start,
|
||||
max: end,
|
||||
},
|
||||
},
|
||||
aggs: {
|
||||
ignored_fields: {
|
||||
filter: {
|
||||
exists: {
|
||||
field: '_ignored',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
|
||||
const esResult = await esClient.search<
|
||||
unknown,
|
||||
{ series: estypes.AggregationsStringTermsAggregate }
|
||||
>({
|
||||
index,
|
||||
track_total_hits: false,
|
||||
size: 0,
|
||||
query: {
|
||||
bool,
|
||||
},
|
||||
aggs,
|
||||
});
|
||||
|
||||
if (!esResult.aggregations) {
|
||||
return { series: [], totalGroups: 0 };
|
||||
}
|
||||
|
||||
const seriesBuckets = (esResult.aggregations?.series?.buckets ??
|
||||
[]) as estypes.AggregationsStringTermsBucket[];
|
||||
|
||||
const seriesDataMap: Record<string, DataStreamTotals[]> = seriesBuckets.reduce((acc, bucket) => {
|
||||
const bucketKey = extractKey({
|
||||
groupBy,
|
||||
bucketKey: Array.isArray(bucket.key) ? bucket.key : [bucket.key],
|
||||
});
|
||||
const timeSeriesBuckets = bucket.timeseries as estypes.AggregationsTimeSeriesAggregate;
|
||||
const timeseries = timeSeriesBuckets.buckets as estypes.AggregationsTimeSeriesBucket[];
|
||||
timeseries.forEach((timeseriesBucket: estypes.AggregationsTimeSeriesBucket) => {
|
||||
const x = (timeseriesBucket as Record<string, estypes.FieldValue>).key as number;
|
||||
const totalCount = timeseriesBucket.doc_count ?? 0;
|
||||
const ignoredCount =
|
||||
(timeseriesBucket.ignored_fields as estypes.AggregationsMultiBucketBase)?.doc_count ?? 0;
|
||||
|
||||
if (acc[bucketKey.join(',')]) {
|
||||
acc[bucketKey.join(',')].push({ x, totalCount, ignoredCount });
|
||||
} else {
|
||||
acc[bucketKey.join(',')] = [{ x, totalCount, ignoredCount }];
|
||||
}
|
||||
});
|
||||
|
||||
return acc;
|
||||
}, {} as Record<string, DataStreamTotals[]>);
|
||||
|
||||
const series = Object.keys(seriesDataMap).map((key) => ({
|
||||
name: key,
|
||||
data: Array.from(
|
||||
seriesDataMap[key]
|
||||
.reduce((map, curr: DataStreamTotals) => {
|
||||
if (!map.has(curr.x)) map.set(curr.x, { ...curr });
|
||||
else {
|
||||
map.get(curr.x).totalCount += curr.totalCount;
|
||||
map.get(curr.x).ignoredCount += curr.ignoredCount;
|
||||
}
|
||||
|
||||
return map;
|
||||
}, new Map())
|
||||
.values()
|
||||
).map((item: DataStreamTotals) => ({
|
||||
x: item.x,
|
||||
y: item.totalCount ? (item.ignoredCount / item.totalCount) * 100 : 0,
|
||||
})),
|
||||
}));
|
||||
|
||||
const filteredSeries = getFilteredBarSeries(series);
|
||||
|
||||
return {
|
||||
series: filteredSeries,
|
||||
totalGroups: series.length,
|
||||
};
|
||||
}
|
|
@ -0,0 +1,151 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { RuleType } from '@kbn/alerting-plugin/server';
|
||||
import { DEFAULT_APP_CATEGORIES } from '@kbn/core/server';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import {
|
||||
DegradedDocsRuleParams,
|
||||
degradedDocsParamsSchema,
|
||||
} from '@kbn/response-ops-rule-params/degraded_docs';
|
||||
import {
|
||||
ALERT_EVALUATION_THRESHOLD,
|
||||
ALERT_EVALUATION_VALUE,
|
||||
ALERT_GROUPING,
|
||||
ALERT_REASON,
|
||||
DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
STACK_ALERTS_FEATURE_ID,
|
||||
} from '@kbn/rule-data-utils';
|
||||
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
|
||||
import {
|
||||
DATASET_QUALITY_REGISTRATION_CONTEXT,
|
||||
DatasetQualityAlert,
|
||||
DatasetQualityAlertContext,
|
||||
DatasetQualityAllowedActionGroups,
|
||||
THRESHOLD_MET_GROUP,
|
||||
} from '../types';
|
||||
import { getRuleExecutor } from './executor';
|
||||
|
||||
export function DegradedDocsRuleType(
|
||||
locatorsClient?: LocatorClient
|
||||
): RuleType<
|
||||
DegradedDocsRuleParams,
|
||||
never,
|
||||
{},
|
||||
{},
|
||||
DatasetQualityAlertContext,
|
||||
DatasetQualityAllowedActionGroups,
|
||||
never,
|
||||
DatasetQualityAlert
|
||||
> {
|
||||
return {
|
||||
id: DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
name: i18n.translate('xpack.datasetQuality.rule.degradedDocs.name', {
|
||||
defaultMessage: 'Degraded docs',
|
||||
}),
|
||||
solution: 'stack',
|
||||
validate: {
|
||||
params: degradedDocsParamsSchema,
|
||||
},
|
||||
schemas: {
|
||||
params: {
|
||||
type: 'config-schema',
|
||||
schema: degradedDocsParamsSchema,
|
||||
},
|
||||
},
|
||||
defaultActionGroupId: THRESHOLD_MET_GROUP.id,
|
||||
actionGroups: [THRESHOLD_MET_GROUP],
|
||||
category: DEFAULT_APP_CATEGORIES.management.id,
|
||||
producer: STACK_ALERTS_FEATURE_ID,
|
||||
minimumLicenseRequired: 'basic',
|
||||
isExportable: true,
|
||||
executor: getRuleExecutor(locatorsClient),
|
||||
doesSetRecoveryContext: true,
|
||||
actionVariables: {
|
||||
context: [
|
||||
{ name: 'reason', description: actionVariableContextReasonLabel },
|
||||
{ name: 'value', description: actionVariableContextValueLabel },
|
||||
{ name: 'grouping', description: actionVariableContextGroupingLabel },
|
||||
{
|
||||
name: 'threshold',
|
||||
description: actionVariableContextThresholdLabel,
|
||||
},
|
||||
{ name: 'viewInAppUrl', description: viewInAppUrlActionVariableDescription },
|
||||
],
|
||||
},
|
||||
alerts: {
|
||||
context: DATASET_QUALITY_REGISTRATION_CONTEXT,
|
||||
mappings: {
|
||||
fieldMap: {
|
||||
[ALERT_REASON]: { type: 'keyword', array: false, required: false },
|
||||
[ALERT_EVALUATION_VALUE]: { type: 'keyword', array: false, required: false },
|
||||
[ALERT_EVALUATION_THRESHOLD]: {
|
||||
type: 'scaled_float',
|
||||
scaling_factor: 100,
|
||||
required: false,
|
||||
},
|
||||
[ALERT_GROUPING]: {
|
||||
type: 'object',
|
||||
dynamic: true,
|
||||
array: false,
|
||||
required: false,
|
||||
},
|
||||
},
|
||||
dynamicTemplates: [
|
||||
{
|
||||
strings_as_keywords: {
|
||||
path_match: 'kibana.alert.grouping.*',
|
||||
match_mapping_type: 'string',
|
||||
mapping: {
|
||||
type: 'keyword',
|
||||
ignore_above: 1024,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
shouldWrite: true,
|
||||
useEcs: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const actionVariableContextReasonLabel = i18n.translate(
|
||||
'xpack.datasetQuality.alerting.actionVariableContextReasonLabel',
|
||||
{
|
||||
defaultMessage: 'A concise description of the reason for the alert.',
|
||||
}
|
||||
);
|
||||
|
||||
const actionVariableContextValueLabel = i18n.translate(
|
||||
'xpack.datasetQuality.alerting.actionVariableContextValueLabel',
|
||||
{
|
||||
defaultMessage: 'The value that met the threshold condition.',
|
||||
}
|
||||
);
|
||||
|
||||
const actionVariableContextThresholdLabel = i18n.translate(
|
||||
'xpack.datasetQuality.alerting.actionVariableContextThresholdLabel',
|
||||
{
|
||||
defaultMessage:
|
||||
'An array of rule threshold values. For between and notBetween thresholds, there are two values.',
|
||||
}
|
||||
);
|
||||
|
||||
const actionVariableContextGroupingLabel = i18n.translate(
|
||||
'xpack.datasetQuality.alerting.actionVariableContextGrouping',
|
||||
{
|
||||
defaultMessage: 'The object containing groups that are reporting data',
|
||||
}
|
||||
);
|
||||
|
||||
export const viewInAppUrlActionVariableDescription = i18n.translate(
|
||||
'xpack.datasetQuality.alerting.actionVariableContextViewInAppUrl',
|
||||
{
|
||||
defaultMessage: 'Link to the alert source',
|
||||
}
|
||||
);
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { extractKey } from './extract_key';
|
||||
|
||||
describe('extractKey', () => {
|
||||
it('returns empty if "_index" is not part of the groupBy', async () => {
|
||||
const result = extractKey({
|
||||
groupBy: ['host.name', 'source.ip'],
|
||||
bucketKey: ['.ds-logs-custom-default-2025.04.08-000001', '127.0.0.1'],
|
||||
});
|
||||
|
||||
expect(result).toEqual(['.ds-logs-custom-default-2025.04.08-000001', '127.0.0.1']);
|
||||
});
|
||||
|
||||
describe('when "_index" is part of the groupBy', () => {
|
||||
it('and is not a backing index name', async () => {
|
||||
const result = extractKey({
|
||||
groupBy: ['_index'],
|
||||
bucketKey: ['logs-custom-default'],
|
||||
});
|
||||
|
||||
expect(result).toEqual(['logs-custom-default']);
|
||||
});
|
||||
|
||||
it('and is the only element in groupBy', async () => {
|
||||
const result = extractKey({
|
||||
groupBy: ['_index'],
|
||||
bucketKey: ['.ds-logs-custom-default-2025.04.08-000001'],
|
||||
});
|
||||
|
||||
expect(result).toEqual(['logs-custom-default']);
|
||||
});
|
||||
|
||||
it('and is at the beginning of the groupBy', async () => {
|
||||
const result = extractKey({
|
||||
groupBy: ['_index', 'cloud.provider'],
|
||||
bucketKey: ['.ds-logs-custom-default-2025.04.08-000001', 'aws'],
|
||||
});
|
||||
|
||||
expect(result).toEqual(['logs-custom-default', 'aws']);
|
||||
});
|
||||
|
||||
it('and is at the end of the groupBy', async () => {
|
||||
const result = extractKey({
|
||||
groupBy: ['cloud.provider', '_index'],
|
||||
bucketKey: ['aws', '.ds-logs-custom-default-2025.04.08-000001'],
|
||||
});
|
||||
|
||||
expect(result).toEqual(['aws', 'logs-custom-default']);
|
||||
});
|
||||
|
||||
it('and is at the middle of the groupBy', async () => {
|
||||
const result = extractKey({
|
||||
groupBy: ['cloud.region', '_index', 'cloud.provider'],
|
||||
bucketKey: ['eu-central-1', '.ds-logs-custom-default-2025.04.08-000001', 'aws'],
|
||||
});
|
||||
|
||||
expect(result).toEqual(['eu-central-1', 'logs-custom-default', 'aws']);
|
||||
});
|
||||
});
|
||||
});
|
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { FieldValue } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { INDEX } from '../../common/es_fields';
|
||||
import { extractIndexNameFromBackingIndex } from '../../common/utils';
|
||||
|
||||
/**
|
||||
* Extracts the key from the bucket key based on the groupBy fields.
|
||||
* If "_index" is not part of the groupBy, bucketKey is returned.
|
||||
* Otherwise, it replaces the "_index" value in the bucketKey with the actual dataStream name.
|
||||
*
|
||||
* @param {Object} params - The parameters object.
|
||||
* @param {string[]} params.groupBy - The groupBy fields.
|
||||
* @param {string[]} params.bucketKey - The bucket key values.
|
||||
* @returns {string[]} The extracted key.
|
||||
* @example
|
||||
* // returns ['logs-dataset-namespace', 'aws']
|
||||
* extractKey({ groupBy: ['_index', 'cloud.provider'], bucketKey: ['.ds-logs-dataset-namespace-2025.04.08-000001', 'aws'] });
|
||||
*/
|
||||
export const extractKey = ({
|
||||
groupBy,
|
||||
bucketKey,
|
||||
}: {
|
||||
groupBy: string[];
|
||||
bucketKey: FieldValue[];
|
||||
}): string[] => {
|
||||
if (!groupBy.includes(INDEX)) {
|
||||
return bucketKey as string[];
|
||||
}
|
||||
|
||||
const dataStreamIndex = groupBy.findIndex((group) => group === INDEX);
|
||||
const dataStreamName = extractIndexNameFromBackingIndex(bucketKey[dataStreamIndex] as string);
|
||||
const key = [
|
||||
...bucketKey.slice(0, dataStreamIndex),
|
||||
dataStreamName,
|
||||
...bucketKey.slice(dataStreamIndex + 1),
|
||||
];
|
||||
|
||||
return key as string[];
|
||||
};
|
|
@ -0,0 +1,120 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
import type { estypes } from '@elastic/elasticsearch';
|
||||
import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
|
||||
import { IScopedClusterClient } from '@kbn/core/server';
|
||||
import { extractKey } from './extract_key';
|
||||
import { MISSING_VALUE } from './types';
|
||||
|
||||
const DEFAULT_GROUPS = 1000;
|
||||
|
||||
export interface FetchEsQueryOpts {
|
||||
index: string;
|
||||
groupBy: string[];
|
||||
query?: {
|
||||
must?: QueryDslQueryContainer | QueryDslQueryContainer[];
|
||||
};
|
||||
services: {
|
||||
scopedClusterClient: IScopedClusterClient;
|
||||
};
|
||||
dateStart: string;
|
||||
}
|
||||
|
||||
export async function fetchEsQuery({
|
||||
index,
|
||||
groupBy,
|
||||
query = {},
|
||||
services,
|
||||
dateStart,
|
||||
}: FetchEsQueryOpts) {
|
||||
const { scopedClusterClient } = services;
|
||||
const esClient = scopedClusterClient.asCurrentUser;
|
||||
const { must } = query;
|
||||
|
||||
const bool = {
|
||||
filter: [
|
||||
{
|
||||
range: {
|
||||
'@timestamp': {
|
||||
gte: new Date(dateStart).getTime(),
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const aggs = {
|
||||
...(groupBy.length > 0
|
||||
? {
|
||||
dataStreams: {
|
||||
...(groupBy.length === 1
|
||||
? {
|
||||
terms: {
|
||||
field: groupBy[0],
|
||||
size: DEFAULT_GROUPS,
|
||||
order: { _count: 'desc' as const },
|
||||
},
|
||||
}
|
||||
: {
|
||||
multi_terms: {
|
||||
terms: groupBy.map((field) => ({ field, missing: MISSING_VALUE })),
|
||||
size: DEFAULT_GROUPS,
|
||||
order: { _count: 'desc' as const },
|
||||
},
|
||||
}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
|
||||
const esResult = await esClient.search<
|
||||
unknown,
|
||||
{ dataStreams: estypes.AggregationsStringTermsAggregate }
|
||||
>({
|
||||
index,
|
||||
track_total_hits: false,
|
||||
size: 0,
|
||||
query: {
|
||||
bool: {
|
||||
...bool,
|
||||
...(must ? { must } : {}),
|
||||
},
|
||||
},
|
||||
aggs,
|
||||
});
|
||||
|
||||
const dataStreamBuckets =
|
||||
(esResult.aggregations?.dataStreams?.buckets as estypes.AggregationsStringTermsBucketKeys[]) ||
|
||||
[];
|
||||
|
||||
// Group values by dataStream name instead of backing index name
|
||||
const groupedDataStreams = dataStreamBuckets.reduce(
|
||||
(acc: Record<string, { bucketKey: string[]; docCount: number }>, bucket) => {
|
||||
const key = extractKey({
|
||||
groupBy,
|
||||
bucketKey: Array.isArray(bucket.key) ? bucket.key : [bucket.key],
|
||||
});
|
||||
return {
|
||||
...acc,
|
||||
[key.join(',')]: {
|
||||
bucketKey: key,
|
||||
docCount: (acc[key.join(',')]?.docCount ?? 0) + bucket.doc_count,
|
||||
},
|
||||
};
|
||||
},
|
||||
{} as Record<string, { bucketKey: string[]; docCount: number }>
|
||||
);
|
||||
|
||||
return Object.keys(groupedDataStreams).reduce((obj, bucket) => {
|
||||
obj[groupedDataStreams[bucket].bucketKey.join(',')] = {
|
||||
docCount: groupedDataStreams[bucket].docCount,
|
||||
bucketKey: groupedDataStreams[bucket].bucketKey,
|
||||
};
|
||||
|
||||
return obj;
|
||||
}, {} as Record<string, { bucketKey: string[]; docCount: number }>);
|
||||
}
|
|
@ -0,0 +1,62 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { IScopedClusterClient } from '@kbn/core/server';
|
||||
import { QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
|
||||
import { fetchEsQuery } from './fetch_es_query';
|
||||
|
||||
export interface DocStat {
|
||||
bucketKey: string[];
|
||||
percentage: number;
|
||||
}
|
||||
|
||||
export const getDocsStats = async ({
|
||||
index,
|
||||
dateStart,
|
||||
groupBy,
|
||||
query,
|
||||
scopedClusterClient,
|
||||
}: {
|
||||
index: string;
|
||||
dateStart: string;
|
||||
groupBy: string[];
|
||||
query?: {
|
||||
must: QueryDslQueryContainer | QueryDslQueryContainer[];
|
||||
};
|
||||
scopedClusterClient: IScopedClusterClient;
|
||||
}): Promise<DocStat[]> => {
|
||||
const dataStreamTotalDocsResults = await fetchEsQuery({
|
||||
index,
|
||||
dateStart,
|
||||
groupBy,
|
||||
services: {
|
||||
scopedClusterClient,
|
||||
},
|
||||
});
|
||||
|
||||
const dataStreamQueryDocsResults = await fetchEsQuery({
|
||||
index,
|
||||
dateStart,
|
||||
groupBy,
|
||||
query,
|
||||
services: {
|
||||
scopedClusterClient,
|
||||
},
|
||||
});
|
||||
|
||||
return Object.keys(dataStreamTotalDocsResults).map((key) => {
|
||||
const totalDocs = dataStreamTotalDocsResults[key].docCount;
|
||||
const queryDocs = dataStreamQueryDocsResults[key]?.docCount ?? 0;
|
||||
const bucketKey =
|
||||
dataStreamTotalDocsResults[key]?.bucketKey ?? dataStreamQueryDocsResults[key]?.bucketKey;
|
||||
|
||||
return {
|
||||
bucketKey,
|
||||
percentage: totalDocs ? (queryDocs / totalDocs) * 100 : 0,
|
||||
};
|
||||
});
|
||||
};
|
|
@ -0,0 +1,17 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { AlertingServerSetup } from '@kbn/alerting-plugin/server';
|
||||
import { LocatorClient } from '@kbn/share-plugin/common/url_service';
|
||||
import { DegradedDocsRuleType } from './degraded_docs/register';
|
||||
|
||||
export function registerBuiltInRuleTypes(
|
||||
alertingPlugin: AlertingServerSetup,
|
||||
locatorsClient?: LocatorClient
|
||||
) {
|
||||
alertingPlugin.registerType(DegradedDocsRuleType(locatorsClient));
|
||||
}
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
ActionGroupIdsOf,
|
||||
AlertInstanceContext as AlertContext,
|
||||
AlertInstanceState as AlertState,
|
||||
} from '@kbn/alerting-plugin/common';
|
||||
import { RuleTypeState } from '@kbn/alerting-plugin/server';
|
||||
import { StackAlert } from '@kbn/alerts-as-data-utils';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import type { DegradedDocsRuleParams } from '@kbn/response-ops-rule-params/degraded_docs';
|
||||
|
||||
export type DatasetQualityRuleParams = DegradedDocsRuleParams;
|
||||
export type DatasetQualityRuleTypeState = RuleTypeState;
|
||||
export type DatasetQualityAlertState = AlertState;
|
||||
export type DatasetQualityAlertContext = AlertContext;
|
||||
export type DatasetQualityAllowedActionGroups = ActionGroupIdsOf<typeof THRESHOLD_MET_GROUP>;
|
||||
export type DatasetQualityAlert = Omit<StackAlert, 'kibana.alert.evaluation.threshold'> & {
|
||||
'kibana.alert.evaluation.threshold'?: string | number | null;
|
||||
'kibana.alert.grouping'?: Record<string, string>;
|
||||
};
|
||||
|
||||
export interface AdditionalContext {
|
||||
[x: string]: any;
|
||||
}
|
||||
|
||||
export const DATASET_QUALITY_REGISTRATION_CONTEXT = 'dataset.quality';
|
||||
|
||||
const THRESHOLD_MET_GROUP_ID = 'threshold_met';
|
||||
export const THRESHOLD_MET_GROUP = {
|
||||
id: THRESHOLD_MET_GROUP_ID,
|
||||
name: i18n.translate('xpack.datasetQuality.alerting.action.thresholdMet', {
|
||||
defaultMessage: 'Threshold met',
|
||||
}),
|
||||
};
|
||||
|
||||
export const MISSING_VALUE = i18n.translate('xpack.datasetQuality.alerting.missingValue', {
|
||||
defaultMessage: 'N/A',
|
||||
});
|
|
@ -5,8 +5,10 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { AlertingServerSetup, AlertingServerStart } from '@kbn/alerting-plugin/server';
|
||||
import { CustomRequestHandlerContext } from '@kbn/core/server';
|
||||
import type { FleetSetupContract, FleetStartContract } from '@kbn/fleet-plugin/server';
|
||||
import type { SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/server';
|
||||
import {
|
||||
TaskManagerSetupContract,
|
||||
TaskManagerStartContract,
|
||||
|
@ -15,17 +17,21 @@ import type { TelemetryPluginSetup, TelemetryPluginStart } from '@kbn/telemetry-
|
|||
import { UsageCollectionSetup, UsageCollectionStart } from '@kbn/usage-collection-plugin/server';
|
||||
|
||||
export interface DatasetQualityPluginSetupDependencies {
|
||||
alerting?: AlertingServerSetup;
|
||||
fleet: FleetSetupContract;
|
||||
telemetry: TelemetryPluginSetup;
|
||||
taskManager: TaskManagerSetupContract;
|
||||
telemetry: TelemetryPluginSetup;
|
||||
usageCollection?: UsageCollectionSetup;
|
||||
share?: SharePluginSetup;
|
||||
}
|
||||
|
||||
export interface DatasetQualityPluginStartDependencies {
|
||||
alerting?: AlertingServerStart;
|
||||
fleet: FleetStartContract;
|
||||
telemetry: TelemetryPluginStart;
|
||||
taskManager: TaskManagerStartContract;
|
||||
telemetry: TelemetryPluginStart;
|
||||
usageCollection?: UsageCollectionStart;
|
||||
share?: SharePluginStart;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-empty-interface
|
||||
|
|
|
@ -29,3 +29,14 @@ export const rangeRt = t.type({
|
|||
start: isoToEpochRt,
|
||||
end: isoToEpochRt,
|
||||
});
|
||||
|
||||
export const groupByRt = new t.Type<string[], string, unknown>(
|
||||
'groupByRt',
|
||||
(input: unknown): input is string[] =>
|
||||
Array.isArray(input) && input.every((value) => typeof value === 'string'),
|
||||
(input: unknown, context: t.Context) =>
|
||||
typeof input === 'string' && input.split(',').every((value) => typeof value === 'string')
|
||||
? t.success(input.split(',') as string[])
|
||||
: t.failure(input, context),
|
||||
(output: string[]) => output.join(',')
|
||||
);
|
||||
|
|
|
@ -10,55 +10,66 @@
|
|||
"../../../../../typings/**/*"
|
||||
],
|
||||
"kbn_references": [
|
||||
"@kbn/core",
|
||||
"@kbn/core-plugins-server",
|
||||
"@kbn/core-elasticsearch-server-mocks",
|
||||
"@kbn/fleet-plugin",
|
||||
"@kbn/server-route-repository",
|
||||
"@kbn/share-plugin",
|
||||
"@kbn/std",
|
||||
"@kbn/i18n",
|
||||
"@kbn/kibana-react-plugin",
|
||||
"@kbn/i18n-react",
|
||||
"@kbn/field-formats-plugin",
|
||||
"@kbn/field-types",
|
||||
"@kbn/io-ts-utils",
|
||||
"@kbn/es-types",
|
||||
"@kbn/deeplinks-observability",
|
||||
"@kbn/router-utils",
|
||||
"@kbn/xstate-utils",
|
||||
"@kbn/shared-ux-utility",
|
||||
"@kbn/data-plugin",
|
||||
"@kbn/unified-search-plugin",
|
||||
"@kbn/timerange",
|
||||
"@kbn/lens-plugin",
|
||||
"@kbn/data-views-plugin",
|
||||
"@kbn/shared-ux-error-boundary",
|
||||
"@kbn/es-query",
|
||||
"@kbn/core-saved-objects-api-server",
|
||||
"@kbn/deeplinks-management",
|
||||
"@kbn/deeplinks-analytics",
|
||||
"@kbn/core-elasticsearch-server",
|
||||
"@kbn/ui-actions-plugin",
|
||||
"@kbn/alerting-comparators",
|
||||
"@kbn/alerting-plugin",
|
||||
"@kbn/alerts-as-data-utils",
|
||||
"@kbn/calculate-auto",
|
||||
"@kbn/discover-plugin",
|
||||
"@kbn/shared-ux-prompt-no-data-views-types",
|
||||
"@kbn/ebt-tools",
|
||||
"@kbn/fields-metadata-plugin",
|
||||
"@kbn/server-route-repository-utils",
|
||||
"@kbn/chart-icons",
|
||||
"@kbn/charts-plugin",
|
||||
"@kbn/charts-theme",
|
||||
"@kbn/core-analytics-browser",
|
||||
"@kbn/core-elasticsearch-server-mocks",
|
||||
"@kbn/core-elasticsearch-server",
|
||||
"@kbn/core-lifecycle-browser",
|
||||
"@kbn/core-notifications-browser",
|
||||
"@kbn/telemetry-plugin",
|
||||
"@kbn/usage-collection-plugin",
|
||||
"@kbn/rison",
|
||||
"@kbn/task-manager-plugin",
|
||||
"@kbn/core-plugins-server",
|
||||
"@kbn/core-saved-objects-api-server",
|
||||
"@kbn/core",
|
||||
"@kbn/data-plugin",
|
||||
"@kbn/data-view-editor-plugin",
|
||||
"@kbn/data-views-plugin",
|
||||
"@kbn/deeplinks-analytics",
|
||||
"@kbn/deeplinks-management",
|
||||
"@kbn/deeplinks-observability",
|
||||
"@kbn/discover-plugin",
|
||||
"@kbn/ebt-tools",
|
||||
"@kbn/es-query",
|
||||
"@kbn/es-types",
|
||||
"@kbn/field-formats-plugin",
|
||||
"@kbn/field-types",
|
||||
"@kbn/field-utils",
|
||||
"@kbn/fields-metadata-plugin",
|
||||
"@kbn/fleet-plugin",
|
||||
"@kbn/i18n-react",
|
||||
"@kbn/i18n",
|
||||
"@kbn/io-ts-utils",
|
||||
"@kbn/kibana-react-plugin",
|
||||
"@kbn/lens-plugin",
|
||||
"@kbn/logging",
|
||||
"@kbn/ui-theme",
|
||||
"@kbn/react-hooks",
|
||||
"@kbn/charts-theme",
|
||||
"@kbn/unified-histogram"
|
||||
"@kbn/response-ops-rule-form",
|
||||
"@kbn/response-ops-rule-params",
|
||||
"@kbn/rison",
|
||||
"@kbn/router-utils",
|
||||
"@kbn/rule-data-utils",
|
||||
"@kbn/server-route-repository-utils",
|
||||
"@kbn/server-route-repository",
|
||||
"@kbn/share-plugin",
|
||||
"@kbn/shared-ux-error-boundary",
|
||||
"@kbn/shared-ux-prompt-no-data-views-types",
|
||||
"@kbn/shared-ux-utility",
|
||||
"@kbn/stack-alerts-plugin",
|
||||
"@kbn/std",
|
||||
"@kbn/task-manager-plugin",
|
||||
"@kbn/telemetry-plugin",
|
||||
"@kbn/timerange",
|
||||
"@kbn/triggers-actions-ui-plugin",
|
||||
"@kbn/ui-actions-plugin",
|
||||
"@kbn/ui-theme",
|
||||
"@kbn/unified-histogram",
|
||||
"@kbn/unified-search-plugin",
|
||||
"@kbn/usage-collection-plugin",
|
||||
"@kbn/xstate-utils",
|
||||
],
|
||||
"exclude": [
|
||||
"target/**/*"
|
||||
|
|
|
@ -68,6 +68,7 @@ export default function createRegisteredRuleTypeTests({ getService }: FtrProvide
|
|||
'apm.anomaly',
|
||||
'apm.error_rate',
|
||||
'apm.transaction_error_rate',
|
||||
'datasetQuality.degradedDocs',
|
||||
];
|
||||
|
||||
expect(
|
||||
|
|
|
@ -36,8 +36,8 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
return fieldStat.name === 'geo_point';
|
||||
}
|
||||
);
|
||||
expect(geoPointFieldStats.count).to.be(55);
|
||||
expect(geoPointFieldStats.index_count).to.be(12);
|
||||
expect(geoPointFieldStats.count).to.be(63);
|
||||
expect(geoPointFieldStats.index_count).to.be(13);
|
||||
|
||||
const geoShapeFieldStats = apiResponse.cluster_stats.indices.mappings.field_types.find(
|
||||
(fieldStat: estypes.ClusterStatsFieldTypes) => {
|
||||
|
|
|
@ -207,7 +207,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
securitySolutionSiemMigrations: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
logs: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
dataQuality: ['all', 'read', 'minimal_all', 'minimal_read', 'manage_rules', 'manage_alerts'],
|
||||
manageReporting: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
apm: ['all', 'read', 'minimal_all', 'minimal_read', 'settings_save'],
|
||||
discover: [
|
||||
|
|
|
@ -312,7 +312,14 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
securitySolutionNotes: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
infrastructure: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
logs: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
dataQuality: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
dataQuality: [
|
||||
'all',
|
||||
'read',
|
||||
'minimal_all',
|
||||
'minimal_read',
|
||||
'manage_rules',
|
||||
'manage_alerts',
|
||||
],
|
||||
manageReporting: ['all', 'read', 'minimal_all', 'minimal_read'],
|
||||
apm: ['all', 'read', 'minimal_all', 'minimal_read', 'settings_save'],
|
||||
discover: [
|
||||
|
|
|
@ -100,6 +100,7 @@ export default function ({ getService }: FtrProviderContext) {
|
|||
'alerting:apm.error_rate',
|
||||
'alerting:apm.transaction_duration',
|
||||
'alerting:apm.transaction_error_rate',
|
||||
'alerting:datasetQuality.degradedDocs',
|
||||
'alerting:logs.alert.document.count',
|
||||
'alerting:metrics.alert.inventory.threshold',
|
||||
'alerting:metrics.alert.threshold',
|
||||
|
|
|
@ -5,37 +5,37 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import {
|
||||
ALERT_EVALUATION_THRESHOLD,
|
||||
ALERT_EVALUATION_VALUE,
|
||||
ALERT_EVALUATION_VALUES,
|
||||
ALERT_RULE_PARAMETERS,
|
||||
ALERT_RULE_TYPE_ID,
|
||||
METRIC_INVENTORY_THRESHOLD_ALERT_TYPE_ID,
|
||||
OBSERVABILITY_THRESHOLD_RULE_TYPE_ID,
|
||||
LOG_THRESHOLD_ALERT_TYPE_ID,
|
||||
ALERT_EVALUATION_THRESHOLD,
|
||||
ApmRuleType,
|
||||
SLO_BURN_RATE_RULE_TYPE_ID,
|
||||
DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
LOG_THRESHOLD_ALERT_TYPE_ID,
|
||||
METRIC_INVENTORY_THRESHOLD_ALERT_TYPE_ID,
|
||||
METRIC_THRESHOLD_ALERT_TYPE_ID,
|
||||
OBSERVABILITY_THRESHOLD_RULE_TYPE_ID,
|
||||
SLO_BURN_RATE_RULE_TYPE_ID,
|
||||
} from '@kbn/rule-data-utils';
|
||||
import { EsQueryRuleParams } from '@kbn/stack-alerts-plugin/public/rule_types/es_query/types';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
import { COMPARATORS } from '@kbn/alerting-comparators';
|
||||
|
||||
import { asDuration, asPercent, convertToBuiltInComparators } from '../../../../common';
|
||||
import { createFormatter } from '../../../../common/custom_threshold_rule/formatters';
|
||||
import { METRIC_FORMATTERS } from '../../../../common/custom_threshold_rule/formatters/snapshot_metric_formats';
|
||||
import { metricValueFormatter } from '../../../../common/custom_threshold_rule/metric_value_formatter';
|
||||
import {
|
||||
BaseMetricExpressionParams,
|
||||
CustomMetricExpressionParams,
|
||||
} from '../../../../common/custom_threshold_rule/types';
|
||||
import {
|
||||
ABOVE_OR_EQ_TEXT,
|
||||
ABOVE_TEXT,
|
||||
BELOW_OR_EQ_TEXT,
|
||||
BELOW_TEXT,
|
||||
} from '../../../../common/i18n';
|
||||
import { asDuration, asPercent, convertToBuiltInComparators } from '../../../../common';
|
||||
import { createFormatter } from '../../../../common/custom_threshold_rule/formatters';
|
||||
import { metricValueFormatter } from '../../../../common/custom_threshold_rule/metric_value_formatter';
|
||||
import { METRIC_FORMATTERS } from '../../../../common/custom_threshold_rule/formatters/snapshot_metric_formats';
|
||||
import {
|
||||
BaseMetricExpressionParams,
|
||||
CustomMetricExpressionParams,
|
||||
} from '../../../../common/custom_threshold_rule/types';
|
||||
import { TopAlert } from '../../../typings/alerts';
|
||||
import { isFieldsSameType } from './is_fields_same_type';
|
||||
export interface FlyoutThresholdData {
|
||||
|
@ -308,6 +308,15 @@ export const mapRuleParamsWithFlyout = (alert: TopAlert): FlyoutThresholdData[]
|
|||
),
|
||||
} as unknown as FlyoutThresholdData;
|
||||
return [SLOBurnRateFlyoutMap];
|
||||
|
||||
case DEGRADED_DOCS_RULE_TYPE_ID:
|
||||
const DegradedDocsFlyoutMap = {
|
||||
observedValue: [asPercent(alert.fields[ALERT_EVALUATION_VALUE], 100)],
|
||||
threshold: [asPercent(alert.fields[ALERT_EVALUATION_THRESHOLD], 100)],
|
||||
comparator: ruleParams.comparator,
|
||||
} as unknown as FlyoutThresholdData;
|
||||
return [DegradedDocsFlyoutMap];
|
||||
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
|
|
|
@ -5,8 +5,12 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import {
|
||||
DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
ES_QUERY_ID,
|
||||
ML_ANOMALY_DETECTION_RULE_TYPE_ID,
|
||||
} from '@kbn/rule-data-utils';
|
||||
import { useMemo } from 'react';
|
||||
import { ES_QUERY_ID, ML_ANOMALY_DETECTION_RULE_TYPE_ID } from '@kbn/rule-data-utils';
|
||||
import { usePluginContext } from './use_plugin_context';
|
||||
|
||||
export function useGetFilteredRuleTypes() {
|
||||
|
@ -15,6 +19,7 @@ export function useGetFilteredRuleTypes() {
|
|||
return useMemo(() => {
|
||||
return [
|
||||
ES_QUERY_ID,
|
||||
DEGRADED_DOCS_RULE_TYPE_ID,
|
||||
ML_ANOMALY_DETECTION_RULE_TYPE_ID,
|
||||
...observabilityRuleTypeRegistry.list(),
|
||||
];
|
||||
|
|
|
@ -0,0 +1,184 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { LogsSynthtraceEsClient } from '@kbn/apm-synthtrace';
|
||||
import { log, timerange } from '@kbn/apm-synthtrace-client';
|
||||
import expect from '@kbn/expect';
|
||||
import rison from '@kbn/rison';
|
||||
import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_context';
|
||||
import { SupertestWithRoleScopeType } from '../../../services';
|
||||
|
||||
const MORE_THAN_1024_CHARS =
|
||||
'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?';
|
||||
|
||||
export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
|
||||
const roleScopedSupertest = getService('roleScopedSupertest');
|
||||
const synthtrace = getService('synthtrace');
|
||||
const start = '2025-04-11T08:00:00.000Z';
|
||||
const end = '2025-04-11T08:05:30.000Z';
|
||||
const type = 'logs';
|
||||
const dataset = 'nginx.access';
|
||||
const namespace = 'default';
|
||||
const serviceName = 'my-service';
|
||||
const otherServiceName = 'my-other-service';
|
||||
|
||||
async function callApiAs(
|
||||
roleScopedSupertestWithCookieCredentials: SupertestWithRoleScopeType,
|
||||
dataStream: string,
|
||||
groupBy = ['_index'],
|
||||
interval = '1m'
|
||||
) {
|
||||
return roleScopedSupertestWithCookieCredentials
|
||||
.get(`/internal/dataset_quality/rule_types/degraded_docs/chart_preview`)
|
||||
.query({
|
||||
index: dataStream,
|
||||
groupBy: rison.encodeArray(groupBy),
|
||||
start: new Date(start).getTime(),
|
||||
end: new Date(end).getTime(),
|
||||
interval,
|
||||
});
|
||||
}
|
||||
|
||||
describe('Chart preview per DataStream', function () {
|
||||
let synthtraceLogsEsClient: LogsSynthtraceEsClient;
|
||||
let supertestEditorWithCookieCredentials: SupertestWithRoleScopeType;
|
||||
|
||||
before(async () => {
|
||||
synthtraceLogsEsClient = await synthtrace.createLogsSynthtraceEsClient();
|
||||
supertestEditorWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope(
|
||||
'editor',
|
||||
{
|
||||
useCookieHeader: true,
|
||||
withInternalHeaders: true,
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe('gets the degraded fields timeseries per data stream', () => {
|
||||
before(async () => {
|
||||
await synthtraceLogsEsClient.index(
|
||||
Array.from({ length: 6 }, (_, i) =>
|
||||
timerange(start, end)
|
||||
.interval('1m')
|
||||
.rate(1)
|
||||
.generator((timestamp, j) =>
|
||||
log
|
||||
.create()
|
||||
.message('This is a log message')
|
||||
.timestamp(timestamp)
|
||||
.dataset(`${dataset}.${i}`)
|
||||
.namespace(namespace)
|
||||
.logLevel(i && j % i === 0 ? MORE_THAN_1024_CHARS : 'error')
|
||||
.defaults({
|
||||
'log.file.path': '/error.log',
|
||||
'service.name': j % 2 ? serviceName + i : otherServiceName + 1,
|
||||
'trace.id': i && j % i === 0 ? MORE_THAN_1024_CHARS : 'trace-id',
|
||||
})
|
||||
)
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await synthtraceLogsEsClient.clean();
|
||||
});
|
||||
|
||||
it('returns proper timeSeries data for degraded fields when querying a single dataStream', async () => {
|
||||
const logsTimeSeriesData = [
|
||||
{
|
||||
name: 'logs-nginx.access.5-default',
|
||||
data: [
|
||||
{ x: 1744358400000, y: 100 },
|
||||
{ x: 1744358460000, y: 0 },
|
||||
{ x: 1744358520000, y: 0 },
|
||||
{ x: 1744358580000, y: 0 },
|
||||
{ x: 1744358640000, y: 0 },
|
||||
{ x: 1744358700000, y: 100 },
|
||||
],
|
||||
},
|
||||
];
|
||||
const resp = await callApiAs(
|
||||
supertestEditorWithCookieCredentials,
|
||||
`${type}-${dataset}.5-*`
|
||||
);
|
||||
|
||||
expect(resp.body.series).to.eql(logsTimeSeriesData);
|
||||
});
|
||||
|
||||
it('returns proper timeSeries data when querying at a specific interval', async () => {
|
||||
const logsTimeSeriesData = [
|
||||
{
|
||||
name: 'logs-nginx.access.1-default',
|
||||
data: [
|
||||
{ x: 1744358400000, y: 100 },
|
||||
{ x: 1744358700000, y: 100 },
|
||||
],
|
||||
},
|
||||
];
|
||||
const resp = await callApiAs(
|
||||
supertestEditorWithCookieCredentials,
|
||||
`${type}-${dataset}.1-*`,
|
||||
['_index'],
|
||||
'5m'
|
||||
);
|
||||
|
||||
expect(resp.body.series).to.eql(logsTimeSeriesData);
|
||||
});
|
||||
|
||||
it('returns proper timeSeries data grouped using multiple keys', async () => {
|
||||
const logsTimeSeriesData = [
|
||||
{
|
||||
name: 'logs-nginx.access.1-default,my-other-service1',
|
||||
data: [
|
||||
{ x: 1744358400000, y: 100 },
|
||||
{ x: 1744358460000, y: 0 },
|
||||
{ x: 1744358520000, y: 100 },
|
||||
{ x: 1744358580000, y: 0 },
|
||||
{ x: 1744358640000, y: 100 },
|
||||
{ x: 1744358700000, y: 0 },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: 'logs-nginx.access.1-default,my-service1',
|
||||
data: [
|
||||
{ x: 1744358400000, y: 0 },
|
||||
{ x: 1744358460000, y: 100 },
|
||||
{ x: 1744358520000, y: 0 },
|
||||
{ x: 1744358580000, y: 100 },
|
||||
{ x: 1744358640000, y: 0 },
|
||||
{ x: 1744358700000, y: 100 },
|
||||
],
|
||||
},
|
||||
];
|
||||
const resp = await callApiAs(
|
||||
supertestEditorWithCookieCredentials,
|
||||
`${type}-${dataset}.1-*`,
|
||||
['_index', 'service.name']
|
||||
);
|
||||
|
||||
expect(resp.body.series).to.eql(logsTimeSeriesData);
|
||||
});
|
||||
|
||||
it('returns maximum 5 proper timeseries but the totalGroups indicates that there were more', async () => {
|
||||
const resp = await callApiAs(
|
||||
supertestEditorWithCookieCredentials,
|
||||
`${type}-${dataset}.*-*`
|
||||
);
|
||||
|
||||
expect(resp.body.series.length).to.be(5);
|
||||
expect(resp.body.totalGroups).to.be(6);
|
||||
});
|
||||
|
||||
it('returns empty when dataStream does not exists or does not have data reported', async () => {
|
||||
const resp = await callApiAs(supertestEditorWithCookieCredentials, `${type}-other-*`);
|
||||
|
||||
expect(resp.body.series.length).to.be(0);
|
||||
expect(resp.body.totalGroups).to.be(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
|
@ -9,20 +9,21 @@ import { DeploymentAgnosticFtrProviderContext } from '../../../ftr_provider_cont
|
|||
|
||||
export default function ({ loadTestFile }: DeploymentAgnosticFtrProviderContext) {
|
||||
describe('Dataset quality', () => {
|
||||
loadTestFile(require.resolve('./integrations'));
|
||||
loadTestFile(require.resolve('./integration_dashboards'));
|
||||
loadTestFile(require.resolve('./degraded_field_analyze'));
|
||||
loadTestFile(require.resolve('./data_stream_settings'));
|
||||
loadTestFile(require.resolve('./data_stream_rollover'));
|
||||
loadTestFile(require.resolve('./update_field_limit'));
|
||||
loadTestFile(require.resolve('./chart_preview'));
|
||||
loadTestFile(require.resolve('./check_and_load_integration'));
|
||||
loadTestFile(require.resolve('./data_stream_details'));
|
||||
loadTestFile(require.resolve('./data_stream_rollover'));
|
||||
loadTestFile(require.resolve('./data_stream_settings'));
|
||||
loadTestFile(require.resolve('./data_stream_total_docs'));
|
||||
loadTestFile(require.resolve('./degraded_docs'));
|
||||
loadTestFile(require.resolve('./degraded_fields'));
|
||||
loadTestFile(require.resolve('./data_stream_details'));
|
||||
loadTestFile(require.resolve('./degraded_field_analyze'));
|
||||
loadTestFile(require.resolve('./degraded_field_values'));
|
||||
loadTestFile(require.resolve('./failed_docs'));
|
||||
loadTestFile(require.resolve('./degraded_fields'));
|
||||
loadTestFile(require.resolve('./failed_docs_errors'));
|
||||
loadTestFile(require.resolve('./failed_docs_stats'));
|
||||
loadTestFile(require.resolve('./failed_docs'));
|
||||
loadTestFile(require.resolve('./integration_dashboards'));
|
||||
loadTestFile(require.resolve('./integrations'));
|
||||
loadTestFile(require.resolve('./update_field_limit'));
|
||||
});
|
||||
}
|
||||
|
|
|
@ -418,8 +418,9 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid
|
|||
dataStream: regularDataStreamName,
|
||||
});
|
||||
|
||||
const discoverButton =
|
||||
await PageObjects.datasetQuality.getDatasetQualityDetailsHeaderButton();
|
||||
await PageObjects.datasetQuality.openDatasetQualityDetailsActionsButton();
|
||||
|
||||
const discoverButton = await PageObjects.datasetQuality.getOpenInDiscoverButton();
|
||||
|
||||
await discoverButton.click();
|
||||
|
||||
|
|
|
@ -156,7 +156,6 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
|
|||
datasetQualityInsufficientPrivileges: 'datasetQualityInsufficientPrivileges',
|
||||
datasetQualityNoDataEmptyState: 'datasetQualityTableNoData',
|
||||
datasetQualityNoPrivilegesEmptyState: 'datasetQualityNoPrivilegesEmptyState',
|
||||
|
||||
superDatePickerToggleQuickMenuButton: 'superDatePickerToggleQuickMenuButton',
|
||||
superDatePickerApplyTimeButton: 'superDatePickerApplyTimeButton',
|
||||
superDatePickerQuickMenu: 'superDatePickerQuickMenu',
|
||||
|
@ -170,6 +169,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
|
|||
'datasetQualityDetailsDegradedFieldFlyoutIssueDoesNotExist',
|
||||
datasetQualityDetailsOverviewDegradedFieldToggleSwitch:
|
||||
'datasetQualityDetailsOverviewDegradedFieldToggleSwitch',
|
||||
datasetQualityDetailsActionsDropdown: 'datasetQualityDetailsActionsDropdown',
|
||||
openInDiscover: 'openInDiscover',
|
||||
};
|
||||
|
||||
return {
|
||||
|
@ -437,6 +438,14 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv
|
|||
return testSubjects.find(testSubjectSelectors.datasetQualityDetailsHeaderButton);
|
||||
},
|
||||
|
||||
openDatasetQualityDetailsActionsButton() {
|
||||
return testSubjects.click(testSubjectSelectors.datasetQualityDetailsActionsDropdown);
|
||||
},
|
||||
|
||||
getOpenInDiscoverButton() {
|
||||
return testSubjects.find(testSubjectSelectors.openInDiscover);
|
||||
},
|
||||
|
||||
openIntegrationActionsMenu() {
|
||||
return testSubjects.click(testSubjectSelectors.datasetQualityDetailsIntegrationActionsButton);
|
||||
},
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue