[ML] Adds change point detection feature (#150308)

## Summary

Part of #145703, adding enhancements to the new Change Point Detection
page for technical preview under the AIOps Labs section in the ML
plugin.

<img width="1406" alt="image"
src="https://user-images.githubusercontent.com/5236598/217035513-86325cd9-17a9-46ed-8aea-77585038a427.png">

- Use the data bounds mode for the Y-axis 
- Add a cardinality check for the split field with a hard limit of
10,000
-  Show series labels above charts 
- Replace fields `select` controls with the `EuiComboBox`
- Add a filter for change point type 
- Display aggregation interval 
- Add a docs link to the Change Point Aggregation 
- Make split field optional 

### Checklist


- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [x] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [x] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [x] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)
This commit is contained in:
Dima Arnautov 2023-02-07 20:51:02 +01:00 committed by GitHub
parent 02af928026
commit c8b75b3b72
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 589 additions and 180 deletions

View file

@ -241,6 +241,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => {
sum: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-sum-aggregation.html`,
top_hits: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-top-hits-aggregation.html`,
top_metrics: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-top-metrics.html`,
change_point: `${ELASTICSEARCH_DOCS}search-aggregations-change-point-aggregation.html`,
},
runtimeFields: {
overview: `${ELASTICSEARCH_DOCS}runtime.html`,

View file

@ -218,6 +218,7 @@ export interface DocLinks {
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
readonly change_point: string;
};
readonly runtimeFields: {
readonly overview: string;

View file

@ -13,6 +13,7 @@ export {
useRefreshIntervalUpdates,
useTimefilter,
useTimeRangeUpdates,
useRefresh,
} from './src/hooks/use_timefilter';
export { DatePickerWrapper } from './src/components/date_picker_wrapper';
export {

View file

@ -7,6 +7,7 @@
import { useEffect, useMemo } from 'react';
import useObservable from 'react-use/lib/useObservable';
import { merge, type Observable } from 'rxjs';
import { distinctUntilChanged, map } from 'rxjs/operators';
import { isEqual } from 'lodash';
@ -14,6 +15,7 @@ import type { TimeRange } from '@kbn/es-query';
import type { TimefilterContract } from '@kbn/data-plugin/public';
import { useDatePickerContext } from './use_date_picker_context';
import { mlTimefilterRefresh$, Refresh } from '../services/timefilter_refresh_service';
/**
* Options interface for the `useTimefilter` custom hook.
@ -100,3 +102,29 @@ export const useTimeRangeUpdates = (absolute = false): TimeRange => {
return useObservable(timeChangeObservable$, getTimeCallback());
};
/**
* Provides the latest refresh, both manual or auto.
*/
export const useRefresh = () => {
const timefilter = useTimefilter();
const getTimeRange = () => {
const { from, to } = timefilter.getTime();
return { start: from, end: to };
};
const refresh$ = useMemo(() => {
return merge(
mlTimefilterRefresh$,
timefilter.getTimeUpdate$().pipe(
map(() => {
return { lastRefresh: Date.now(), timeRange: getTimeRange() };
})
)
) as Observable<Refresh>;
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
return useObservable<Refresh>(refresh$);
};

View file

@ -19,6 +19,8 @@ import { startWith } from 'rxjs';
import type { Query, Filter } from '@kbn/es-query';
import { usePageUrlState } from '@kbn/ml-url-state';
import { useTimefilter, useTimeRangeUpdates } from '@kbn/ml-date-picker';
import { DEFAULT_AGG_FUNCTION } from './constants';
import { useSplitFieldCardinality } from './use_split_field_cardinality';
import {
createMergedEsQuery,
getEsQueryFromSavedSearch,
@ -36,11 +38,12 @@ export interface ChangePointDetectionPageUrlState {
export interface ChangePointDetectionRequestParams {
fn: string;
splitField: string;
splitField?: string;
metricField: string;
interval: string;
query: Query;
filters: Filter[];
changePointType?: ChangePointType[];
}
export const ChangePointDetectionContext = createContext<{
@ -61,6 +64,7 @@ export const ChangePointDetectionContext = createContext<{
pageCount: number;
updatePagination: (newPage: number) => void;
};
splitFieldCardinality: number | null;
}>({
isLoading: false,
splitFieldsOptions: [],
@ -79,6 +83,7 @@ export const ChangePointDetectionContext = createContext<{
pageCount: 1,
updatePagination: () => {},
},
splitFieldCardinality: null,
});
export type ChangePointType =
@ -95,13 +100,14 @@ export interface ChangePointAnnotation {
label: string;
reason: string;
timestamp: string;
group_field: string;
group?: {
name: string;
value: string;
};
type: ChangePointType;
p_value: number;
}
const DEFAULT_AGG_FUNCTION = 'min';
export const ChangePointDetectionContextProvider: FC = ({ children }) => {
const { dataView, savedSearch } = useDataSource();
const {
@ -181,12 +187,9 @@ export const ChangePointDetectionContextProvider: FC = ({ children }) => {
if (!params.metricField && metricFieldOptions.length > 0) {
params.metricField = metricFieldOptions[0].name;
}
if (!params.splitField && splitFieldsOptions.length > 0) {
params.splitField = splitFieldsOptions[0].name;
}
params.interval = bucketInterval?.expression!;
return params;
}, [requestParamsFromUrl, metricFieldOptions, splitFieldsOptions, bucketInterval]);
}, [requestParamsFromUrl, metricFieldOptions, bucketInterval]);
const updateFilters = useCallback(
(update: Filter[]) => {
@ -240,12 +243,14 @@ export const ChangePointDetectionContextProvider: FC = ({ children }) => {
return mergedQuery;
}, [resultFilters, resultQuery, uiSettings, dataView, timeRange]);
const splitFieldCardinality = useSplitFieldCardinality(requestParams.splitField, combinedQuery);
const {
results: annotations,
isLoading: annotationsLoading,
progress,
pagination,
} = useChangePointResults(requestParams, combinedQuery);
} = useChangePointResults(requestParams, combinedQuery, splitFieldCardinality);
if (!bucketInterval) return null;
@ -263,6 +268,7 @@ export const ChangePointDetectionContextProvider: FC = ({ children }) => {
updateFilters,
resultQuery,
pagination,
splitFieldCardinality,
};
return (

View file

@ -7,24 +7,29 @@
import React, { FC, useCallback } from 'react';
import {
EuiBadge,
EuiCallOut,
EuiDescriptionList,
EuiEmptyPrompt,
EuiFlexGrid,
EuiFlexGroup,
EuiFlexItem,
EuiHorizontalRule,
EuiIcon,
EuiPagination,
EuiPanel,
EuiProgress,
EuiSpacer,
EuiTitle,
EuiText,
EuiToolTip,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n-react';
import type { Query } from '@kbn/es-query';
import { useDataSource } from '../../hooks/use_data_source';
import { SPLIT_FIELD_CARDINALITY_LIMIT } from './constants';
import { ChangePointTypeFilter } from './change_point_type_filter';
import { SearchBarWrapper } from './search_bar';
import { useChangePointDetectionContext } from './change_point_detection_context';
import { ChangePointType, useChangePointDetectionContext } from './change_point_detection_context';
import { MetricFieldSelector } from './metric_field_selector';
import { SplitFieldSelector } from './split_field_selector';
import { FunctionPicker } from './function_picker';
@ -40,8 +45,13 @@ export const ChangePointDetectionPage: FC = () => {
resultQuery,
progress,
pagination,
splitFieldCardinality,
splitFieldsOptions,
metricFieldOptions,
} = useChangePointDetectionContext();
const { dataView } = useDataSource();
const setFn = useCallback(
(fn: string) => {
updateRequestParams({ fn });
@ -50,7 +60,7 @@ export const ChangePointDetectionPage: FC = () => {
);
const setSplitField = useCallback(
(splitField: string) => {
(splitField: string | undefined) => {
updateRequestParams({ splitField });
},
[updateRequestParams]
@ -70,7 +80,37 @@ export const ChangePointDetectionPage: FC = () => {
[updateRequestParams]
);
const selectControlCss = { width: '200px' };
const setChangePointType = useCallback(
(changePointType: ChangePointType[] | undefined) => {
updateRequestParams({ changePointType });
},
[updateRequestParams]
);
const selectControlCss = { width: '300px' };
const cardinalityExceeded =
splitFieldCardinality && splitFieldCardinality > SPLIT_FIELD_CARDINALITY_LIMIT;
if (metricFieldOptions.length === 0) {
return (
<EuiCallOut
title={i18n.translate('xpack.aiops.index.dataViewWithoutMetricNotificationTitle', {
defaultMessage: 'The data view "{dataViewTitle}" does not contain any metric fields.',
values: { dataViewTitle: dataView.getName() },
})}
color="danger"
iconType="alert"
>
<p>
{i18n.translate('xpack.aiops.index.dataViewWithoutMetricNotificationDescription', {
defaultMessage:
'Change point detection can only be run on data views with a metric field.',
})}
</p>
</EuiCallOut>
);
}
return (
<div data-test-subj="aiopsChangePointDetectionPage">
@ -90,9 +130,11 @@ export const ChangePointDetectionPage: FC = () => {
<EuiFlexItem grow={false} css={selectControlCss}>
<MetricFieldSelector value={requestParams.metricField} onChange={setMetricField} />
</EuiFlexItem>
<EuiFlexItem grow={false} css={selectControlCss}>
<SplitFieldSelector value={requestParams.splitField} onChange={setSplitField} />
</EuiFlexItem>
{splitFieldsOptions.length > 0 ? (
<EuiFlexItem grow={false} css={selectControlCss}>
<SplitFieldSelector value={requestParams.splitField} onChange={setSplitField} />
</EuiFlexItem>
) : null}
<EuiFlexItem css={{ visibility: progress === 100 ? 'hidden' : 'visible' }} grow={false}>
<EuiProgress
@ -113,6 +155,51 @@ export const ChangePointDetectionPage: FC = () => {
<EuiSpacer size="m" />
{cardinalityExceeded ? (
<>
<EuiCallOut
title={i18n.translate('xpack.aiops.changePointDetection.cardinalityWarningTitle', {
defaultMessage: 'Analysis has been limited',
})}
color="warning"
iconType="alert"
>
<p>
{i18n.translate('xpack.aiops.changePointDetection.cardinalityWarningMessage', {
defaultMessage:
'The "{splitField}" field cardinality is {cardinality} which exceeds the limit of {cardinalityLimit}. Only the first {cardinalityLimit} partitions, sorted by document count, are analyzed.',
values: {
cardinality: splitFieldCardinality,
cardinalityLimit: SPLIT_FIELD_CARDINALITY_LIMIT,
splitField: requestParams.splitField,
},
})}
</p>
</EuiCallOut>
<EuiSpacer size="m" />
</>
) : null}
<EuiFlexGroup alignItems={'center'} justifyContent={'spaceBetween'}>
<EuiFlexItem grow={false}>
<EuiText size={'s'}>
<FormattedMessage
id="xpack.aiops.changePointDetection.aggregationIntervalTitle"
defaultMessage="Aggregation interval: "
/>
{requestParams.interval}
</EuiText>
</EuiFlexItem>
<EuiFlexItem grow={false} css={{ minWidth: '400px' }}>
<ChangePointTypeFilter
value={requestParams.changePointType}
onChange={setChangePointType}
/>
</EuiFlexItem>
</EuiFlexGroup>
<EuiSpacer size="s" />
{annotations.length === 0 && progress === 100 ? (
<>
<EuiEmptyPrompt
@ -129,7 +216,7 @@ export const ChangePointDetectionPage: FC = () => {
<p>
<FormattedMessage
id="xpack.aiops.changePointDetection.noChangePointsFoundMessage"
defaultMessage="Try to extend the time range or update the query"
defaultMessage="Detect statistically significant change points such as dips, spikes, and distribution changes in a metric. Select a metric and set a time range to start detecting change points in your data."
/>
</p>
}
@ -140,46 +227,70 @@ export const ChangePointDetectionPage: FC = () => {
<EuiFlexGrid columns={annotations.length >= 2 ? 2 : 1} responsive gutterSize={'m'}>
{annotations.map((v) => {
return (
<EuiFlexItem key={v.group_field}>
<EuiFlexItem key={v.group?.value ?? 'single_metric'}>
<EuiPanel paddingSize="s" hasBorder hasShadow={false}>
<EuiFlexGroup justifyContent={'spaceBetween'} alignItems={'center'}>
<EuiFlexGroup
alignItems={'center'}
justifyContent={'spaceBetween'}
gutterSize={'s'}
>
<EuiFlexItem grow={false}>
<EuiFlexGroup alignItems={'center'} gutterSize={'s'}>
<EuiFlexItem grow={false}>
<EuiTitle size="xxs">
<h3>{v.group_field}</h3>
</EuiTitle>
</EuiFlexItem>
{v.reason ? (
<EuiFlexItem grow={false}>
<EuiToolTip position="top" content={v.reason}>
<EuiIcon
tabIndex={0}
color={'warning'}
type="alert"
title={i18n.translate(
'xpack.aiops.changePointDetection.notResultsWarning',
{
defaultMessage: 'No change point agg results warning',
}
)}
/>
</EuiToolTip>
</EuiFlexItem>
) : null}
</EuiFlexGroup>
{v.group ? (
<EuiDescriptionList
type="inline"
listItems={[{ title: v.group.name, description: v.group.value }]}
/>
) : null}
{v.reason ? (
<EuiToolTip position="top" content={v.reason}>
<EuiIcon
tabIndex={0}
color={'warning'}
type="alert"
title={i18n.translate(
'xpack.aiops.changePointDetection.notResultsWarning',
{
defaultMessage: 'No change point agg results warning',
}
)}
/>
</EuiToolTip>
) : null}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText color={'subdued'} size={'s'}>
{requestParams.fn}({requestParams.metricField})
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule margin="xs" />
<EuiFlexGroup justifyContent={'spaceBetween'} alignItems={'center'}>
{v.p_value !== undefined ? (
<EuiFlexItem grow={false}>
<EuiDescriptionList
type="inline"
listItems={[
{
title: (
<FormattedMessage
id="xpack.aiops.explainLogRateSpikes.spikeAnalysisTableGroups.pValueLabel"
defaultMessage="p-value"
/>
),
description: v.p_value.toPrecision(3),
},
]}
/>
</EuiFlexItem>
) : null}
<EuiFlexItem grow={false}>
<EuiBadge color="hollow">{v.type}</EuiBadge>
</EuiFlexItem>
</EuiFlexGroup>
{v.p_value !== undefined ? (
<EuiDescriptionList
type="inline"
listItems={[{ title: 'p-value', description: v.p_value.toPrecision(3) }]}
/>
) : null}
<ChartComponent annotation={v} />
</EuiPanel>
</EuiFlexItem>

View file

@ -8,7 +8,7 @@
import React, { FC } from 'react';
import { pick } from 'lodash';
import { EuiSpacer } from '@elastic/eui';
import { EuiCallOut, EuiSpacer } from '@elastic/eui';
import { DataView } from '@kbn/data-views-plugin/common';
import type { SavedSearch } from '@kbn/saved-search-plugin/public';
@ -19,6 +19,7 @@ import { DatePickerContextProvider } from '@kbn/ml-date-picker';
import { UI_SETTINGS } from '@kbn/data-plugin/common';
import { toMountPoint, wrapWithTheme } from '@kbn/kibana-react-plugin/public';
import { i18n } from '@kbn/i18n';
import { DataSourceContext } from '../../hooks/use_data_source';
import { AiopsAppContext, AiopsAppDependencies } from '../../hooks/use_aiops_app_context';
import { AIOPS_STORAGE_KEYS } from '../../types/storage';
@ -48,6 +49,25 @@ export const ChangePointDetectionAppState: FC<ChangePointDetectionAppStateProps>
uiSettingsKeys: UI_SETTINGS,
};
if (!dataView.isTimeBased()) {
return (
<EuiCallOut
title={i18n.translate('xpack.aiops.index.dataViewNotBasedOnTimeSeriesNotificationTitle', {
defaultMessage: 'The data view "{dataViewTitle}" is not based on a time series.',
values: { dataViewTitle: dataView.getName() },
})}
color="danger"
iconType="alert"
>
<p>
{i18n.translate('xpack.aiops.index.changePointTimeSeriesNotificationDescription', {
defaultMessage: 'Change point detection only runs over time-based indices.',
})}
</p>
</EuiCallOut>
);
}
return (
<AiopsAppContext.Provider value={appDependencies}>
<UrlStateProvider>

View file

@ -0,0 +1,140 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React, { type FC, useCallback, useMemo } from 'react';
import {
EuiComboBox,
type EuiComboBoxOptionOption,
type EuiComboBoxOptionsListProps,
EuiFlexGroup,
EuiFlexItem,
EuiFormRow,
EuiIcon,
EuiToolTip,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { isDefined } from '@kbn/ml-is-defined';
import { type ChangePointType } from './change_point_detection_context';
export type ChangePointUIValue = ChangePointType | undefined;
interface ChangePointTypeFilterProps {
value: ChangePointType[] | undefined;
onChange: (changePointType: ChangePointType[] | undefined) => void;
}
const changePointTypes: Array<{ value: ChangePointType; description: string }> = [
{
value: 'dip',
description: i18n.translate('xpack.aiops.changePointDetection.dipDescription', {
defaultMessage: 'A significant dip occurs at this point.',
}),
},
{
value: 'spike',
description: i18n.translate('xpack.aiops.changePointDetection.spikeDescription', {
defaultMessage: 'A significant spike occurs at this point.',
}),
},
{
value: 'distribution_change',
description: i18n.translate('xpack.aiops.changePointDetection.distributionChangeDescription', {
defaultMessage: 'The overall distribution of the values has changed significantly.',
}),
},
{
value: 'step_change',
description: i18n.translate('xpack.aiops.changePointDetection.stepChangeDescription', {
defaultMessage:
'The change indicates a statistically significant step up or down in value distribution.',
}),
},
{
value: 'trend_change',
description: i18n.translate('xpack.aiops.changePointDetection.trendChangeDescription', {
defaultMessage: 'An overall trend change occurs at this point.',
}),
},
];
interface FilterOption {
value: ChangePointUIValue;
label: string;
description: string;
}
type ChangePointTypeFilterOptions = Array<EuiComboBoxOptionOption<ChangePointUIValue>>;
export const ChangePointTypeFilter: FC<ChangePointTypeFilterProps> = ({ value, onChange }) => {
const options = useMemo<ChangePointTypeFilterOptions>(() => {
return [{ value: undefined, description: '' }, ...changePointTypes].map((v) => ({
value: v.value,
label:
v.value ??
i18n.translate('xpack.aiops.changePointDetection.selectAllChangePoints', {
defaultMessage: 'Select all',
}),
description: v.description,
}));
}, []);
const selection: ChangePointTypeFilterOptions = !value
? [options[0]]
: options.filter((v) => value.includes(v.value!));
const onChangeCallback = useCallback(
(selectedOptions: ChangePointTypeFilterOptions) => {
if (
selectedOptions.length === 0 ||
selectedOptions[selectedOptions.length - 1].value === undefined
) {
onChange(undefined);
return;
}
onChange(selectedOptions.map((v) => v.value as ChangePointType).filter(isDefined));
},
[onChange]
);
const renderOption = useCallback((option: FilterOption) => {
const { label, description } = option;
if (!description) {
return <>{label}</>;
}
return (
<EuiToolTip position="left" content={description}>
<EuiFlexGroup gutterSize={'s'} alignItems={'center'}>
<EuiFlexItem grow={false}>
<EuiIcon type="iInCircle" color={'primary'} />
</EuiFlexItem>
<EuiFlexItem>{label}</EuiFlexItem>
</EuiFlexGroup>
</EuiToolTip>
);
}, []) as unknown as EuiComboBoxOptionsListProps<ChangePointUIValue>['renderOption'];
return (
<EuiFormRow
label={i18n.translate('xpack.aiops.changePointDetection.changePointTypeLabel', {
defaultMessage: 'Change point type',
})}
display={'columnCompressed'}
fullWidth
>
<EuiComboBox<ChangePointType | undefined>
options={options}
selectedOptions={selection}
onChange={onChangeCallback}
isClearable
data-test-subj="aiopsChangePointTypeFilter"
renderOption={renderOption}
/>
</EuiFormRow>
);
};

View file

@ -14,16 +14,14 @@ import { useTimeRangeUpdates } from '@kbn/ml-date-picker';
import { useDataSource } from '../../hooks/use_data_source';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
import { useChangePointDetectionContext } from './change_point_detection_context';
import {
type ChangePointAnnotation,
useChangePointDetectionContext,
} from './change_point_detection_context';
import { fnOperationTypeMapping } from './constants';
export interface ChartComponentProps {
annotation: {
group_field: string;
label: string;
timestamp: string;
reason: string;
};
annotation: ChangePointAnnotation;
}
export const ChartComponent: FC<ChartComponentProps> = React.memo(({ annotation }) => {
@ -35,37 +33,38 @@ export const ChartComponent: FC<ChartComponentProps> = React.memo(({ annotation
const { dataView } = useDataSource();
const { requestParams, bucketInterval } = useChangePointDetectionContext();
const filters = useMemo(
() => [
{
meta: {
index: dataView.id!,
alias: null,
negate: false,
disabled: false,
type: 'phrase',
key: requestParams.splitField,
params: {
query: annotation.group_field,
const filters = useMemo(() => {
return annotation.group
? [
{
meta: {
index: dataView.id!,
alias: null,
negate: false,
disabled: false,
type: 'phrase',
key: annotation.group.name,
params: {
query: annotation.group.value,
},
},
query: {
match_phrase: {
[annotation.group.name]: annotation.group.value,
},
},
$state: {
store: FilterStateStore.APP_STATE,
},
},
},
query: {
match_phrase: {
[requestParams.splitField]: annotation.group_field,
},
},
$state: {
store: FilterStateStore.APP_STATE,
},
},
],
[dataView.id, requestParams.splitField, annotation.group_field]
);
]
: [];
}, [dataView.id, annotation.group]);
// @ts-ignore incorrect types for attributes
const attributes = useMemo<TypedLensByValueInput['attributes']>(() => {
return {
title: annotation.group_field,
title: annotation.group?.value ?? '',
description: '',
visualizationType: 'lnsXY',
type: 'lens',
@ -83,6 +82,9 @@ export const ChartComponent: FC<ChartComponentProps> = React.memo(({ annotation
],
state: {
visualization: {
yLeftExtent: {
mode: 'dataBounds',
},
legend: {
isVisible: false,
position: 'right',
@ -204,7 +206,7 @@ export const ChartComponent: FC<ChartComponentProps> = React.memo(({ annotation
return (
<EmbeddableComponent
id={`changePointChart_${annotation.group_field}`}
id={`changePointChart_${annotation.group ? annotation.group.value : annotation.label}`}
style={{ height: 350 }}
timeRange={timeRange}
attributes={attributes}

View file

@ -11,3 +11,9 @@ export const fnOperationTypeMapping: Record<string, string> = {
sum: 'sum',
avg: 'average',
} as const;
export const DEFAULT_AGG_FUNCTION = 'avg';
export const SPLIT_FIELD_CARDINALITY_LIMIT = 10000;
export const COMPOSITE_AGG_SIZE = 500;

View file

@ -5,9 +5,9 @@
* 2.0.
*/
import React, { FC, useMemo } from 'react';
import React, { FC, useCallback, useMemo } from 'react';
import { i18n } from '@kbn/i18n';
import { EuiFormRow, EuiSelect, EuiSelectOption } from '@elastic/eui';
import { EuiComboBox, type EuiComboBoxOptionOption, EuiFormRow } from '@elastic/eui';
import { useChangePointDetectionContext } from './change_point_detection_context';
interface MetricFieldSelectorProps {
@ -19,19 +19,34 @@ export const MetricFieldSelector: FC<MetricFieldSelectorProps> = React.memo(
({ value, onChange }) => {
const { metricFieldOptions } = useChangePointDetectionContext();
const options = useMemo<EuiSelectOption[]>(() => {
return metricFieldOptions.map((v) => ({ value: v.name, text: v.displayName }));
const options = useMemo<EuiComboBoxOptionOption[]>(() => {
return metricFieldOptions.map((v) => ({ value: v.name, label: v.displayName }));
}, [metricFieldOptions]);
const selection = options.filter((v) => v.value === value);
const onChangeCallback = useCallback(
(selectedOptions: EuiComboBoxOptionOption[]) => {
const option = selectedOptions[0];
if (typeof option !== 'undefined') {
onChange(option.label);
}
},
[onChange]
);
return (
<EuiFormRow>
<EuiSelect
<EuiComboBox
prepend={i18n.translate('xpack.aiops.changePointDetection.selectMetricFieldLabel', {
defaultMessage: 'Metric field',
})}
singleSelection={{ asPlainText: true }}
options={options}
value={value}
onChange={(e) => onChange(e.target.value)}
selectedOptions={selection}
onChange={onChangeCallback}
isClearable={false}
data-test-subj="aiopsChangePointMetricField"
/>
</EuiFormRow>
);

View file

@ -5,32 +5,57 @@
* 2.0.
*/
import React, { FC, useMemo } from 'react';
import React, { FC, useMemo, useCallback } from 'react';
import { i18n } from '@kbn/i18n';
import { EuiFormRow, EuiSelect, type EuiSelectOption } from '@elastic/eui';
import { EuiComboBox, type EuiComboBoxOptionOption, EuiFormRow } from '@elastic/eui';
import { useChangePointDetectionContext } from './change_point_detection_context';
interface SplitFieldSelectorProps {
value: string;
onChange: (value: string) => void;
value: string | undefined;
onChange: (value: string | undefined) => void;
}
export const SplitFieldSelector: FC<SplitFieldSelectorProps> = React.memo(({ value, onChange }) => {
const { splitFieldsOptions } = useChangePointDetectionContext();
const options = useMemo<EuiSelectOption[]>(() => {
return splitFieldsOptions.map((v) => ({ value: v.name, text: v.displayName }));
const options = useMemo<Array<EuiComboBoxOptionOption<string>>>(() => {
return [
{
name: undefined,
displayName: i18n.translate('xpack.aiops.changePointDetection.notSelectedSplitFieldLabel', {
defaultMessage: '--- Not selected ---',
}),
},
...splitFieldsOptions,
].map((v) => ({
value: v.name,
label: v.displayName,
}));
}, [splitFieldsOptions]);
const selection = options.filter((v) => v.value === value);
const onChangeCallback = useCallback(
(selectedOptions: Array<EuiComboBoxOptionOption<string>>) => {
const option = selectedOptions[0];
const newValue = option?.value;
onChange(newValue);
},
[onChange]
);
return (
<EuiFormRow>
<EuiSelect
<EuiComboBox
prepend={i18n.translate('xpack.aiops.changePointDetection.selectSpitFieldLabel', {
defaultMessage: 'Split field',
})}
singleSelection={{ asPlainText: true }}
options={options}
value={value}
onChange={(e) => onChange(e.target.value)}
selectedOptions={selection}
onChange={onChangeCallback}
isClearable
data-test-subj="aiopsChangePointSplitField"
/>
</EuiFormRow>
);

View file

@ -8,6 +8,8 @@
import { useEffect, useCallback, useState, useMemo } from 'react';
import { type QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import { i18n } from '@kbn/i18n';
import { useRefresh } from '@kbn/ml-date-picker';
import { isDefined } from '@kbn/ml-is-defined';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
import {
ChangePointAnnotation,
@ -16,78 +18,87 @@ import {
} from './change_point_detection_context';
import { useDataSource } from '../../hooks/use_data_source';
import { useCancellableSearch } from '../../hooks/use_cancellable_search';
import { useSplitFieldCardinality } from './use_split_field_cardinality';
import { SPLIT_FIELD_CARDINALITY_LIMIT, COMPOSITE_AGG_SIZE } from './constants';
interface RequestOptions {
index: string;
fn: string;
metricField: string;
splitField: string;
splitField?: string;
timeField: string;
timeInterval: string;
afterKey?: string;
}
export const COMPOSITE_AGG_SIZE = 500;
function getChangePointDetectionRequestBody(
{ index, fn, metricField, splitField, timeInterval, timeField, afterKey }: RequestOptions,
query: QueryDslQueryContainer
) {
const timeSeriesAgg = {
over_time: {
date_histogram: {
field: timeField,
fixed_interval: timeInterval,
},
aggs: {
function_value: {
[fn]: {
field: metricField,
},
},
},
},
change_point_request: {
change_point: {
buckets_path: 'over_time>function_value',
},
},
// Bucket selecting and sorting are only applicable for partitions
...(isDefined(splitField)
? {
select: {
bucket_selector: {
buckets_path: { p_value: 'change_point_request.p_value' },
script: 'params.p_value < 1',
},
},
sort: {
bucket_sort: {
sort: [{ 'change_point_request.p_value': { order: 'asc' } }],
},
},
}
: {}),
};
const aggregations = splitField
? {
groupings: {
composite: {
size: COMPOSITE_AGG_SIZE,
...(afterKey !== undefined ? { after: { splitFieldTerm: afterKey } } : {}),
sources: [
{
splitFieldTerm: {
terms: {
field: splitField,
},
},
},
],
},
aggregations: timeSeriesAgg,
},
}
: timeSeriesAgg;
return {
params: {
index,
size: 0,
body: {
query,
aggregations: {
groupings: {
composite: {
size: COMPOSITE_AGG_SIZE,
...(afterKey !== undefined ? { after: { splitFieldTerm: afterKey } } : {}),
sources: [
{
splitFieldTerm: {
terms: {
field: splitField,
},
},
},
],
},
aggregations: {
over_time: {
date_histogram: {
field: timeField,
fixed_interval: timeInterval,
},
aggs: {
function_value: {
[fn]: {
field: metricField,
},
},
},
},
change_point_request: {
change_point: {
buckets_path: 'over_time>function_value',
},
},
select: {
bucket_selector: {
buckets_path: { p_value: 'change_point_request.p_value' },
script: 'params.p_value < 1',
},
},
sort: {
bucket_sort: {
sort: [{ 'change_point_request.p_value': { order: 'asc' } }],
},
},
},
},
},
aggregations,
},
},
};
@ -97,7 +108,8 @@ const CHARTS_PER_PAGE = 6;
export function useChangePointResults(
requestParams: ChangePointDetectionRequestParams,
query: QueryDslQueryContainer
query: QueryDslQueryContainer,
splitFieldCardinality: number | null
) {
const {
notifications: { toasts },
@ -105,11 +117,19 @@ export function useChangePointResults(
const { dataView } = useDataSource();
const refresh = useRefresh();
const [results, setResults] = useState<ChangePointAnnotation[]>([]);
const [activePage, setActivePage] = useState<number>(0);
const [progress, setProgress] = useState<number>(0);
const splitFieldCardinality = useSplitFieldCardinality(requestParams.splitField, query);
const isSingleMetric = !isDefined(requestParams.splitField);
const totalAggPages = useMemo<number>(() => {
return Math.ceil(
Math.min(splitFieldCardinality ?? 0, SPLIT_FIELD_CARDINALITY_LIMIT) / COMPOSITE_AGG_SIZE
);
}, [splitFieldCardinality]);
const { runRequest, cancelRequest, isLoading } = useCancellableSearch();
@ -121,9 +141,9 @@ export function useChangePointResults(
}, [cancelRequest]);
const fetchResults = useCallback(
async (afterKey?: string, prevBucketsCount?: number) => {
async (pageNumber: number = 1, afterKey?: string) => {
try {
if (!splitFieldCardinality) {
if (!isSingleMetric && !totalAggPages) {
setProgress(100);
return;
}
@ -150,22 +170,28 @@ export function useChangePointResults(
return;
}
const buckets = result.rawResponse.aggregations.groupings.buckets;
const buckets = (
isSingleMetric
? [result.rawResponse.aggregations]
: result.rawResponse.aggregations.groupings.buckets
) as ChangePointAggResponse['aggregations']['groupings']['buckets'];
setProgress(
Math.min(
Math.round(((buckets.length + (prevBucketsCount ?? 0)) / splitFieldCardinality) * 100),
100
)
);
setProgress(Math.min(Math.round((pageNumber / totalAggPages) * 100), 100));
const groups = buckets.map((v) => {
let groups = buckets.map((v) => {
const changePointType = Object.keys(v.change_point_request.type)[0] as ChangePointType;
const timeAsString = v.change_point_request.bucket?.key;
const rawPValue = v.change_point_request.type[changePointType].p_value;
return {
group_field: v.key.splitFieldTerm,
...(isSingleMetric
? {}
: {
group: {
name: requestParams.splitField,
value: v.key.splitFieldTerm,
},
}),
type: changePointType,
p_value: rawPValue,
timestamp: timeAsString,
@ -174,6 +200,10 @@ export function useChangePointResults(
} as ChangePointAnnotation;
});
if (Array.isArray(requestParams.changePointType)) {
groups = groups.filter((v) => requestParams.changePointType!.includes(v.type));
}
setResults((prev) => {
return (
(prev ?? [])
@ -183,10 +213,13 @@ export function useChangePointResults(
);
});
if (result.rawResponse.aggregations.groupings.after_key?.splitFieldTerm) {
if (
result.rawResponse.aggregations?.groupings?.after_key?.splitFieldTerm &&
pageNumber < totalAggPages
) {
await fetchResults(
result.rawResponse.aggregations.groupings.after_key.splitFieldTerm,
buckets.length + (prevBucketsCount ?? 0)
pageNumber + 1,
result.rawResponse.aggregations.groupings.after_key.splitFieldTerm
);
} else {
setProgress(100);
@ -199,7 +232,7 @@ export function useChangePointResults(
});
}
},
[runRequest, requestParams, query, dataView, splitFieldCardinality, toasts]
[runRequest, requestParams, query, dataView, totalAggPages, toasts, isSingleMetric]
);
useEffect(
@ -211,7 +244,7 @@ export function useChangePointResults(
cancelRequest();
};
},
[requestParams, query, splitFieldCardinality, fetchResults, reset, cancelRequest]
[requestParams, query, splitFieldCardinality, fetchResults, reset, cancelRequest, refresh]
);
const pagination = useMemo(() => {
@ -230,11 +263,15 @@ export function useChangePointResults(
return { results: resultPerPage, isLoading, reset, progress, pagination };
}
/**
* Response type for aggregation with composite agg pagination.
* TODO: update type for the single metric
*/
interface ChangePointAggResponse {
took: number;
timed_out: boolean;
_shards: { total: number; failed: number; successful: number; skipped: number };
hits: { hits: any[]; total: number; max_score: null };
hits: { hits: unknown[]; total: number; max_score: null };
aggregations: {
groupings: {
after_key?: {

View file

@ -19,8 +19,11 @@ import { useDataSource } from '../../hooks/use_data_source';
* @param splitField
* @param query
*/
export function useSplitFieldCardinality(splitField: string, query: QueryDslQueryContainer) {
const [cardinality, setCardinality] = useState<number>();
export function useSplitFieldCardinality(
splitField: string | undefined,
query: QueryDslQueryContainer
) {
const [cardinality, setCardinality] = useState<number | null>(null);
const { dataView } = useDataSource();
const requestPayload = useMemo(() => {
@ -46,6 +49,10 @@ export function useSplitFieldCardinality(splitField: string, query: QueryDslQuer
useEffect(
function performCardinalityCheck() {
if (splitField === undefined) {
return;
}
cancelRequest();
getSplitFieldCardinality<
@ -62,7 +69,7 @@ export function useSplitFieldCardinality(splitField: string, query: QueryDslQuer
}
});
},
[getSplitFieldCardinality, requestPayload, cancelRequest]
[getSplitFieldCardinality, requestPayload, cancelRequest, splitField]
);
return cardinality;

View file

@ -47,6 +47,7 @@
"@kbn/ml-date-picker",
"@kbn/ml-local-storage",
"@kbn/ml-query-utils",
"@kbn/ml-is-defined",
],
"exclude": [
"target/**/*",

View file

@ -7,7 +7,7 @@
import React, { FC } from 'react';
import { pick } from 'lodash';
import { i18n } from '@kbn/i18n';
import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
@ -62,7 +62,12 @@ export const ChangePointDetectionPage: FC = () => {
])}
/>
) : null}
<HelpMenu docLink={services.docLinks.links.ml.guide} />
<HelpMenu
docLink={services.docLinks.links.aggs.change_point}
appName={i18n.translate('xpack.ml.changePointDetection.pageHeader', {
defaultMessage: 'Change point detection',
})}
/>
</>
);
};

View file

@ -11,17 +11,20 @@ import { useMlKibana } from '../../contexts/kibana';
interface HelpMenuProps {
docLink: string;
appName?: string;
}
// Component for adding a documentation link to the help menu
export const HelpMenu: FC<HelpMenuProps> = React.memo(({ docLink }) => {
export const HelpMenu: FC<HelpMenuProps> = React.memo(({ docLink, appName }) => {
const { chrome } = useMlKibana().services;
useEffect(() => {
chrome.setHelpExtension({
appName: i18n.translate('xpack.ml.chrome.help.appName', {
defaultMessage: 'Machine Learning',
}),
appName:
appName ??
i18n.translate('xpack.ml.chrome.help.appName', {
defaultMessage: 'Machine Learning',
}),
links: [
{
href: docLink,