github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[AO] Use data view timestamp in the new threshold rule (#162542)

Browse source 
Closes #159777

## 📝 Summary 

In this PR, I changed the timestamp used in rule execution and the
preview chart of the new threshold rule.

I created a separate ticket
(https://github.com/elastic/kibana/issues/162560) to implement
`infra/metrics_explorer` in the observability plugin, and in the
meantime, I adjusted the API also to accept timeFieldName.

Also, I have a separate ticket for improving data view validation
(https://github.com/elastic/kibana/issues/162554)

## 🧪 How to test
### New threshold rule
- Create a data view of a timestamp other than `@timestamp`
- Create a rule with this data view and make sure the preview and
generated alerts are as expected
- Create a rule with grouping and filtering and check the preview and
generated alerts

### Metric threshold rule
- Create a metric threshold rule to make sure preview and rule execution
works as before
This commit is contained in:
Maryam Saeidi 2023-07-28 16:43:00 +02:00 • committed by GitHub
parent 3eda5fca4e
commit 9584a3ae0d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 89 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

15
x-pack/plugins/infra/common/http_api/metrics_api.ts
View file

@ -10,11 +10,16 @@ import * as rt from 'io-ts';
import { MetricsUIAggregationRT } from '../inventory_models/types';
import { afterKeyObjectRT } from './metrics_explorer';
export const MetricsAPITimerangeRT = rt.type({
from: rt.number,
to: rt.number,
interval: rt.string,
});
export const MetricsAPITimerangeRT = rt.intersection([
rt.type({
from: rt.number,
to: rt.number,
interval: rt.string,
}),
rt.partial({
timeFieldName: rt.string,
}),
]);
const groupByRT = rt.union([rt.string, rt.null, rt.undefined]);

15
x-pack/plugins/infra/common/http_api/metrics_explorer.ts
View file

@ -74,11 +74,16 @@ export const metricsExplorerMetricRT = rt.intersection([
metricsExplorerMetricOptionalFieldsRT,
]);
export const timeRangeRT = rt.type({
from: rt.number,
to: rt.number,
interval: rt.string,
});
export const timeRangeRT = rt.intersection([
rt.type({
from: rt.number,
to: rt.number,
interval: rt.string,
}),
rt.partial({
timeFieldName: rt.string,
}),
]);
export const metricsExplorerRequestBodyRequiredFieldsRT = rt.type({
timerange: timeRangeRT,

2
x-pack/plugins/infra/server/lib/metrics/index.ts
View file

@ -42,7 +42,7 @@ export const query = async (
const filter: Array<Record<string, any>> = [
{
range: {
[TIMESTAMP_FIELD]: {
[options.timerange.timeFieldName || TIMESTAMP_FIELD]: {
gte: options.timerange.from,
lte: options.timerange.to,
format: 'epoch_millis',

2
x-pack/plugins/infra/server/lib/metrics/lib/create_aggregations.ts
View file

@ -37,7 +37,7 @@ const createMetricHistogramAggs = (options: MetricsAPIRequest): HistogramAggrega
return {
histogram: {
date_histogram: {
field: TIMESTAMP_FIELD,
field: options.timerange.timeFieldName || TIMESTAMP_FIELD,
fixed_interval: intervalString,
offset: options.alignDataToEnd ? calculateDateHistogramOffset(options.timerange) : '0s',
extended_bounds: {

1
x-pack/plugins/observability/common/threshold_rule/constants.ts
View file

@ -6,7 +6,6 @@
*/
export const SNAPSHOT_CUSTOM_AGGREGATIONS = ['avg', 'max', 'min', 'rate'] as const;
export const TIMESTAMP_FIELD = '@timestamp';
export const METRIC_EXPLORER_AGGREGATIONS = [
'avg',
'max',

5
x-pack/plugins/observability/public/components/threshold/components/expression_chart.tsx
View file

@ -58,6 +58,7 @@ interface Props {
groupBy?: string | string[];
hideTitle?: boolean;
timeRange?: TimeRange;
timeFieldName?: string;
}
export function ExpressionChart({
@ -69,6 +70,7 @@ export function ExpressionChart({
groupBy,
hideTitle = false,
timeRange,
timeFieldName,
}: Props) {
const { charts, uiSettings } = useKibana().services;
const { isLoading, data } = useMetricsExplorerChartData(
@ -76,7 +78,8 @@ export function ExpressionChart({
derivedIndexPattern,
filterQuery,
groupBy,
timeRange
timeRange,
timeFieldName
);
const chartRef = useRef(null);

7
x-pack/plugins/observability/public/components/threshold/hooks/use_metrics_explorer_chart_data.ts
View file

@ -19,13 +19,15 @@ import {
} from './use_metrics_explorer_options';
const DEFAULT_TIME_RANGE = {};
const DEFAULT_TIMESTAMP = '@timestamp';
export const useMetricsExplorerChartData = (
expression: MetricExpression,
derivedIndexPattern: DataViewBase,
filterQuery?: string,
groupBy?: string | string[],
timeRange: TimeRange = DEFAULT_TIME_RANGE
timeRange: TimeRange = DEFAULT_TIME_RANGE,
timeFieldName: string = DEFAULT_TIMESTAMP
) => {
const { timeSize, timeUnit } = expression || { timeSize: 1, timeUnit: 'm' };
@ -69,8 +71,9 @@ export const useMetricsExplorerChartData = (
interval: `>=${timeSize || 1}${timeUnit}`,
fromTimestamp,
toTimestamp,
timeFieldName,
};
}, [timeRange, timeSize, timeUnit]);
}, [timeRange.from, timeRange.to, timeSize, timeUnit, timeFieldName]);
return useMetricsExplorerData(options, derivedIndexPattern, timestamps);
};

13
x-pack/plugins/observability/public/components/threshold/hooks/use_metrics_explorer_data.test.tsx
View file

@ -20,9 +20,9 @@ import { DataViewBase } from '@kbn/es-query';
import {
createSeries,
derivedIndexPattern,
mockedTimestamps,
options,
resp,
timestamps,
} from '../../../utils/metrics_explorer';
const mockedFetch = jest.fn();
@ -59,7 +59,7 @@ const renderUseMetricsExplorerDataHook = () => {
initialProps: {
options,
derivedIndexPattern,
timestamps,
timestamps: mockedTimestamps,
},
wrapper,
}
@ -154,7 +154,7 @@ describe('useMetricsExplorerData Hook', () => {
metrics: [{ aggregation: 'count' }],
},
derivedIndexPattern,
timestamps,
timestamps: mockedTimestamps,
});
expect(result.current.isLoading).toBe(true);
await waitForNextUpdate();
@ -177,7 +177,12 @@ describe('useMetricsExplorerData Hook', () => {
rerender({
options,
derivedIndexPattern,
timestamps: { fromTimestamp: 1678378092225, toTimestamp: 1678381693477, interval: '>=10s' },
timestamps: {
fromTimestamp: 1678378092225,
toTimestamp: 1678381693477,
interval: '>=10s',
timeFieldName: 'mockedTimeFieldName',
},
});
expect(result.current.isLoading).toBe(true);
await waitForNextUpdate();

3
x-pack/plugins/observability/public/components/threshold/hooks/use_metrics_explorer_data.ts
View file

@ -24,7 +24,7 @@ import { decodeOrThrow } from '../helpers/runtime_types';
export function useMetricsExplorerData(
options: MetricsExplorerOptions,
derivedIndexPattern: DataViewBase,
{ fromTimestamp, toTimestamp, interval }: MetricsExplorerTimestampsRT,
{ fromTimestamp, toTimestamp, interval, timeFieldName }: MetricsExplorerTimestampsRT,
enabled = true
) {
const { http } = useKibana().services;
@ -62,6 +62,7 @@ export function useMetricsExplorerData(
void 0,
timerange: {
interval,
timeFieldName,
from: fromTimestamp,
to: toTimestamp,
},

8
x-pack/plugins/observability/public/components/threshold/hooks/use_metrics_explorer_options.ts
View file

@ -82,6 +82,7 @@ export const metricsExplorerTimestampsRT = t.type({
fromTimestamp: t.number,
toTimestamp: t.number,
interval: t.string,
timeFieldName: t.string,
});
export type MetricsExplorerTimestampsRT = t.TypeOf<typeof metricsExplorerTimestampsRT>;
@ -181,9 +182,10 @@ export const useMetricsExplorerOptions = () => {
to,
interval: DEFAULT_TIMERANGE.interval,
});
const [timestamps, setTimestamps] = useState<MetricsExplorerTimestampsRT>(
getDefaultTimeRange({ from, to })
);
const [timestamps, setTimestamps] = useState<MetricsExplorerTimestampsRT>({
...getDefaultTimeRange({ from, to }),
timeFieldName: '@timestamp',
});
useSyncKibanaTimeFilterTime(TIME_DEFAULTS, {
from: timeRange.from,

1
x-pack/plugins/observability/public/components/threshold/threshold_rule_expression.tsx
View file

@ -402,6 +402,7 @@ export default function Expressions(props: Props) {
derivedIndexPattern={derivedIndexPattern}
filterQuery={ruleParams.filterQueryText}
groupBy={ruleParams.groupBy}
timeFieldName={dataView?.timeFieldName}
/>
</ExpressionRow>
);

3
x-pack/plugins/observability/public/utils/metrics_explorer.ts
View file

@ -56,10 +56,11 @@ export const timeRange: MetricsExplorerTimeOptions = {
interval: '>=10s',
};
export const timestamps: MetricsExplorerTimestampsRT = {
export const mockedTimestamps: MetricsExplorerTimestampsRT = {
fromTimestamp: 1678376367166,
toTimestamp: 1678379973620,
interval: '>=10s',
timeFieldName: '@timestamp',
};
export const createSeries = (id: string): MetricsExplorerSeries => ({

3
x-pack/plugins/observability/server/lib/rules/threshold/lib/check_missing_group.ts
View file

@ -21,6 +21,7 @@ export const checkMissingGroups = async (
esClient: ElasticsearchClient,
metricParams: MetricExpressionParams,
indexPattern: string,
timeFieldName: string,
groupBy: string | undefined | string[],
filterQuery: string | undefined,
logger: Logger,
@ -31,7 +32,7 @@ export const checkMissingGroups = async (
return missingGroups;
}
const currentTimeframe = calculateCurrentTimeframe(metricParams, timeframe);
const baseFilters = createBaseFilters(metricParams, currentTimeframe, filterQuery);
const baseFilters = createBaseFilters(metricParams, currentTimeframe, timeFieldName, filterQuery);
const groupByFields = isString(groupBy) ? [groupBy] : groupBy ? groupBy : [];
const searches = missingGroups.flatMap((group) => {

3
x-pack/plugins/observability/server/lib/rules/threshold/lib/create_bucket_selector.ts
View file

@ -23,6 +23,7 @@ const EMPTY_SHOULD_WARN = {
export const createBucketSelector = (
condition: MetricExpressionParams,
alertOnGroupDisappear: boolean = false,
timeFieldName: string,
groupBy?: string | string[],
lastPeriodEnd?: number
) => {
@ -70,7 +71,7 @@ export const createBucketSelector = (
};
if (hasGroupBy && alertOnGroupDisappear && lastPeriodEnd) {
const wrappedPeriod = createLastPeriod(lastPeriodEnd, condition);
const wrappedPeriod = createLastPeriod(lastPeriodEnd, condition, timeFieldName);
aggs.lastPeriod = wrappedPeriod.lastPeriod;
aggs.missingGroup = {
bucket_script: {

7
x-pack/plugins/observability/server/lib/rules/threshold/lib/create_rate_aggregation.ts
View file

@ -6,7 +6,6 @@
*/
import moment from 'moment';
import { TIMESTAMP_FIELD } from '../../../../../common/threshold_rule/constants';
import { calculateRateTimeranges } from '../utils';
export const createRateAggsBucketScript = (
@ -31,7 +30,7 @@ export const createRateAggsBucketScript = (
};
export const createRateAggsBuckets = (
timeframe: { start: number; end: number },
timeframe: { start: number; end: number; timeFieldName: string },
id: string,
field: string
) => {
@ -44,7 +43,7 @@ export const createRateAggsBuckets = (
[`${id}_first_bucket`]: {
filter: {
range: {
[TIMESTAMP_FIELD]: {
[timeframe.timeFieldName]: {
gte: moment(firstBucketRange.from).toISOString(),
lt: moment(firstBucketRange.to).toISOString(),
},
@ -55,7 +54,7 @@ export const createRateAggsBuckets = (
[`${id}_second_bucket`]: {
filter: {
range: {
[TIMESTAMP_FIELD]: {
[timeframe.timeFieldName]: {
gte: moment(secondBucketRange.from).toISOString(),
lt: moment(secondBucketRange.to).toISOString(),
},

3
x-pack/plugins/observability/server/lib/rules/threshold/lib/evaluate_rule.ts
View file

@ -39,6 +39,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate
esClient: ElasticsearchClient,
params: Params,
dataView: string,
timeFieldName: string,
compositeSize: number,
alertOnGroupDisappear: boolean,
logger: Logger,
@ -64,6 +65,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate
esClient,
criterion,
dataView,
timeFieldName,
groupBy,
filterQuery,
compositeSize,
@ -77,6 +79,7 @@ export const evaluateRule = async <Params extends EvaluatedRuleParams = Evaluate
esClient,
criterion,
dataView,
timeFieldName,
groupBy,
filterQuery,
logger,

3
x-pack/plugins/observability/server/lib/rules/threshold/lib/get_data.ts
View file

@ -117,6 +117,7 @@ export const getData = async (
esClient: ElasticsearchClient,
params: MetricExpressionParams,
index: string,
timeFieldName: string,
groupBy: string | undefined | string[],
filterQuery: string | undefined,
compositeSize: number,
@ -191,6 +192,7 @@ export const getData = async (
esClient,
params,
index,
timeFieldName,
groupBy,
filterQuery,
compositeSize,
@ -265,6 +267,7 @@ export const getData = async (
body: getElasticsearchMetricQuery(
params,
timeframe,
timeFieldName,
compositeSize,
alertOnGroupDisappear,
lastPeriodEnd,

8
x-pack/plugins/observability/server/lib/rules/threshold/lib/metric_query.test.ts
View file

@ -24,6 +24,7 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
};
const groupBy = 'host.doggoname';
const timeFieldName = 'mockedTimeFieldName';
const timeframe = {
start: moment().subtract(5, 'minutes').valueOf(),
end: moment().valueOf(),
@ -33,6 +34,7 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
const searchBody = getElasticsearchMetricQuery(
expressionParams,
timeframe,
timeFieldName,
100,
true,
void 0,
@ -60,6 +62,7 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
const searchBody = getElasticsearchMetricQuery(
expressionParams,
timeframe,
timeFieldName,
100,
true,
void 0,
@ -74,7 +77,10 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
test('includes a metric field filter', () => {
expect(searchBody.query.bool.filter).toMatchObject(
expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }])
expect.arrayContaining([
{ range: { mockedTimeFieldName: expect.any(Object) } },
{ exists: { field: 'system.is.a.good.puppy.dog' } },
])
);
});
});

14
x-pack/plugins/observability/server/lib/rules/threshold/lib/metric_query.ts
View file

@ -44,12 +44,13 @@ export const calculateCurrentTimeframe = (
export const createBaseFilters = (
metricParams: MetricExpressionParams,
timeframe: { start: number; end: number },
timeFieldName: string,
filterQuery?: string
) => {
const rangeFilters = [
{
range: {
'@timestamp': {
[timeFieldName]: {
gte: moment(timeframe.start).toISOString(),
lte: moment(timeframe.end).toISOString(),
},
@ -76,6 +77,7 @@ export const createBaseFilters = (
export const getElasticsearchMetricQuery = (
metricParams: MetricExpressionParams,
timeframe: { start: number; end: number },
timeFieldName: string,
compositeSize: number,
alertOnGroupDisappear: boolean,
lastPeriodEnd?: number,
@ -91,9 +93,12 @@ export const getElasticsearchMetricQuery = (
);
}
// We need to make a timeframe that represents the current timeframe as oppose
// We need to make a timeframe that represents the current timeframe as opposed
// to the total timeframe (which includes the last period).
const currentTimeframe = calculateCurrentTimeframe(metricParams, timeframe);
const currentTimeframe = {
...calculateCurrentTimeframe(metricParams, timeframe),
timeFieldName,
};
const metricAggregations =
aggType === Aggregators.COUNT
@ -119,6 +124,7 @@ export const getElasticsearchMetricQuery = (
const bucketSelectorAggregations = createBucketSelector(
metricParams,
alertOnGroupDisappear,
timeFieldName,
groupBy,
lastPeriodEnd
);
@ -232,7 +238,7 @@ export const getElasticsearchMetricQuery = (
aggs.groupings.composite.after = afterKey;
}
const baseFilters = createBaseFilters(metricParams, timeframe, filterQuery);
const baseFilters = createBaseFilters(metricParams, timeframe, timeFieldName, filterQuery);
return {
track_total_hits: true,

10
x-pack/plugins/observability/server/lib/rules/threshold/lib/wrap_in_period.ts
View file

@ -7,18 +7,18 @@
import moment from 'moment';
import { MetricExpressionParams } from '../../../../../common/threshold_rule/types';
import { TIMESTAMP_FIELD } from '../../../../../common/threshold_rule/constants';
export const createLastPeriod = (
lastPeriodEnd: number,
{ timeUnit, timeSize }: MetricExpressionParams
{ timeUnit, timeSize }: MetricExpressionParams,
timeFieldName: string
) => {
const start = moment(lastPeriodEnd).subtract(timeSize, timeUnit).toISOString();
return {
lastPeriod: {
filter: {
range: {
[TIMESTAMP_FIELD]: {
[timeFieldName]: {
gte: start,
lte: moment(lastPeriodEnd).toISOString(),
},
@ -29,7 +29,7 @@ export const createLastPeriod = (
};
export const wrapInCurrentPeriod = <Aggs extends {}>(
timeframe: { start: number; end: number },
timeframe: { start: number; end: number; timeFieldName: string },
aggs: Aggs
) => {
return {
@ -38,7 +38,7 @@ export const wrapInCurrentPeriod = <Aggs extends {}>(
filters: {
all: {
range: {
[TIMESTAMP_FIELD]: {
[timeframe.timeFieldName]: {
gte: moment(timeframe.start).toISOString(),
lte: moment(timeframe.end).toISOString(),
},

4
x-pack/plugins/observability/server/lib/rules/threshold/threshold_executor.ts
View file

@ -205,14 +205,18 @@ export const createMetricThresholdExecutor = ({
const initialSearchSource = await searchSourceClient.create(params.searchConfiguration!);
const dataView = initialSearchSource.getField('index')!.getIndexPattern();
const timeFieldName = initialSearchSource.getField('index')?.timeFieldName;
if (!dataView) {
throw new Error('No matched data view');
} else if (!timeFieldName) {
throw new Error('No timestamp field is specified');
}
const alertResults = await evaluateRule(
services.scopedClusterClient.asCurrentUser,
params as EvaluatedRuleParams,
dataView,
timeFieldName,
compositeSize,
alertOnGroupDisappear,
logger,