github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[SLO] Account for the built-in delay for burn rate alerting (#169011)

Browse source 
## Summary

This PR introduces a delay based on:

- The interval of the date histogram which defaults to `1m`
- The sync delay of the transform which defaults to `1m`
- The frequency of the transform which defaults to `1m`

On average the SLO data is about `180s` behind for occurances. If the
user uses `5m` time slices then the delay is around `420s`. This PR
attempts to mitigate this delay by subtracting the `interval + syncDelay
+ frequency` from `now` on any calculation for the burn rate so that the
last 5 minutes is aligned with the data. Below is a visualization that
shows how much delay we are seeing in an optimal environment.

<img width="884" alt="image"
src="2a1587cd-c789-403c-97e2-f48c65db2b89">

Using `interval + syncDelay + frequency` accounts for the "best case
scenario". Due to the nature of the transform system, the delays varies
from best case of `180s` for occurrences to worst case of around `240s`
which happens right before the next scheduled query; the transform query
runs every `60s` which accounts for the variation between the worst and
best case delay. Since the rules run on a seperate schedule, it's hard
to know where we are in the `60s` cycle so the best we can do is account
for the "best case".

This PR also fixes #168747

### Note to the reviewer

The changes made to `evaluate.ts` and `build_query.ts` look more
extensive than they really are. I felt like #168735 made some
unnecessary refactors when they simply could have done a minimal change
and left the rest of the code alone; it would have been less risky. This
also cause several issues during the merge which is why I ultimately
decided to revert the changes from #168735.
This commit is contained in:
Chris Cowan 2023-10-19 14:09:18 -06:00 • committed by GitHub
parent 36776af50f
commit dce8eedf56
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 337 additions and 227 deletions
Download patch file Download diff file Expand all files Collapse all files

9
x-pack/plugins/observability/public/components/slo/error_rate_chart/error_rate_chart.tsx
View file

@ -10,6 +10,7 @@ import { SLOResponse } from '@kbn/slo-schema';
import moment from 'moment';
import React from 'react';
import { useKibana } from '../../../utils/kibana_react';
import { getDelayInSecondsFromSLO } from '../../../utils/slo/get_delay_in_seconds_from_slo';
import { useLensDefinition } from './use_lens_definition';
interface Props {
@ -22,14 +23,18 @@ export function ErrorRateChart({ slo, fromRange }: Props) {
lens: { EmbeddableComponent },
} = useKibana().services;
const lensDef = useLensDefinition(slo);
const delayInSeconds = getDelayInSecondsFromSLO(slo);
const from = moment(fromRange).subtract(delayInSeconds, 'seconds').toISOString();
const to = moment().subtract(delayInSeconds, 'seconds').toISOString();
return (
<EmbeddableComponent
id="sloErrorRateChart"
style={{ height: 190 }}
timeRange={{
from: fromRange.toISOString(),
to: moment().toISOString(),
from,
to,
}}
attributes={lensDef}
viewMode={ViewMode.VIEW}

8
x-pack/plugins/observability/public/components/slo/error_rate_chart/use_lens_definition.ts
View file

@ -7,11 +7,15 @@
import { useEuiTheme } from '@elastic/eui';
import { TypedLensByValueInput } from '@kbn/lens-plugin/public';
import { ALL_VALUE, SLOResponse } from '@kbn/slo-schema';
import { ALL_VALUE, SLOResponse, timeslicesBudgetingMethodSchema } from '@kbn/slo-schema';
export function useLensDefinition(slo: SLOResponse): TypedLensByValueInput['attributes'] {
const { euiTheme } = useEuiTheme();
const interval = timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)
? slo.objective.timesliceWindow
: '60s';
return {
title: 'SLO Error Rate',
description: '',
@ -125,7 +129,7 @@ export function useLensDefinition(slo: SLOResponse): TypedLensByValueInput['attr
scale: 'interval',
params: {
// @ts-ignore
interval: 'auto',
interval,
includeEmptyRows: true,
dropPartials: false,
},

29
x-pack/plugins/observability/public/utils/slo/get_delay_in_seconds_from_slo.ts Normal file
View file

@ -0,0 +1,29 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SLOResponse, timeslicesBudgetingMethodSchema, durationType } from '@kbn/slo-schema';
import { isLeft } from 'fp-ts/lib/Either';
export function getDelayInSecondsFromSLO(slo: SLOResponse) {
const fixedInterval = timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)
? durationStringToSeconds(slo.objective.timesliceWindow)
: 60;
const syncDelay = durationStringToSeconds(slo.settings.syncDelay);
const frequency = durationStringToSeconds(slo.settings.frequency);
return fixedInterval + syncDelay + frequency;
}
function durationStringToSeconds(duration: string | undefined) {
if (!duration) {
return 0;
}
const result = durationType.decode(duration);
if (isLeft(result)) {
throw new Error(`Invalid duration string: ${duration}`);
}
return result.right.asSeconds();
}

18
x-pack/plugins/observability/server/domain/services/get_delay_in_seconds_from_slo.ts Normal file
View file

@ -0,0 +1,18 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { timeslicesBudgetingMethodSchema } from '@kbn/slo-schema';
import { SLO } from '../models';
export function getDelayInSecondsFromSLO(slo: SLO) {
const fixedInterval = timeslicesBudgetingMethodSchema.is(slo.budgetingMethod)
? slo.objective.timesliceWindow!.asSeconds()
: 60;
const syncDelay = slo.settings.syncDelay.asSeconds();
const frequency = slo.settings.frequency.asSeconds();
return fixedInterval + syncDelay + frequency;
}

23
x-pack/plugins/observability/server/domain/services/get_lookback_date_range.ts Normal file
View file

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import moment from 'moment';
import { Duration, toMomentUnitOfTime } from '../models';
export function getLookbackDateRange(
startedAt: Date,
duration: Duration,
delayInSeconds = 0
): { from: Date; to: Date } {
const unit = toMomentUnitOfTime(duration.unit);
const now = moment(startedAt).subtract(delayInSeconds, 'seconds').startOf('minute');
const from = now.clone().subtract(duration.value, unit).startOf('minute');
return {
from: from.toDate(),
to: now.toDate(),
};
}

35
x-pack/plugins/observability/server/lib/rules/slo_burn_rate/executor.ts
View file

@ -19,7 +19,7 @@ import { LocatorPublic } from '@kbn/share-plugin/common';
import { upperCase } from 'lodash';
import { addSpaceIdToPath } from '@kbn/spaces-plugin/server';
import { ALL_VALUE, toDurationUnit } from '@kbn/slo-schema';
import { ALL_VALUE } from '@kbn/slo-schema';
import { AlertsLocatorParams, getAlertUrl } from '../../../../common';
import {
SLO_ID_FIELD,
@ -89,22 +89,10 @@ export const getRuleExecutor = ({
return { state: {} };
}
const burnRateWindows = getBurnRateWindows(params.windows);
const longestLookbackWindow = burnRateWindows.reduce((acc, winDef) => {
return winDef.longDuration.isShorterThan(acc.longDuration) ? acc : winDef;
}, burnRateWindows[0]);
const { dateStart, dateEnd } = getTimeRange(
`${longestLookbackWindow.longDuration.value}${longestLookbackWindow.longDuration.unit}`
);
const results = await evaluate(
esClient.asCurrentUser,
slo,
params,
dateStart,
dateEnd,
burnRateWindows
);
// We only need the end timestamp to base all of queries on. The length of the time range
// doesn't matter for our use case since we allow the user to customize the window sizes,
const { dateEnd } = getTimeRange('1m');
const results = await evaluate(esClient.asCurrentUser, slo, params, new Date(dateEnd));
if (results.length > 0) {
for (const result of results) {
@ -212,19 +200,6 @@ export const getRuleExecutor = ({
return { state: {} };
};
export function getBurnRateWindows(windows: WindowSchema[]) {
return windows.map((winDef) => {
return {
...winDef,
longDuration: new Duration(winDef.longWindow.value, toDurationUnit(winDef.longWindow.unit)),
shortDuration: new Duration(
winDef.shortWindow.value,
toDurationUnit(winDef.shortWindow.unit)
),
};
});
}
function getActionGroupName(id: string) {
switch (id) {
case HIGH_PRIORITY_ACTION.id:

124
x-pack/plugins/observability/server/lib/rules/slo_burn_rate/lib/__snapshots__/build_query.test.ts.snap generated
View file

@ -21,8 +21,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T22:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -70,8 +70,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:55:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T23:52:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -119,8 +119,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T18:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T17:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -168,8 +168,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:30:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T23:27:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -217,8 +217,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-30T23:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -266,8 +266,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T22:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T21:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -315,8 +315,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-29T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-28T23:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -364,8 +364,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T18:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T17:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -445,8 +445,8 @@ Object {
Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-29T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-28T23:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -478,8 +478,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T22:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -505,7 +505,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -527,8 +527,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:55:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T23:51:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -554,7 +554,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -576,8 +576,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T18:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T17:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -603,7 +603,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -625,8 +625,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:30:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T23:26:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -652,7 +652,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -674,8 +674,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-30T23:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -701,7 +701,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -723,8 +723,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T22:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T21:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -750,7 +750,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -772,8 +772,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-29T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-28T23:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -799,7 +799,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -821,8 +821,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T18:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T17:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -848,7 +848,7 @@ Object {
},
"script": Object {
"params": Object {
"target": 0.999,
"target": 0.98,
},
"source": "params.total != null && params.total > 0 ? (1 - (params.good / params.total)) / (1 - params.target) : 0",
},
@ -902,8 +902,8 @@ Object {
Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-29T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-28T23:56:00.000Z",
"lt": "2022-12-31T23:56:00.000Z",
},
},
},
@ -935,8 +935,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T22:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -984,8 +984,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:55:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T23:52:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1033,8 +1033,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T18:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T17:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1082,8 +1082,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T23:30:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T23:27:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1131,8 +1131,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-30T23:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1180,8 +1180,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T22:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T21:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1229,8 +1229,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-29T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-28T23:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1278,8 +1278,8 @@ Object {
"filter": Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-31T18:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-31T17:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},
@ -1362,8 +1362,8 @@ Object {
Object {
"range": Object {
"@timestamp": Object {
"gte": "2022-12-29T00:00:00.000Z",
"lt": "2023-01-01T00:00:00.000Z",
"gte": "2022-12-28T23:57:00.000Z",
"lt": "2022-12-31T23:57:00.000Z",
},
},
},

26
x-pack/plugins/observability/server/lib/rules/slo_burn_rate/lib/build_query.test.ts
View file

@ -7,11 +7,13 @@
import { createBurnRateRule } from '../fixtures/rule';
import { buildQuery } from './build_query';
import { createKQLCustomIndicator, createSLO } from '../../../../services/slo/fixtures/slo';
import { getBurnRateWindows } from '../executor';
import {
createKQLCustomIndicator,
createSLO,
createSLOWithTimeslicesBudgetingMethod,
} from '../../../../services/slo/fixtures/slo';
const DATE_START = '2022-12-29T00:00:00.000Z';
const DATE_END = '2023-01-01T00:00:00.000Z';
const STARTED_AT = new Date('2023-01-01T00:00:00.000Z');
describe('buildQuery()', () => {
it('should return a valid query for occurrences', () => {
@ -20,8 +22,7 @@ describe('buildQuery()', () => {
indicator: createKQLCustomIndicator(),
});
const rule = createBurnRateRule(slo);
const burnRateWindows = getBurnRateWindows(rule.windows);
expect(buildQuery(slo, DATE_START, DATE_END, burnRateWindows)).toMatchSnapshot();
expect(buildQuery(STARTED_AT, slo, rule)).toMatchSnapshot();
});
it('should return a valid query with afterKey', () => {
const slo = createSLO({
@ -29,21 +30,14 @@ describe('buildQuery()', () => {
indicator: createKQLCustomIndicator(),
});
const rule = createBurnRateRule(slo);
const burnRateWindows = getBurnRateWindows(rule.windows);
expect(
buildQuery(slo, DATE_START, DATE_END, burnRateWindows, {
instanceId: 'example',
})
).toMatchSnapshot();
expect(buildQuery(STARTED_AT, slo, rule, { instanceId: 'example' })).toMatchSnapshot();
});
it('should return a valid query for timeslices', () => {
const slo = createSLO({
const slo = createSLOWithTimeslicesBudgetingMethod({
id: 'test-slo',
indicator: createKQLCustomIndicator(),
budgetingMethod: 'timeslices',
});
const rule = createBurnRateRule(slo);
const burnRateWindows = getBurnRateWindows(rule.windows);
expect(buildQuery(slo, DATE_START, DATE_END, burnRateWindows)).toMatchSnapshot();
expect(buildQuery(STARTED_AT, slo, rule)).toMatchSnapshot();
});
});

62
x-pack/plugins/observability/server/lib/rules/slo_burn_rate/lib/build_query.ts
View file

@ -5,12 +5,13 @@
* 2.0.
*/
import moment from 'moment';
import { timeslicesBudgetingMethodSchema } from '@kbn/slo-schema';
import { Duration, SLO, toMomentUnitOfTime } from '../../../../domain/models';
import { WindowSchema } from '../types';
import { Duration, SLO, toDurationUnit } from '../../../../domain/models';
import { BurnRateRuleParams, WindowSchema } from '../types';
import { getDelayInSecondsFromSLO } from '../../../../domain/services/get_delay_in_seconds_from_slo';
import { getLookbackDateRange } from '../../../../domain/services/get_lookback_date_range';
export type BurnRateWindowWithDuration = WindowSchema & {
type BurnRateWindowWithDuration = WindowSchema & {
longDuration: Duration;
shortDuration: Duration;
};
@ -100,13 +101,14 @@ function buildWindowAgg(
}
function buildWindowAggs(
startedAt: string,
startedAt: Date,
slo: SLO,
burnRateWindows: BurnRateWindowWithDuration[]
burnRateWindows: BurnRateWindowWithDuration[],
delayInSeconds = 0
) {
return burnRateWindows.reduce((acc, winDef, index) => {
const shortDateRange = getLookbackDateRange(startedAt, winDef.shortDuration);
const longDateRange = getLookbackDateRange(startedAt, winDef.longDuration);
const shortDateRange = getLookbackDateRange(startedAt, winDef.shortDuration, delayInSeconds);
const longDateRange = getLookbackDateRange(startedAt, winDef.longDuration, delayInSeconds);
const windowId = generateWindowId(index);
return {
...acc,
@ -154,12 +156,32 @@ function buildEvaluation(burnRateWindows: BurnRateWindowWithDuration[]) {
}
export function buildQuery(
startedAt: Date,
slo: SLO,
dateStart: string,
dateEnd: string,
burnRateWindows: BurnRateWindowWithDuration[],
params: BurnRateRuleParams,
afterKey?: EvaluationAfterKey
) {
const delayInSeconds = getDelayInSecondsFromSLO(slo);
const burnRateWindows = params.windows.map((winDef) => {
return {
...winDef,
longDuration: new Duration(winDef.longWindow.value, toDurationUnit(winDef.longWindow.unit)),
shortDuration: new Duration(
winDef.shortWindow.value,
toDurationUnit(winDef.shortWindow.unit)
),
};
});
const longestLookbackWindow = burnRateWindows.reduce((acc, winDef) => {
return winDef.longDuration.isShorterThan(acc.longDuration) ? acc : winDef;
}, burnRateWindows[0]);
const longestDateRange = getLookbackDateRange(
startedAt,
longestLookbackWindow.longDuration,
delayInSeconds
);
return {
size: 0,
query: {
@ -170,8 +192,8 @@ export function buildQuery(
{
range: {
'@timestamp': {
gte: dateStart,
lt: dateEnd,
gte: longestDateRange.from.toISOString(),
lt: longestDateRange.to.toISOString(),
},
},
},
@ -186,22 +208,10 @@ export function buildQuery(
sources: [{ instanceId: { terms: { field: 'slo.instanceId' } } }],
},
aggs: {
...buildWindowAggs(dateEnd, slo, burnRateWindows),
...buildWindowAggs(startedAt, slo, burnRateWindows, delayInSeconds),
...buildEvaluation(burnRateWindows),
},
},
},
};
}
function getLookbackDateRange(startedAt: string, duration: Duration): { from: Date; to: Date } {
const unit = toMomentUnitOfTime(duration.unit);
const now = moment(startedAt);
const from = now.clone().subtract(duration.value, unit);
const to = now.clone();
return {
from: from.toDate(),
to: to.toDate(),
};
}

19
x-pack/plugins/observability/server/lib/rules/slo_burn_rate/lib/evaluate.ts
View file

@ -12,7 +12,6 @@ import { BurnRateRuleParams } from '../types';
import { SLO_DESTINATION_INDEX_PATTERN } from '../../../../assets/constants';
import {
buildQuery,
BurnRateWindowWithDuration,
EvaluationAfterKey,
generateAboveThresholdKey,
generateBurnRateKey,
@ -66,13 +65,12 @@ export interface EvalutionAggResults {
async function queryAllResults(
esClient: ElasticsearchClient,
slo: SLO,
dateStart: string,
dateEnd: string,
burnRateWindows: BurnRateWindowWithDuration[],
params: BurnRateRuleParams,
startedAt: Date,
buckets: EvaluationBucket[] = [],
lastAfterKey?: { instanceId: string }
): Promise<EvaluationBucket[]> {
const queryAndAggs = buildQuery(slo, dateStart, dateEnd, burnRateWindows, lastAfterKey);
const queryAndAggs = buildQuery(startedAt, slo, params, lastAfterKey);
const results = await esClient.search<undefined, EvalutionAggResults>({
index: SLO_DESTINATION_INDEX_PATTERN,
...queryAndAggs,
@ -86,9 +84,8 @@ async function queryAllResults(
return queryAllResults(
esClient,
slo,
dateStart,
dateEnd,
burnRateWindows,
params,
startedAt,
[...buckets, ...results.aggregations.instances.buckets],
results.aggregations.instances.after_key
);
@ -98,11 +95,9 @@ export async function evaluate(
esClient: ElasticsearchClient,
slo: SLO,
params: BurnRateRuleParams,
dateStart: string,
dateEnd: string,
burnRateWindows: BurnRateWindowWithDuration[]
startedAt: Date
) {
const buckets = await queryAllResults(esClient, slo, dateStart, dateEnd, burnRateWindows);
const buckets = await queryAllResults(esClient, slo, params, startedAt);
return transformBucketToResults(buckets, params);
}

75
x-pack/plugins/observability/server/services/slo/sli_client.test.ts
View file

@ -27,11 +27,18 @@ const commonEsResponse = {
},
};
const TEST_DATE = new Date('2023-01-01T00:00:00.000Z');
describe('SummaryClient', () => {
let esClientMock: ElasticsearchClientMock;
beforeEach(() => {
esClientMock = elasticsearchServiceMock.createElasticsearchClient();
jest.useFakeTimers().setSystemTime(TEST_DATE);
});
afterAll(() => {
jest.useRealTimers();
});
describe('fetchSLIDataFrom', () => {
@ -51,11 +58,11 @@ describe('SummaryClient', () => {
[LONG_WINDOW]: {
buckets: [
{
key: '2022-11-08T13:53:00.000Z-2022-11-08T14:53:00.000Z',
from: 1667915580000,
from_as_string: '2022-11-08T13:53:00.000Z',
to: 1667919180000,
to_as_string: '2022-11-08T14:53:00.000Z',
key: '2022-12-31T22:54:00.000Z-2022-12-31T23:54:00.000Z',
from: 1672527240000,
from_as_string: '2022-12-31T22:54:00.000Z',
to: 1672530840000,
to_as_string: '2022-12-31T23:54:00.000Z',
doc_count: 60,
total: {
value: 32169,
@ -69,11 +76,11 @@ describe('SummaryClient', () => {
[SHORT_WINDOW]: {
buckets: [
{
key: '2022-11-08T14:48:00.000Z-2022-11-08T14:53:00.000Z',
from: 1667918880000,
from_as_string: '2022-11-08T14:48:00.000Z',
to: 1667919180000,
to_as_string: '2022-11-08T14:53:00.000Z',
key: '2022-12-31T23:49:00.000Z-2022-12-31T23:54:00.000Z',
from: 1672530540000,
from_as_string: '2022-12-31T23:49:00.000Z',
to: 1672530840000,
to_as_string: '2022-12-31T23:54:00.000Z',
doc_count: 5,
total: {
value: 2211,
@ -95,7 +102,12 @@ describe('SummaryClient', () => {
[LONG_WINDOW]: {
date_range: {
field: '@timestamp',
ranges: [{ from: 'now-1h/m', to: 'now/m' }],
ranges: [
{
from: '2022-12-31T22:54:00.000Z',
to: '2022-12-31T23:54:00.000Z',
},
],
},
aggs: {
good: { sum: { field: 'slo.numerator' } },
@ -105,7 +117,12 @@ describe('SummaryClient', () => {
[SHORT_WINDOW]: {
date_range: {
field: '@timestamp',
ranges: [{ from: 'now-5m/m', to: 'now/m' }],
ranges: [
{
from: '2022-12-31T23:49:00.000Z',
to: '2022-12-31T23:54:00.000Z',
},
],
},
aggs: {
good: { sum: { field: 'slo.numerator' } },
@ -141,11 +158,11 @@ describe('SummaryClient', () => {
[LONG_WINDOW]: {
buckets: [
{
key: '2022-11-08T13:53:00.000Z-2022-11-08T14:53:00.000Z',
from: 1667915580000,
from_as_string: '2022-11-08T13:53:00.000Z',
to: 1667919180000,
to_as_string: '2022-11-08T14:53:00.000Z',
key: '2022-12-31T22:36:00.000Z-2022-12-31T23:36:00.000Z',
from: 1672526160000,
from_as_string: '2022-12-31T22:36:00.000Z',
to: 1672529760000,
to_as_string: '2022-12-31T23:36:00.000Z',
doc_count: 60,
total: {
value: 32169,
@ -159,11 +176,11 @@ describe('SummaryClient', () => {
[SHORT_WINDOW]: {
buckets: [
{
key: '2022-11-08T14:48:00.000Z-2022-11-08T14:53:00.000Z',
from: 1667918880000,
from_as_string: '2022-11-08T14:48:00.000Z',
to: 1667919180000,
to_as_string: '2022-11-08T14:53:00.000Z',
key: '2022-12-31T23:31:00.000Z-2022-12-31T23:36:00.000Z',
from: 1672529460000,
from_as_string: '2022-12-31T23:31:00.000Z',
to: 1672529760000,
to_as_string: '2022-12-31T23:36:00.000Z',
doc_count: 5,
total: {
value: 2211,
@ -185,7 +202,12 @@ describe('SummaryClient', () => {
[LONG_WINDOW]: {
date_range: {
field: '@timestamp',
ranges: [{ from: 'now-1h/m', to: 'now/m' }],
ranges: [
{
from: '2022-12-31T22:36:00.000Z',
to: '2022-12-31T23:36:00.000Z',
},
],
},
aggs: {
good: {
@ -203,7 +225,12 @@ describe('SummaryClient', () => {
[SHORT_WINDOW]: {
date_range: {
field: '@timestamp',
ranges: [{ from: 'now-5m/m', to: 'now/m' }],
ranges: [
{
from: '2022-12-31T23:31:00.000Z',
to: '2022-12-31T23:36:00.000Z',
},
],
},
aggs: {
good: {

136
x-pack/plugins/observability/server/services/slo/sli_client.ts
View file

@ -18,13 +18,13 @@ import {
ALL_VALUE,
occurrencesBudgetingMethodSchema,
timeslicesBudgetingMethodSchema,
toMomentUnitOfTime,
} from '@kbn/slo-schema';
import { assertNever } from '@kbn/std';
import moment from 'moment';
import { SLO_DESTINATION_INDEX_PATTERN } from '../../assets/constants';
import { DateRange, Duration, IndicatorData, SLO } from '../../domain/models';
import { InternalQueryError } from '../../errors';
import { getDelayInSecondsFromSLO } from '../../domain/services/get_delay_in_seconds_from_slo';
import { getLookbackDateRange } from '../../domain/services/get_lookback_date_range';
export interface SLIClient {
fetchSLIDataFrom(
@ -55,13 +55,22 @@ export class DefaultSLIClient implements SLIClient {
a.duration.isShorterThan(b.duration) ? 1 : -1
);
const longestLookbackWindow = sortedLookbackWindows[0];
const longestDateRange = getLookbackDateRange(longestLookbackWindow.duration);
const delayInSeconds = getDelayInSecondsFromSLO(slo);
const longestDateRange = getLookbackDateRange(
new Date(),
longestLookbackWindow.duration,
delayInSeconds
);
if (occurrencesBudgetingMethodSchema.is(slo.budgetingMethod)) {
const result = await this.esClient.search<unknown, EsAggregations>({
...commonQuery(slo, instanceId, longestDateRange),
index: SLO_DESTINATION_INDEX_PATTERN,
aggs: toLookbackWindowsAggregationsQuery(sortedLookbackWindows),
aggs: toLookbackWindowsAggregationsQuery(
longestDateRange.to,
sortedLookbackWindows,
delayInSeconds
),
});
return handleWindowedResult(result.aggregations, lookbackWindows);
@ -71,7 +80,11 @@ export class DefaultSLIClient implements SLIClient {
const result = await this.esClient.search<unknown, EsAggregations>({
...commonQuery(slo, instanceId, longestDateRange),
index: SLO_DESTINATION_INDEX_PATTERN,
aggs: toLookbackWindowsSlicedAggregationsQuery(slo, sortedLookbackWindows),
aggs: toLookbackWindowsSlicedAggregationsQuery(
longestDateRange.to,
sortedLookbackWindows,
delayInSeconds
),
});
return handleWindowedResult(result.aggregations, lookbackWindows);
@ -110,53 +123,82 @@ function commonQuery(
};
}
function toLookbackWindowsAggregationsQuery(sortedLookbackWindow: LookbackWindow[]) {
function toLookbackWindowsAggregationsQuery(
startedAt: Date,
sortedLookbackWindow: LookbackWindow[],
delayInSeconds = 0
) {
return sortedLookbackWindow.reduce<Record<string, AggregationsAggregationContainer>>(
(acc, lookbackWindow) => ({
...acc,
[lookbackWindow.name]: {
date_range: {
field: '@timestamp',
ranges: [{ from: `now-${lookbackWindow.duration.format()}/m`, to: 'now/m' }],
(acc, lookbackWindow) => {
const lookbackDateRange = getLookbackDateRange(
startedAt,
lookbackWindow.duration,
delayInSeconds
);
return {
...acc,
[lookbackWindow.name]: {
date_range: {
field: '@timestamp',
ranges: [
{
from: lookbackDateRange.from.toISOString(),
to: lookbackDateRange.to.toISOString(),
},
],
},
aggs: {
good: { sum: { field: 'slo.numerator' } },
total: { sum: { field: 'slo.denominator' } },
},
},
aggs: {
good: { sum: { field: 'slo.numerator' } },
total: { sum: { field: 'slo.denominator' } },
},
},
}),
};
},
{}
);
}
function toLookbackWindowsSlicedAggregationsQuery(slo: SLO, lookbackWindows: LookbackWindow[]) {
function toLookbackWindowsSlicedAggregationsQuery(
startedAt: Date,
lookbackWindows: LookbackWindow[],
delayInSeconds = 0
) {
return lookbackWindows.reduce<Record<string, AggregationsAggregationContainer>>(
(acc, lookbackWindow) => ({
...acc,
[lookbackWindow.name]: {
date_range: {
field: '@timestamp',
ranges: [
{
from: `now-${lookbackWindow.duration.format()}/m`,
to: 'now/m',
},
],
},
aggs: {
good: {
sum: {
field: 'slo.isGoodSlice',
},
(acc, lookbackWindow) => {
const lookbackDateRange = getLookbackDateRange(
startedAt,
lookbackWindow.duration,
delayInSeconds
);
return {
...acc,
[lookbackWindow.name]: {
date_range: {
field: '@timestamp',
ranges: [
{
from: lookbackDateRange.from.toISOString(),
to: lookbackDateRange.to.toISOString(),
},
],
},
total: {
value_count: {
field: 'slo.isGoodSlice',
aggs: {
good: {
sum: {
field: 'slo.isGoodSlice',
},
},
total: {
value_count: {
field: 'slo.isGoodSlice',
},
},
},
},
},
}),
};
},
{}
);
}
@ -191,15 +233,3 @@ function handleWindowedResult(
return indicatorDataPerLookbackWindow;
}
function getLookbackDateRange(duration: Duration): { from: Date; to: Date } {
const unit = toMomentUnitOfTime(duration.unit);
const now = moment.utc().startOf('minute');
const from = now.clone().subtract(duration.value, unit);
const to = now.clone();
return {
from: from.toDate(),
to: to.toDate(),
};
}