feature(slo): Run transforms unattended (#167170)

This commit is contained in:
Kevin Delemme 2023-09-26 08:05:47 -04:00 committed by GitHub
parent 9198475715
commit df29ea62a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 109 additions and 39 deletions

View file

@ -6,6 +6,7 @@
*/
export const SLO_RESOURCES_VERSION = 2;
export const SLO_SUMMARY_TRANSFORMS_VERSION = 3;
export const SLO_COMPONENT_TEMPLATE_MAPPINGS_NAME = '.slo-observability.sli-mappings';
export const SLO_COMPONENT_TEMPLATE_SETTINGS_NAME = '.slo-observability.sli-settings';

View file

@ -36,6 +36,7 @@ export const getSLOTransformTemplate = (
dest: destination,
settings: {
deduce_mappings: false,
unattended: true,
},
sync: {
time: {

View file

@ -55,6 +55,7 @@ describe('CreateSLO', () => {
expect(mockTransformManager.install).toHaveBeenCalledWith(
expect.objectContaining({ ...sloParams, id: 'unique-id' })
);
expect(mockTransformManager.preview).toHaveBeenCalledWith('slo-transform-id');
expect(mockTransformManager.start).toHaveBeenCalledWith('slo-transform-id');
expect(response).toEqual(expect.objectContaining({ id: 'unique-id' }));
expect(esClientMock.index.mock.calls[0]).toMatchSnapshot();
@ -100,6 +101,16 @@ describe('CreateSLO', () => {
expect(mockRepository.deleteById).toBeCalled();
});
it('removes the transform and deletes the SLO when transform preview fails', async () => {
mockTransformManager.install.mockResolvedValue('slo-transform-id');
mockTransformManager.preview.mockRejectedValue(new Error('Transform preview error'));
const sloParams = createSLOParams({ indicator: createAPMTransactionErrorRateIndicator() });
await expect(createSLO.execute(sloParams)).rejects.toThrowError('Transform preview error');
expect(mockTransformManager.uninstall).toBeCalledWith('slo-transform-id');
expect(mockRepository.deleteById).toBeCalled();
});
it('removes the transform and deletes the SLO when transform start fails', async () => {
mockTransformManager.install.mockResolvedValue('slo-transform-id');
mockTransformManager.start.mockRejectedValue(new Error('Transform start error'));

View file

@ -36,6 +36,7 @@ export class CreateSLO {
}
try {
await this.transformManager.preview(sloTransformId);
await this.transformManager.start(sloTransformId);
} catch (err) {
await Promise.all([

View file

@ -28,6 +28,7 @@ const createSummaryTransformInstallerMock = (): jest.Mocked<SummaryTransformInst
const createTransformManagerMock = (): jest.Mocked<TransformManager> => {
return {
install: jest.fn(),
preview: jest.fn(),
uninstall: jest.fn(),
start: jest.fn(),
stop: jest.fn(),

View file

@ -7,7 +7,7 @@ Array [
"_meta": Object {
"managed": true,
"managed_by": "observability",
"version": 2,
"version": 3,
},
"description": "Summarize every SLO with timeslices budgeting method and a 7 days rolling time window",
"dest": Object {
@ -173,6 +173,7 @@ Array [
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": ".slo-observability.sli-v2*",
@ -235,7 +236,7 @@ Array [
"_meta": Object {
"managed": true,
"managed_by": "observability",
"version": 2,
"version": 3,
},
"description": "Summarize every SLO with timeslices budgeting method and a 30 days rolling time window",
"dest": Object {
@ -401,6 +402,7 @@ Array [
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": ".slo-observability.sli-v2*",
@ -463,7 +465,7 @@ Array [
"_meta": Object {
"managed": true,
"managed_by": "observability",
"version": 2,
"version": 3,
},
"description": "Summarize every SLO with timeslices budgeting method and a 90 days rolling time window",
"dest": Object {
@ -629,6 +631,7 @@ Array [
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": ".slo-observability.sli-v2*",
@ -691,7 +694,7 @@ Array [
"_meta": Object {
"managed": true,
"managed_by": "observability",
"version": 2,
"version": 3,
},
"description": "Summarize every SLO with timeslices budgeting method and a weekly calendar aligned time window",
"dest": Object {
@ -870,6 +873,7 @@ Array [
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": ".slo-observability.sli-v2*",
@ -932,7 +936,7 @@ Array [
"_meta": Object {
"managed": true,
"managed_by": "observability",
"version": 2,
"version": 3,
},
"description": "Summarize every SLO with timeslices budgeting method and a monthly calendar aligned time window",
"dest": Object {
@ -1126,6 +1130,7 @@ Array [
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": ".slo-observability.sli-v2*",

View file

@ -8,7 +8,7 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import type { ElasticsearchClient, Logger } from '@kbn/core/server';
import {
SLO_RESOURCES_VERSION,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../assets/constants';
import { retryTransientEsErrors } from '../../../utils/retry';
@ -32,7 +32,7 @@ export class DefaultSummaryTransformInstaller implements SummaryTransformInstall
const alreadyInstalled =
summaryTransforms.count === allTransformIds.length &&
summaryTransforms.transforms.every(
(transform) => transform._meta?.version === SLO_RESOURCES_VERSION
(transform) => transform._meta?.version === SLO_SUMMARY_TRANSFORMS_VERSION
) &&
summaryTransforms.transforms.every((transform) => allTransformIds.includes(transform.id));
@ -46,9 +46,9 @@ export class DefaultSummaryTransformInstaller implements SummaryTransformInstall
const transform = summaryTransforms.transforms.find((t) => t.id === transformId);
const transformAlreadyInstalled =
!!transform && transform._meta?.version === SLO_RESOURCES_VERSION;
!!transform && transform._meta?.version === SLO_SUMMARY_TRANSFORMS_VERSION;
const previousTransformAlreadyInstalled =
!!transform && transform._meta?.version !== SLO_RESOURCES_VERSION;
!!transform && transform._meta?.version !== SLO_SUMMARY_TRANSFORMS_VERSION;
if (transformAlreadyInstalled) {
this.logger.info(`SLO summary transform [${transformId}] already installed - skipping`);

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -143,10 +143,10 @@ export const SUMMARY_OCCURRENCES_30D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
max_page_search_size: 8000,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -143,9 +143,10 @@ export const SUMMARY_OCCURRENCES_7D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -143,9 +143,10 @@ export const SUMMARY_OCCURRENCES_90D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -141,9 +141,10 @@ export const SUMMARY_OCCURRENCES_MONTHLY_ALIGNED: TransformPutTransformRequest =
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -141,9 +141,10 @@ export const SUMMARY_OCCURRENCES_WEEKLY_ALIGNED: TransformPutTransformRequest =
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -143,9 +143,10 @@ export const SUMMARY_TIMESLICES_30D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -143,9 +143,10 @@ export const SUMMARY_TIMESLICES_7D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -143,9 +143,10 @@ export const SUMMARY_TIMESLICES_90D_ROLLING: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -171,9 +171,10 @@ export const SUMMARY_TIMESLICES_MONTHLY_ALIGNED: TransformPutTransformRequest =
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -8,9 +8,9 @@
import { TransformPutTransformRequest } from '@elastic/elasticsearch/lib/api/types';
import {
SLO_DESTINATION_INDEX_PATTERN,
SLO_RESOURCES_VERSION,
SLO_SUMMARY_DESTINATION_INDEX_NAME,
SLO_SUMMARY_INGEST_PIPELINE_NAME,
SLO_SUMMARY_TRANSFORMS_VERSION,
SLO_SUMMARY_TRANSFORM_NAME_PREFIX,
} from '../../../../assets/constants';
import { groupBy } from './common';
@ -156,9 +156,10 @@ export const SUMMARY_TIMESLICES_WEEKLY_ALIGNED: TransformPutTransformRequest = {
},
settings: {
deduce_mappings: false,
unattended: true,
},
_meta: {
version: SLO_RESOURCES_VERSION,
version: SLO_SUMMARY_TRANSFORMS_VERSION,
managed: true,
managed_by: 'observability',
},

View file

@ -739,6 +739,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": "metrics-apm*",
@ -1013,6 +1014,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": "metrics-apm*",

View file

@ -708,6 +708,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": "metrics-apm*",
@ -971,6 +972,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": "metrics-apm*",

View file

@ -220,6 +220,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [
@ -476,6 +477,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [

View file

@ -235,6 +235,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [
@ -465,6 +466,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [

View file

@ -244,6 +244,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [
@ -512,6 +513,7 @@ Object {
},
"settings": Object {
"deduce_mappings": false,
"unattended": true,
},
"source": Object {
"index": Array [

View file

@ -47,7 +47,7 @@ describe('TransformManager', () => {
await expect(
service.install(createSLO({ indicator: createAPMTransactionErrorRateIndicator() }))
).rejects.toThrowError('Unsupported SLI type: sli.apm.transactionErrorRate');
).rejects.toThrowError('Unsupported indicator type [sli.apm.transactionErrorRate]');
});
it('throws when transform generator fails', async () => {
@ -80,6 +80,20 @@ describe('TransformManager', () => {
});
});
describe('Preview', () => {
it('previews the transform', async () => {
// @ts-ignore defining only a subset of the possible SLI
const generators: Record<IndicatorTypes, TransformGenerator> = {
'sli.apm.transactionErrorRate': new ApmTransactionErrorRateTransformGenerator(),
};
const transformManager = new DefaultTransformManager(generators, esClientMock, loggerMock);
await transformManager.preview('slo-transform-id');
expect(esClientMock.transform.previewTransform).toHaveBeenCalledTimes(1);
});
});
describe('Start', () => {
it('starts the transform', async () => {
// @ts-ignore defining only a subset of the possible SLI

View file

@ -15,6 +15,7 @@ type TransformId = string;
export interface TransformManager {
install(slo: SLO): Promise<TransformId>;
preview(transformId: TransformId): Promise<void>;
start(transformId: TransformId): Promise<void>;
stop(transformId: TransformId): Promise<void>;
uninstall(transformId: TransformId): Promise<void>;
@ -30,8 +31,8 @@ export class DefaultTransformManager implements TransformManager {
async install(slo: SLO): Promise<TransformId> {
const generator = this.generators[slo.indicator.type];
if (!generator) {
this.logger.error(`No transform generator found for ${slo.indicator.type} SLO type`);
throw new Error(`Unsupported SLI type: ${slo.indicator.type}`);
this.logger.error(`No transform generator found for indicator type [${slo.indicator.type}]`);
throw new Error(`Unsupported indicator type [${slo.indicator.type}]`);
}
const transformParams = generator.getTransformParams(slo);
@ -40,13 +41,25 @@ export class DefaultTransformManager implements TransformManager {
logger: this.logger,
});
} catch (err) {
this.logger.error(`Cannot create transform for ${slo.indicator.type} SLI type: ${err}`);
this.logger.error(`Cannot create SLO transform for indicator type [${slo.indicator.type}]`);
throw err;
}
return transformParams.transform_id;
}
async preview(transformId: string): Promise<void> {
try {
await retryTransientEsErrors(
() => this.esClient.transform.previewTransform({ transform_id: transformId }),
{ logger: this.logger }
);
} catch (err) {
this.logger.error(`Cannot preview SLO transform [${transformId}]`);
throw err;
}
}
async start(transformId: TransformId): Promise<void> {
try {
await retryTransientEsErrors(
@ -55,7 +68,7 @@ export class DefaultTransformManager implements TransformManager {
{ logger: this.logger }
);
} catch (err) {
this.logger.error(`Cannot start transform id ${transformId}: ${err}`);
this.logger.error(`Cannot start SLO transform [${transformId}]`);
throw err;
}
}
@ -71,7 +84,7 @@ export class DefaultTransformManager implements TransformManager {
{ logger: this.logger }
);
} catch (err) {
this.logger.error(`Cannot stop transform id ${transformId}: ${err}`);
this.logger.error(`Cannot stop SLO transform [${transformId}]`);
throw err;
}
}
@ -87,7 +100,7 @@ export class DefaultTransformManager implements TransformManager {
{ logger: this.logger }
);
} catch (err) {
this.logger.error(`Cannot delete transform id ${transformId}: ${err}`);
this.logger.error(`Cannot delete SLO transform [${transformId}]`);
throw err;
}
}

View file

@ -141,8 +141,9 @@ describe('UpdateSLO', () => {
});
function expectInstallationOfNewSLOTransform() {
expect(mockTransformManager.start).toBeCalled();
expect(mockTransformManager.install).toBeCalled();
expect(mockTransformManager.preview).toBeCalled();
expect(mockTransformManager.start).toBeCalled();
}
function expectDeletionOfObsoleteSLOData(originalSlo: SLO) {

View file

@ -37,9 +37,12 @@ export class UpdateSLO {
validateSLO(updatedSlo);
await this.deleteObsoleteSLORevisionData(originalSlo);
const updatedSloTransformId = getSLOTransformId(updatedSlo.id, updatedSlo.revision);
await this.repository.save(updatedSlo);
await this.transformManager.install(updatedSlo);
await this.transformManager.start(getSLOTransformId(updatedSlo.id, updatedSlo.revision));
await this.transformManager.preview(updatedSloTransformId);
await this.transformManager.start(updatedSloTransformId);
await this.esClient.index({
index: SLO_SUMMARY_TEMP_INDEX_NAME,