[Fleet] Create Synthetics migration for 8.8.0 (#154952)

## Summary

Resolves https://github.com/elastic/kibana/issues/155215
Resolves https://github.com/elastic/kibana/issues/142653

Handles two primary migrations for Synthetics integration policies.

1. Rounds deprecated schedules to supported schedules
2. Transforms old deprecated throttling schema `5u/3u/20l` to support
JSON schema `{ download: 5, upload: 3, latency: 20 }`

Schedule migration
---

Before 16m schedule
<img width="200" alt="Screen Shot 2023-04-19 at 6 17 35 PM"
src="https://user-images.githubusercontent.com/11356435/233211993-086ce51e-a056-407f-900b-15638238a243.png">
After
<img width="214" alt="Screen Shot 2023-04-19 at 6 44 08 PM"
src="https://user-images.githubusercontent.com/11356435/233215688-e049e2b7-6958-4376-9362-a192aa8e94db.png">

Before 4m schedule
<img width="190" alt="Screen Shot 2023-04-19 at 6 17 29 PM"
src="https://user-images.githubusercontent.com/11356435/233211995-6b7b2927-6851-4947-ab8c-7fb041da03e1.png">
After
<img width="208" alt="Screen Shot 2023-04-19 at 6 44 30 PM"
src="https://user-images.githubusercontent.com/11356435/233215685-89f093e6-3308-4103-9b98-5acb7a9b5a0a.png">

Before 8m schedule
<img width="202" alt="Screen Shot 2023-04-19 at 6 17 23 PM"
src="https://user-images.githubusercontent.com/11356435/233211997-9a60c54c-4867-4ccd-a8eb-5b62d59060bf.png">
After
<img width="201" alt="Screen Shot 2023-04-19 at 6 44 22 PM"
src="https://user-images.githubusercontent.com/11356435/233215687-f3a900cf-b3ec-44f5-b84b-d292dbad623d.png">

Before 2m schedule
<img width="193" alt="Screen Shot 2023-04-19 at 6 17 16 PM"
src="https://user-images.githubusercontent.com/11356435/233211999-3d42ad71-b72b-4876-911e-5d79564f2351.png">
After 
<img width="194" alt="Screen Shot 2023-04-19 at 6 43 55 PM"
src="https://user-images.githubusercontent.com/11356435/233215690-fd3f13dc-5e32-4904-b804-cbcfcabf0760.png">

Throttling migration
---
Before throttling: false
<img width="163" alt="Screen Shot 2023-04-19 at 6 17 00 PM"
src="https://user-images.githubusercontent.com/11356435/233212002-3a891b25-fc2e-4cce-a730-abf8695d5423.png">
After
<img width="185" alt="Screen Shot 2023-04-19 at 6 49 50 PM"
src="https://user-images.githubusercontent.com/11356435/233216370-eed97645-26e8-44f2-8f72-8d1e19f39c35.png">

Before custom throttling
<img width="274" alt="Screen Shot 2023-04-19 at 6 16 54 PM"
src="https://user-images.githubusercontent.com/11356435/233212004-a9fe82fc-d23a-4d54-a5ad-20971c3df211.png">
After
<img width="169" alt="Screen Shot 2023-04-19 at 6 49 44 PM"
src="https://user-images.githubusercontent.com/11356435/233216389-f2803ca9-3429-4f85-b1ee-0f5a8e02db92.png">

Before default throttling
<img width="212" alt="Screen Shot 2023-04-19 at 6 16 48 PM"
src="https://user-images.githubusercontent.com/11356435/233212007-d9fbcae7-6f6e-4cd0-b909-629cfc72b7dc.png">
After
<img width="181" alt="Screen Shot 2023-04-19 at 6 49 35 PM"
src="https://user-images.githubusercontent.com/11356435/233216412-70336d0e-1e45-4809-bd92-c4524a144b99.png">


### Testing

1. Check out the 8.7.0 branch
2. Create a Synthetics private location at
`app/synthetics/settings/private-locations`.
3. Create a monitor, configured at that private location, with invalid
schedules, ideally 1 or each type (http, icmp, browser, tcp) with an
invalid schedule (for example, 2, 8, 11, 16, 333, etc) at
`app/uptime/add-monitor`. Note: you must use Uptime to create monitors
with arbitrary schedules. The Synthetics app will not let you.
4. Create a browser monitor, configured with your private location, with
throttling turned off.
5. Create a browser monitor, configured for your private location, with
a custom throttling profile.
6. Check out this PR and wait for saved object migration to run
7. Navigate to the agent policy for your monitor. Confirm the schedules
were updated to supported schedules. Confirm the throttling configs now
appear in yaml. Confirm that `throttling: false` remains for the
throttling config that was turned off.
This commit is contained in:
Dominique Clarke 2023-04-21 09:55:38 -04:00 committed by GitHub
parent 3c21b451b3
commit b494716bdb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1538 additions and 27 deletions

View file

@ -54,10 +54,8 @@ import {
migrateInstallationToV860,
migratePackagePolicyToV860,
} from './migrations/to_v8_6_0';
import {
migratePackagePolicyToV870,
migratePackagePolicyToV880,
} from './migrations/security_solution';
import { migratePackagePolicyToV870 } from './migrations/security_solution';
import { migratePackagePolicyToV880 } from './migrations/to_v8_8_0';
/*
* Saved object types and mappings

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { migratePackagePolicyToV880 } from './to_v8_8_0';

View file

@ -0,0 +1,167 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SavedObjectMigrationContext } from '@kbn/core/server';
import { getBrowserPolicy, httpPolicy, icmpPolicy, tcpPolicy } from './fixtures/8.7.0';
import { migratePackagePolicyToV880 as migration } from './to_v8_8_0';
describe('8.8.0 Synthetics Package Policy migration', () => {
describe('schedule migration', () => {
const testSchedules = [
['4', '3'],
['4.5', '5'],
['7', '5'],
['8', '10'],
['9.5', '10'],
['12', '10'],
['13', '15'],
['16', '15'],
['18', '20'],
['21', '20'],
['25', '20'],
['26', '30'],
['31', '30'],
['45', '30'],
['46', '60'],
['61', '60'],
['90', '60'],
['91', '120'],
['121', '120'],
['195', '240'],
['600', '240'],
];
it.each(testSchedules)('handles a variety of schedules', (invalidSchedule, validSchedule) => {
const actual = migration(
{
...httpPolicy,
attributes: {
...httpPolicy.attributes,
inputs: [
{
...httpPolicy.attributes.inputs[0],
streams: [
{
...httpPolicy.attributes.inputs[0].streams[0],
vars: {
...httpPolicy.attributes.inputs[0].streams[0].vars,
schedule: {
value: `"@every ${invalidSchedule}m"`,
type: 'text',
},
},
},
],
},
],
},
},
{} as SavedObjectMigrationContext
);
expect(actual.attributes?.inputs[0]?.streams[0]?.vars?.schedule?.value).toEqual(
`"@every ${validSchedule}m"`
);
expect(actual.attributes?.inputs[0]?.streams[0]?.compiled_stream?.schedule).toEqual(
`@every ${validSchedule}m`
);
});
it('handles browserPolicy with 2 minute', () => {
const actual = migration(getBrowserPolicy(), {} as SavedObjectMigrationContext);
expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.schedule?.value).toEqual(
'"@every 1m"'
);
expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream?.schedule).toEqual(
`@every 1m`
);
});
it('handles httpPolicy with 4 minute schedule', () => {
const actual = migration(httpPolicy, {} as SavedObjectMigrationContext);
expect(actual.attributes?.inputs[0]?.streams[0]?.vars?.schedule?.value).toEqual(
'"@every 3m"'
);
expect(actual.attributes?.inputs[0]?.streams[0]?.compiled_stream?.schedule).toEqual(
`@every 3m`
);
});
it('handles tcp with 8 minute schedule', () => {
const actual = migration(tcpPolicy, {} as SavedObjectMigrationContext);
expect(actual.attributes?.inputs[1]?.streams[0]?.vars?.schedule?.value).toEqual(
'"@every 10m"'
);
expect(actual.attributes?.inputs[1]?.streams[0]?.compiled_stream?.schedule).toEqual(
`@every 10m`
);
});
it('handles icmpPolicy with 16 minute schedule', () => {
const actual = migration(icmpPolicy, {} as SavedObjectMigrationContext);
expect(actual.attributes?.inputs[2]?.streams[0]?.vars?.schedule?.value).toEqual(
'"@every 15m"'
);
expect(actual.attributes?.inputs[2]?.streams[0]?.compiled_stream?.schedule).toEqual(
`@every 15m`
);
});
});
describe('throttling migration', () => {
it('handles throtling config for throttling: false', () => {
const actual = migration(getBrowserPolicy('false'), {} as SavedObjectMigrationContext);
expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
'false'
);
expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream?.throttling).toEqual(false);
});
it('handles throttling config for default throttling', () => {
const actual = migration(getBrowserPolicy(), {} as SavedObjectMigrationContext);
expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
JSON.stringify({ download: 5, upload: 3, latency: 20 })
);
expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream.throttling).toEqual({
download: 5,
upload: 3,
latency: 20,
});
});
it('handles throttling config for custom throttling', () => {
const actual = migration(
getBrowserPolicy('1.6d/0.75u/150l'),
{} as SavedObjectMigrationContext
);
expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
JSON.stringify({ download: 1.6, upload: 0.75, latency: 150 })
);
expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream.throttling).toEqual({
download: 1.6,
upload: 0.75,
latency: 150,
});
});
it('handles edge cases', () => {
const actual = migration(
getBrowserPolicy('not a valid value'),
{} as SavedObjectMigrationContext
);
expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
JSON.stringify({ download: 5, upload: 3, latency: 20 })
);
expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream.throttling).toEqual({
download: 5,
upload: 3,
latency: 20,
});
});
});
});

View file

@ -0,0 +1,107 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SavedObjectMigrationFn, SavedObjectUnsanitizedDoc } from '@kbn/core/server';
import type { PackagePolicy } from '../../../../common';
export const ALLOWED_SCHEDULES_IN_MINUTES = [
'1',
'3',
'5',
'10',
'15',
'20',
'30',
'60',
'120',
'240',
];
export const migratePackagePolicyToV880: SavedObjectMigrationFn<PackagePolicy, PackagePolicy> = (
packagePolicyDoc
) => {
if (packagePolicyDoc.attributes.package?.name !== 'synthetics') {
return packagePolicyDoc;
}
const updatedPackagePolicyDoc: SavedObjectUnsanitizedDoc<PackagePolicy> = packagePolicyDoc;
const enabledInput = updatedPackagePolicyDoc.attributes.inputs.find(
(input) => input.enabled === true
);
const enabledStream = enabledInput?.streams.find((stream) => {
return ['browser', 'http', 'icmp', 'tcp'].includes(stream.data_stream.dataset);
});
if (!enabledStream) {
return updatedPackagePolicyDoc;
}
if (
enabledStream.vars &&
enabledStream.vars.schedule?.value &&
enabledStream.compiled_stream?.schedule
) {
const schedule = enabledStream.vars.schedule.value.match(/\d+\.?\d*/g)?.[0];
const updatedSchedule = getNearestSupportedSchedule(schedule);
const formattedUpdatedSchedule = `@every ${updatedSchedule}m`;
enabledStream.vars.schedule.value = `"${formattedUpdatedSchedule}"`;
enabledStream.compiled_stream.schedule = formattedUpdatedSchedule;
}
if (
enabledStream.data_stream.dataset === 'browser' &&
enabledStream.vars?.['throttling.config'] &&
enabledStream.compiled_stream?.throttling
) {
const throttling = enabledStream.vars['throttling.config'].value;
if (throttling) {
const formattedThrottling = handleThrottling(throttling);
enabledStream.vars['throttling.config'].value = JSON.stringify(formattedThrottling);
enabledStream.compiled_stream.throttling = formattedThrottling;
}
}
return updatedPackagePolicyDoc;
};
const handleThrottling = (
throttling: string
): { download: number; upload: number; latency: number } => {
try {
const [download = 5, upload = 3, latency = 20] = throttling.match(/\d+\.?\d*/g) || [];
return {
download: Number(download),
upload: Number(upload),
latency: Number(latency),
};
} catch {
return {
download: 5,
upload: 3,
latency: 20,
};
}
};
const getNearestSupportedSchedule = (currentSchedule: string): string => {
try {
const closest = ALLOWED_SCHEDULES_IN_MINUTES.reduce(function (prev, curr) {
const supportedSchedule = parseFloat(curr);
const currSchedule = parseFloat(currentSchedule);
const prevSupportedSchedule = parseFloat(prev);
return Math.abs(supportedSchedule - currSchedule) <
Math.abs(prevSupportedSchedule - currSchedule)
? curr
: prev;
});
return closest;
} catch {
return '10';
}
};

View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SavedObjectMigrationFn } from '@kbn/core/server';
import type { PackagePolicy } from '../../../common';
import { migratePackagePolicyToV880 as SecSolMigratePackagePolicyToV880 } from './security_solution';
import { migratePackagePolicyToV880 as SyntheticsMigratePackagePolicyToV880 } from './synthetics';
export const migratePackagePolicyToV880: SavedObjectMigrationFn<PackagePolicy, PackagePolicy> = (
packagePolicyDoc,
migrationContext
) => {
let updatedPackagePolicyDoc = packagePolicyDoc;
// Endpoint specific migrations
if (packagePolicyDoc.attributes.package?.name === 'endpoint') {
updatedPackagePolicyDoc = SecSolMigratePackagePolicyToV880(packagePolicyDoc, migrationContext);
}
// Synthetics specific migrations
if (packagePolicyDoc.attributes.package?.name === 'synthetics') {
updatedPackagePolicyDoc = SyntheticsMigratePackagePolicyToV880(
packagePolicyDoc,
migrationContext
);
}
return updatedPackagePolicyDoc;
};

View file

@ -99,6 +99,7 @@ const PackagePolicyBaseSchema = {
namespace: NamespaceSchema,
policy_id: schema.string(),
enabled: schema.boolean(),
is_managed: schema.maybe(schema.boolean()),
package: schema.maybe(
schema.object({
name: schema.string(),

View file

@ -48,9 +48,11 @@ export const CUSTOM_LABEL = i18n.translate('xpack.synthetics.connectionProfile.c
defaultMessage: 'Custom',
});
export const DEFAULT_THROTTLING_VALUE = { download: '5', upload: '3', latency: '20' };
export const PROFILE_VALUES: ThrottlingConfig[] = [
{
value: { download: '5', upload: '3', latency: '20' },
value: DEFAULT_THROTTLING_VALUE,
id: PROFILE_VALUES_ENUM.DEFAULT,
label: i18n.translate('xpack.synthetics.connectionProfile.default', {
defaultMessage: 'Default',

View file

@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { ConfigKey } from '../../runtime_types';
import { throttlingFormatter } from './formatters';
describe('formatters', () => {
describe('throttling formatter', () => {
it('formats for no throttling', () => {
expect(
throttlingFormatter!(
{
[ConfigKey.THROTTLING_CONFIG]: {
value: {
download: '0',
upload: '0',
latency: '0',
},
label: 'No throttling',
id: 'no-throttling',
},
},
ConfigKey.THROTTLING_CONFIG
)
).toEqual('false');
});
it('formats for default throttling', () => {
expect(
throttlingFormatter!(
{
[ConfigKey.THROTTLING_CONFIG]: {
value: {
download: '5',
upload: '3',
latency: '20',
},
label: 'Default',
id: 'default',
},
},
ConfigKey.THROTTLING_CONFIG
)
).toEqual(JSON.stringify({ download: 5, upload: 3, latency: 20 }));
});
it('formats for custom throttling', () => {
expect(
throttlingFormatter!(
{
[ConfigKey.THROTTLING_CONFIG]: {
value: {
download: '1.25',
upload: '0.75',
latency: '150',
},
label: 'Custom',
id: 'custom',
},
},
ConfigKey.THROTTLING_CONFIG
)
).toEqual(JSON.stringify({ download: 1.25, upload: 0.75, latency: 150 }));
});
});
});

View file

@ -12,6 +12,7 @@ import {
objectToJsonFormatter,
stringToJsonFormatter,
} from '../formatting_utils';
import { DEFAULT_THROTTLING_VALUE } from '../../constants/monitor_defaults';
import { tlsFormatters } from '../tls/formatters';
@ -24,7 +25,11 @@ export const throttlingFormatter: Formatter = (fields) => {
return 'false';
}
return `${throttling.value.download}d/${throttling.value.upload}u/${throttling.value.latency}l`;
return JSON.stringify({
download: Number(throttling?.value?.download || DEFAULT_THROTTLING_VALUE.download),
upload: Number(throttling?.value?.upload || DEFAULT_THROTTLING_VALUE.upload),
latency: Number(throttling?.value?.latency || DEFAULT_THROTTLING_VALUE),
});
};
export const browserFormatters: BrowserFormatMap = {

View file

@ -422,7 +422,7 @@ describe('formatSyntheticsPolicy', () => {
},
'throttling.config': {
type: 'text',
value: '5d/3u/20l',
value: JSON.stringify({ download: 5, upload: 3, latency: 20 }),
},
timeout: {
type: 'text',

View file

@ -130,11 +130,15 @@ export const testDataMonitor = {
'filter_journeys.match': '',
'filter_journeys.tags': [],
ignore_https_errors: false,
'throttling.is_enabled': true,
'throttling.download_speed': '5',
'throttling.upload_speed': '3',
'throttling.latency': '20',
'throttling.config': '5d/3u/20l',
throttling: {
id: 'custom',
label: 'Custom',
value: {
download: '5',
upload: '3',
latency: '20',
},
},
'ssl.certificate_authorities': '',
'ssl.certificate': '',
'ssl.key': '',

View file

@ -48,11 +48,15 @@ export const importMonitors = async ({
'filter_journeys.match': '',
'filter_journeys.tags': [],
ignore_https_errors: false,
'throttling.is_enabled': true,
'throttling.download_speed': '5',
'throttling.upload_speed': '3',
'throttling.latency': '20',
'throttling.config': '5d/3u/20l',
throttling: {
id: 'custom',
label: 'Custom',
value: {
download: '5',
upload: '3',
latency: '20',
},
},
};
const id = '1c215bd0-f580-11ec-89e5-694db461b7a5';

View file

@ -169,11 +169,15 @@ const testMonitors = [
'filter_journeys.match': '',
'filter_journeys.tags': [],
ignore_https_errors: false,
'throttling.is_enabled': true,
'throttling.download_speed': '5',
'throttling.upload_speed': '3',
'throttling.latency': '20',
'throttling.config': '5d/3u/20l',
throttling: {
id: 'custom',
label: 'Custom',
value: {
download: '5',
upload: '3',
latency: '20',
},
},
'ssl.certificate_authorities': '',
'ssl.certificate': '',
'ssl.verification_mode': 'full',

View file

@ -263,7 +263,7 @@ describe('SyntheticsPrivateLocation', () => {
},
'throttling.config': {
type: 'text',
value: '5d/3u/20l',
value: JSON.stringify({ download: 5, upload: 3, latency: 20 }),
},
timeout: {
type: 'text',

View file

@ -185,7 +185,10 @@ export const getTestBrowserSyntheticsPolicy = ({
screenshots: { value: 'on', type: 'text' },
synthetics_args: { value: null, type: 'text' },
ignore_https_errors: { value: false, type: 'bool' },
'throttling.config': { value: '5d/3u/20l', type: 'text' },
'throttling.config': {
value: JSON.stringify({ download: 5, upload: 3, latency: 20 }),
type: 'text',
},
'filter_journeys.tags': { value: null, type: 'yaml' },
'filter_journeys.match': { value: null, type: 'text' },
'source.zip_url.ssl.certificate_authorities': { type: 'yaml' },
@ -218,7 +221,7 @@ export const getTestBrowserSyntheticsPolicy = ({
enabled: true,
schedule: '@every 3m',
timeout: '16s',
throttling: '5d/3u/20l',
throttling: { download: 5, upload: 3, latency: 20 },
tags: ['cookie-test', 'browser'],
'source.inline.script':
'step("Visit /users api route", async () => {\\n const response = await page.goto(\'https://nextjs-test-synthetics.vercel.app/api/users\');\\n expect(response.status()).toEqual(200);\\n});',

View file

@ -213,7 +213,10 @@ export const getTestProjectSyntheticsPolicy = (
screenshots: { value: 'on', type: 'text' },
synthetics_args: { value: null, type: 'text' },
ignore_https_errors: { value: false, type: 'bool' },
'throttling.config': { value: '5d/3u/20l', type: 'text' },
'throttling.config': {
value: JSON.stringify({ download: 5, upload: 3, latency: 20 }),
type: 'text',
},
'filter_journeys.tags': { value: null, type: 'yaml' },
'filter_journeys.match': { value: '"check if title is present"', type: 'text' },
'source.zip_url.ssl.certificate_authorities': { type: 'yaml' },
@ -246,7 +249,7 @@ export const getTestProjectSyntheticsPolicy = (
'run_from.geo.name': locationName,
'run_from.id': locationName,
timeout: null,
throttling: '5d/3u/20l',
throttling: { download: 5, upload: 3, latency: 20 },
'source.project.content':
'UEsDBBQACAAIAON5qVQAAAAAAAAAAAAAAAAfAAAAZXhhbXBsZXMvdG9kb3MvYmFzaWMuam91cm5leS50c22Q0WrDMAxF3/sVF7MHB0LMXlc6RvcN+wDPVWNviW0sdUsp/fe5SSiD7UFCWFfHujIGlpnkybwxFTZfoY/E3hsaLEtwhs9RPNWKDU12zAOxkXRIbN4tB9d9pFOJdO6EN2HMqQguWN9asFBuQVMmJ7jiWNII9fIXrbabdUYr58l9IhwhQQZCYORCTFFUC31Btj21NRc7Mq4Nds+4bDD/pNVgT9F52Jyr2Fa+g75LAPttg8yErk+S9ELpTmVotlVwnfNCuh2lepl3+JflUmSBJ3uggt1v9INW/lHNLKze9dJe1J3QJK8pSvWkm6aTtCet5puq+x63+AFQSwcIAPQ3VfcAAACcAQAAUEsBAi0DFAAIAAgA43mpVAD0N1X3AAAAnAEAAB8AAAAAAAAAAAAgAKSBAAAAAGV4YW1wbGVzL3RvZG9zL2Jhc2ljLmpvdXJuZXkudHNQSwUGAAAAAAEAAQBNAAAARAEAAAAA',
playwright_options: { headless: true, chromiumSandbox: false },