github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Synthetics] Improve project monitors creation performance (#140525)

Browse source
This commit is contained in:
Shahzad 2022-09-19 21:03:54 +02:00 • committed by GitHub
parent f290977064
commit 5cec30a7c4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 1196 additions and 200 deletions
Download patch file Download diff file Expand all files Collapse all files

25
x-pack/plugins/synthetics/server/routes/monitor_cruds/add_monitor.ts
View file

@ -13,12 +13,14 @@ import {
SavedObjectsErrorHelpers,
} from '@kbn/core/server';
import { isValidNamespace } from '@kbn/fleet-plugin/common';
import { getSyntheticsPrivateLocations } from '../../legacy_uptime/lib/saved_objects/private_locations';
import { SyntheticsMonitorClient } from '../../synthetics_service/synthetics_monitor/synthetics_monitor_client';
import {
ConfigKey,
MonitorFields,
SyntheticsMonitor,
EncryptedSyntheticsMonitor,
PrivateLocation,
} from '../../../common/runtime_types';
import { formatKibanaNamespace } from '../../../common/formatters';
import { SyntheticsRestApiRouteFactory } from '../../legacy_uptime/routes/types';
@ -54,6 +56,8 @@ export const addSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => ({
// usually id is auto generated, but this is useful for testing
const { id } = request.query;
const spaceId = server.spaces.spacesService.getSpaceId(request);
const monitor: SyntheticsMonitor = request.body as SyntheticsMonitor;
const monitorType = monitor[ConfigKey.MONITOR_TYPE];
const monitorWithDefaults = {
@ -68,6 +72,10 @@ export const addSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => ({
return response.badRequest({ body: { message, attributes: { details, ...payload } } });
}
const privateLocations: PrivateLocation[] = await getSyntheticsPrivateLocations(
savedObjectsClient
);
try {
const { errors, newMonitor } = await syncNewMonitor({
normalizedMonitor: monitorWithDefaults,
@ -77,6 +85,8 @@ export const addSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => ({
savedObjectsClient,
request,
id,
privateLocations,
spaceId,
});
if (errors && errors.length > 0) {
@ -136,6 +146,8 @@ export const syncNewMonitor = async ({
savedObjectsClient,
request,
normalizedMonitor,
privateLocations,
spaceId,
}: {
id?: string;
monitor: SyntheticsMonitor;
@ -144,6 +156,8 @@ export const syncNewMonitor = async ({
syntheticsMonitorClient: SyntheticsMonitorClient;
savedObjectsClient: SavedObjectsClientContract;
request: KibanaRequest;
privateLocations: PrivateLocation[];
spaceId: string;
}) => {
const newMonitorId = id ?? uuidV4();
const { preserve_namespace: preserveNamespace } = request.query as Record<
@ -166,14 +180,15 @@ export const syncNewMonitor = async ({
savedObjectsClient,
});
const syncErrorsPromise = syntheticsMonitorClient.addMonitor(
monitorWithNamespace as MonitorFields,
newMonitorId,
const syncErrorsPromise = syntheticsMonitorClient.addMonitors(
[{ monitor: monitorWithNamespace as MonitorFields, id: newMonitorId }],
request,
savedObjectsClient
savedObjectsClient,
privateLocations,
spaceId
);
const [monitorSavedObjectN, syncErrors] = await Promise.all([
const [monitorSavedObjectN, { syncErrors }] = await Promise.all([
newMonitorPromise,
syncErrorsPromise,
]);

145
x-pack/plugins/synthetics/server/routes/monitor_cruds/bulk_cruds/add_monitor_bulk.ts Normal file
View file

@ -0,0 +1,145 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { SavedObjectsClientContract, KibanaRequest, SavedObject } from '@kbn/core/server';
import pMap from 'p-map';
import { SavedObjectsBulkResponse } from '@kbn/core-saved-objects-api-server';
import { v4 as uuidV4 } from 'uuid';
import { formatTelemetryEvent, sendTelemetryEvents } from '../../telemetry/monitor_upgrade_sender';
import { deleteMonitor } from '../delete_monitor';
import { UptimeServerSetup } from '../../../legacy_uptime/lib/adapters';
import { formatSecrets } from '../../../synthetics_service/utils';
import { syntheticsMonitorType } from '../../../../common/types/saved_objects';
import {
ConfigKey,
EncryptedSyntheticsMonitor,
MonitorFields,
PrivateLocation,
ServiceLocationErrors,
SyntheticsMonitor,
} from '../../../../common/runtime_types';
import { SyntheticsMonitorClient } from '../../../synthetics_service/synthetics_monitor/synthetics_monitor_client';
export const createNewSavedObjectMonitorBulk = async ({
soClient,
monitorsToCreate,
}: {
soClient: SavedObjectsClientContract;
monitorsToCreate: Array<{ id: string; monitor: MonitorFields }>;
}) => {
const newMonitors = monitorsToCreate.map(({ id, monitor }) => ({
id,
type: syntheticsMonitorType,
attributes: formatSecrets({
...monitor,
revision: 1,
}),
}));
return await soClient.bulkCreate<EncryptedSyntheticsMonitor>(newMonitors);
};
export const syncNewMonitorBulk = async ({
normalizedMonitors,
server,
syntheticsMonitorClient,
soClient,
request,
privateLocations,
spaceId,
}: {
normalizedMonitors: SyntheticsMonitor[];
server: UptimeServerSetup;
syntheticsMonitorClient: SyntheticsMonitorClient;
soClient: SavedObjectsClientContract;
request: KibanaRequest;
privateLocations: PrivateLocation[];
spaceId: string;
}) => {
let newMonitors: SavedObjectsBulkResponse<EncryptedSyntheticsMonitor> | null = null;
const monitorsToCreate = normalizedMonitors.map((monitor) => ({
id: uuidV4(),
monitor: monitor as MonitorFields,
}));
try {
const [createdMonitors, { syncErrors }] = await Promise.all([
createNewSavedObjectMonitorBulk({
monitorsToCreate,
soClient,
}),
syntheticsMonitorClient.addMonitors(
monitorsToCreate,
request,
soClient,
privateLocations,
spaceId
),
]);
newMonitors = createdMonitors;
sendNewMonitorTelemetry(server, newMonitors.saved_objects, syncErrors);
return { errors: syncErrors, newMonitors: newMonitors.saved_objects };
} catch (e) {
await rollBackNewMonitorBulk(
monitorsToCreate,
server,
soClient,
syntheticsMonitorClient,
request
);
throw e;
}
};
const rollBackNewMonitorBulk = async (
monitorsToCreate: Array<{ id: string; monitor: MonitorFields }>,
server: UptimeServerSetup,
soClient: SavedObjectsClientContract,
syntheticsMonitorClient: SyntheticsMonitorClient,
request: KibanaRequest
) => {
try {
await pMap(
monitorsToCreate,
async (monitor) =>
deleteMonitor({
server,
request,
savedObjectsClient: soClient,
monitorId: monitor.id,
syntheticsMonitorClient,
}),
{ concurrency: 100 }
);
} catch (e) {
// ignore errors here
server.logger.error(e);
}
};
const sendNewMonitorTelemetry = (
server: UptimeServerSetup,
monitors: Array<SavedObject<EncryptedSyntheticsMonitor>>,
errors?: ServiceLocationErrors | null
) => {
for (const monitor of monitors) {
sendTelemetryEvents(
server.logger,
server.telemetry,
formatTelemetryEvent({
errors,
monitor,
isInlineScript: Boolean((monitor.attributes as MonitorFields)[ConfigKey.SOURCE_INLINE]),
kibanaVersion: server.kibanaVersion,
})
);
}
};

5
x-pack/plugins/synthetics/server/routes/monitor_cruds/delete_monitor.ts
View file

@ -88,6 +88,8 @@ export const deleteMonitor = async ({
request: KibanaRequest;
}) => {
const { logger, telemetry, kibanaVersion, encryptedSavedObjects } = server;
const spaceId = server.spaces.spacesService.getSpaceId(request);
const encryptedSavedObjectsClient = encryptedSavedObjects.getClient();
let normalizedMonitor;
try {
@ -115,7 +117,8 @@ export const deleteMonitor = async ({
monitorId,
},
request,
savedObjectsClient
savedObjectsClient,
spaceId
);
const deletePromise = savedObjectsClient.delete(syntheticsMonitorType, monitorId);

5
x-pack/plugins/synthetics/server/routes/monitor_cruds/edit_monitor.test.ts
View file

@ -47,6 +47,10 @@ describe('syncEditedMonitor', () => {
.fn()
.mockReturnValue({ integrations: { writeIntegrationPolicies: true } }),
},
packagePolicyService: {
get: jest.fn().mockReturnValue({}),
buildPackagePolicyFromPackage: jest.fn().mockReturnValue({}),
},
},
} as unknown as UptimeServerSetup;
@ -96,6 +100,7 @@ describe('syncEditedMonitor', () => {
request: {} as unknown as KibanaRequest,
savedObjectsClient:
serverMock.authSavedObjectsClient as unknown as SavedObjectsClientContract,
spaceId: 'test-space',
});
expect(syntheticsService.editConfig).toHaveBeenCalledWith(

8
x-pack/plugins/synthetics/server/routes/monitor_cruds/edit_monitor.ts
View file

@ -58,6 +58,8 @@ export const editSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => (
const monitor = request.body as SyntheticsMonitor;
const { monitorId } = request.params;
const spaceId = server.spaces.spacesService.getSpaceId(request);
try {
const previousMonitor: SavedObject<EncryptedSyntheticsMonitor> = await savedObjectsClient.get(
syntheticsMonitorType,
@ -101,6 +103,7 @@ export const editSyntheticsMonitorRoute: SyntheticsRestApiRouteFactory = () => (
request,
normalizedMonitor: editedMonitor,
monitorWithRevision: formattedMonitor,
spaceId,
});
// Return service sync errors in OK response
@ -131,6 +134,7 @@ export const syncEditedMonitor = async ({
syntheticsMonitorClient,
savedObjectsClient,
request,
spaceId,
}: {
normalizedMonitor: SyntheticsMonitor;
monitorWithRevision: SyntheticsMonitorWithSecrets;
@ -140,6 +144,7 @@ export const syncEditedMonitor = async ({
syntheticsMonitorClient: SyntheticsMonitorClient;
savedObjectsClient: SavedObjectsClientContract;
request: KibanaRequest;
spaceId: string;
}) => {
try {
const editedSOPromise = savedObjectsClient.update<MonitorFields>(
@ -152,7 +157,8 @@ export const syncEditedMonitor = async ({
normalizedMonitor as MonitorFields,
previousMonitor.id,
request,
savedObjectsClient
savedObjectsClient,
spaceId
);
const [editedMonitorSavedObject, errors] = await Promise.all([

16
x-pack/plugins/synthetics/server/synthetics_service/get_all_locations.ts
View file

@ -18,7 +18,7 @@ export async function getAllLocations(
try {
const [privateLocations, { locations: publicLocations, throttling }] = await Promise.all([
getPrivateLocations(syntheticsMonitorClient, savedObjectsClient),
getServiceLocations(server),
getServicePublicLocations(server, syntheticsMonitorClient),
]);
return { publicLocations, privateLocations, throttling };
} catch (e) {
@ -26,3 +26,17 @@ export async function getAllLocations(
return { publicLocations: [], privateLocations: [] };
}
}
const getServicePublicLocations = async (
server: UptimeServerSetup,
syntheticsMonitorClient: SyntheticsMonitorClient
) => {
if (syntheticsMonitorClient.syntheticsService.locations.length === 0) {
return await getServiceLocations(server);
}
return {
locations: syntheticsMonitorClient.syntheticsService.locations,
throttling: syntheticsMonitorClient.syntheticsService.throttling,
};
};

12
x-pack/plugins/synthetics/server/synthetics_service/normalizers/browser.test.ts
View file

@ -11,6 +11,7 @@ import {
Locations,
LocationStatus,
ProjectBrowserMonitor,
PrivateLocation,
} from '../../../common/runtime_types';
import { DEFAULT_FIELDS } from '../../../common/constants/monitor_defaults';
import { normalizeProjectMonitors } from './browser';
@ -42,11 +43,12 @@ describe('browser normalizers', () => {
status: LocationStatus.GA,
},
];
const privateLocations: Locations = [
const privateLocations: PrivateLocation[] = [
{
id: 'germany',
label: 'Germany',
isServiceManaged: false,
concurrentMonitors: 1,
agentPolicyId: 'germany',
},
];
const monitors: ProjectBrowserMonitor[] = [
@ -234,11 +236,7 @@ describe('browser normalizers', () => {
url: 'test-url',
status: 'ga',
},
{
id: 'germany',
isServiceManaged: false,
label: 'Germany',
},
privateLocations[0],
],
name: 'test-name-3',
params: JSON.stringify(params),

7
x-pack/plugins/synthetics/server/synthetics_service/normalizers/browser.ts
View file

@ -5,6 +5,7 @@
* 2.0.
*/
import { PrivateLocation } from '../../../common/runtime_types';
import { DEFAULT_FIELDS } from '../../../common/constants/monitor_defaults';
import { formatKibanaNamespace } from '../../../common/formatters';
import {
@ -51,7 +52,7 @@ export const normalizeProjectMonitor = ({
namespace,
}: {
locations: Locations;
privateLocations: Locations;
privateLocations: PrivateLocation[];
monitor: ProjectBrowserMonitor;
projectId: string;
namespace: string;
@ -121,7 +122,7 @@ export const normalizeProjectMonitors = ({
namespace,
}: {
locations: Locations;
privateLocations: Locations;
privateLocations: PrivateLocation[];
monitors: ProjectBrowserMonitor[];
projectId: string;
namespace: string;
@ -137,7 +138,7 @@ export const getMonitorLocations = ({
monitor,
}: {
monitor: ProjectBrowserMonitor;
privateLocations: Locations;
privateLocations: PrivateLocation[];
publicLocations: Locations;
}) => {
const publicLocs =

37
x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.test.ts
View file

@ -14,15 +14,17 @@ import {
ScheduleUnit,
SourceType,
HeartbeatConfig,
PrivateLocation,
} from '../../../common/runtime_types';
import { SyntheticsPrivateLocation } from './synthetics_private_location';
import { testMonitorPolicy } from './test_policy';
describe('SyntheticsPrivateLocation', () => {
const mockPrivateLocation = {
const mockPrivateLocation: PrivateLocation = {
id: 'policyId',
label: 'Test Location',
isServiceManaged: false,
concurrentMonitors: 1,
agentPolicyId: 'policyId',
};
const testConfig = {
id: 'testId',
@ -77,6 +79,7 @@ describe('SyntheticsPrivateLocation', () => {
},
packagePolicyService: {
get: jest.fn().mockReturnValue({}),
buildPackagePolicyFromPackage: jest.fn(),
},
},
spaces: {
@ -87,13 +90,10 @@ describe('SyntheticsPrivateLocation', () => {
} as unknown as UptimeServerSetup;
it.each([
[
true,
'Unable to create Synthetics package policy for monitor Test Monitor with private location Test Location',
],
[true, 'Unable to create Synthetics package policy for private location'],
[
false,
'Unable to create Synthetics package policy for monitor Test Monitor. Fleet write permissions are needed to use Synthetics private locations.',
'Unable to create Synthetics package policy for monitor. Fleet write permissions are needed to use Synthetics private locations.',
],
])('throws errors for create monitor', async (writeIntegrationPolicies, error) => {
const syntheticsPrivateLocation = new SyntheticsPrivateLocation({
@ -107,10 +107,12 @@ describe('SyntheticsPrivateLocation', () => {
});
try {
await syntheticsPrivateLocation.createMonitor(
testConfig,
await syntheticsPrivateLocation.createMonitors(
[testConfig],
{} as unknown as KibanaRequest,
savedObjectsClientMock
savedObjectsClientMock,
[mockPrivateLocation],
'test-space'
);
} catch (e) {
expect(e).toEqual(new Error(error));
@ -118,10 +120,7 @@ describe('SyntheticsPrivateLocation', () => {
});
it.each([
[
true,
'Unable to update Synthetics package policy for monitor Test Monitor with private location Test Location',
],
[true, 'Unable to create Synthetics package policy for private location'],
[
false,
'Unable to update Synthetics package policy for monitor Test Monitor. Fleet write permissions are needed to use Synthetics private locations.',
@ -141,7 +140,8 @@ describe('SyntheticsPrivateLocation', () => {
await syntheticsPrivateLocation.editMonitor(
testConfig,
{} as unknown as KibanaRequest,
savedObjectsClientMock
savedObjectsClientMock,
'test-space'
);
} catch (e) {
expect(e).toEqual(new Error(error));
@ -155,7 +155,7 @@ describe('SyntheticsPrivateLocation', () => {
],
[
false,
'Unable to delete Synthetics package policy for monitor Test Monitor. Fleet write permissions are needed to use Synthetics private locations.',
'Unable to delete Synthetics package policy for monitor. Fleet write permissions are needed to use Synthetics private locations.',
],
])('throws errors for delete monitor', async (writeIntegrationPolicies, error) => {
const syntheticsPrivateLocation = new SyntheticsPrivateLocation({
@ -171,10 +171,11 @@ describe('SyntheticsPrivateLocation', () => {
await syntheticsPrivateLocation.deleteMonitor(
testConfig,
{} as unknown as KibanaRequest,
savedObjectsClientMock
savedObjectsClientMock,
'test-space'
);
} catch (e) {
expect(e).toEqual(new Error(e));
expect(e).toEqual(new Error(error));
}
});

167
x-pack/plugins/synthetics/server/synthetics_service/private_location/synthetics_private_location.ts
View file

@ -6,13 +6,14 @@
*/
import { KibanaRequest, SavedObjectsClientContract } from '@kbn/core/server';
import { NewPackagePolicy } from '@kbn/fleet-plugin/common';
import { NewPackagePolicyWithId } from '@kbn/fleet-plugin/server/services/package_policy';
import { formatSyntheticsPolicy } from '../../../common/formatters/format_synthetics_policy';
import { getSyntheticsPrivateLocations } from '../../legacy_uptime/lib/saved_objects/private_locations';
import {
ConfigKey,
HeartbeatConfig,
MonitorFields,
PrivateLocation,
HeartbeatConfig,
SourceType,
} from '../../../common/runtime_types';
import { UptimeServerSetup } from '../../legacy_uptime/lib/adapters';
@ -24,43 +25,39 @@ export class SyntheticsPrivateLocation {
this.server = _server;
}
getSpaceId(request: KibanaRequest) {
return this.server.spaces.spacesService.getSpaceId(request);
async buildNewPolicy(
savedObjectsClient: SavedObjectsClientContract
): Promise<NewPackagePolicy | undefined> {
return await this.server.fleet.packagePolicyService.buildPackagePolicyFromPackage(
savedObjectsClient,
'synthetics',
this.server.logger
);
}
getPolicyId(config: HeartbeatConfig, { id: locId }: PrivateLocation, request: KibanaRequest) {
getPolicyId(config: HeartbeatConfig, { id: locId }: PrivateLocation, spaceId: string) {
if (config[ConfigKey.MONITOR_SOURCE_TYPE] === SourceType.PROJECT) {
return `${config.id}-${locId}`;
}
return `${config.id}-${locId}-${this.getSpaceId(request)}`;
return `${config.id}-${locId}-${spaceId}`;
}
async generateNewPolicy(
config: HeartbeatConfig,
privateLocation: PrivateLocation,
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
newPolicyTemplate: NewPackagePolicy,
spaceId: string
): Promise<NewPackagePolicy | null> {
if (!savedObjectsClient) {
throw new Error('Could not find savedObjectsClient');
}
const { label: locName } = privateLocation;
const spaceId = this.getSpaceId(request);
const newPolicy = { ...newPolicyTemplate };
try {
const newPolicy = await this.server.fleet.packagePolicyService.buildPackagePolicyFromPackage(
savedObjectsClient,
'synthetics',
this.server.logger
);
if (!newPolicy) {
throw new Error(
`Unable to create Synthetics package policy for private location ${privateLocation.label}`
);
}
newPolicy.is_managed = true;
newPolicy.policy_id = privateLocation.agentPolicyId;
if (config[ConfigKey.MONITOR_SOURCE_TYPE] === SourceType.PROJECT) {
@ -95,66 +92,81 @@ export class SyntheticsPrivateLocation {
}
}
async createMonitor(
config: HeartbeatConfig,
async createMonitors(
configs: HeartbeatConfig[],
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
privateLocations: PrivateLocation[],
spaceId: string
) {
const { locations } = config;
await this.checkPermissions(
request,
`Unable to create Synthetics package policy for monitor ${
config[ConfigKey.NAME]
}. Fleet write permissions are needed to use Synthetics private locations.`
`Unable to create Synthetics package policy for monitor. Fleet write permissions are needed to use Synthetics private locations.`
);
const privateLocations: PrivateLocation[] = await getSyntheticsPrivateLocations(
savedObjectsClient
);
const newPolicies: NewPackagePolicyWithId[] = [];
const fleetManagedLocations = locations.filter((loc) => !loc.isServiceManaged);
const newPolicyTemplate = await this.buildNewPolicy(savedObjectsClient);
for (const privateLocation of fleetManagedLocations) {
const location = privateLocations?.find((loc) => loc.id === privateLocation.id);
if (!location) {
throw new Error(
`Unable to find Synthetics private location for agentId ${privateLocation.id}`
);
}
const newPolicy = await this.generateNewPolicy(config, location, request, savedObjectsClient);
if (!newPolicy) {
throw new Error(
`Unable to create Synthetics package policy for monitor ${
config[ConfigKey.NAME]
} with private location ${location.label}`
);
}
if (!newPolicyTemplate) {
throw new Error(`Unable to create Synthetics package policy for private location`);
}
for (const config of configs) {
try {
await this.createPolicy(
newPolicy,
this.getPolicyId(config, location, request),
savedObjectsClient
);
const { locations } = config;
const fleetManagedLocations = locations.filter((loc) => !loc.isServiceManaged);
for (const privateLocation of fleetManagedLocations) {
const location = privateLocations?.find((loc) => loc.id === privateLocation.id)!;
if (!location) {
throw new Error(
`Unable to find Synthetics private location for agentId ${privateLocation.id}`
);
}
const newPolicy = await this.generateNewPolicy(
config,
location,
savedObjectsClient,
newPolicyTemplate,
spaceId
);
if (!newPolicy) {
throw new Error(
`Unable to create Synthetics package policy for monitor ${
config[ConfigKey.NAME]
} with private location ${location.label}`
);
}
if (newPolicy) {
newPolicies.push({ ...newPolicy, id: this.getPolicyId(config, location, spaceId) });
}
}
} catch (e) {
this.server.logger.error(e);
throw new Error(
`Unable to create Synthetics package policy for monitor ${
config[ConfigKey.NAME]
} with private location ${location.label}`
);
}
}
if (newPolicies.length === 0) {
throw new Error('Failed to build package policies for all monitors');
}
try {
return await this.createPolicyBulk(newPolicies, savedObjectsClient);
} catch (e) {
this.server.logger.error(e);
}
}
async editMonitor(
config: HeartbeatConfig,
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
spaceId: string
) {
await this.checkPermissions(
request,
@ -167,19 +179,26 @@ export class SyntheticsPrivateLocation {
const allPrivateLocations = await getSyntheticsPrivateLocations(savedObjectsClient);
const newPolicyTemplate = await this.buildNewPolicy(savedObjectsClient);
if (!newPolicyTemplate) {
throw new Error(`Unable to create Synthetics package policy for private location`);
}
const monitorPrivateLocations = locations.filter((loc) => !loc.isServiceManaged);
for (const privateLocation of allPrivateLocations) {
const hasLocation = monitorPrivateLocations?.some((loc) => loc.id === privateLocation.id);
const currId = this.getPolicyId(config, privateLocation, request);
const currId = this.getPolicyId(config, privateLocation, spaceId);
const hasPolicy = await this.getMonitor(currId, savedObjectsClient);
try {
if (hasLocation) {
const newPolicy = await this.generateNewPolicy(
config,
privateLocation,
request,
savedObjectsClient
savedObjectsClient,
newPolicyTemplate,
spaceId
);
if (!newPolicy) {
@ -222,6 +241,21 @@ export class SyntheticsPrivateLocation {
}
}
async createPolicyBulk(
newPolicies: NewPackagePolicyWithId[],
savedObjectsClient: SavedObjectsClientContract
) {
const soClient = savedObjectsClient;
const esClient = this.server.uptimeEsClient.baseESClient;
if (soClient && esClient) {
return await this.server.fleet.packagePolicyService.bulkCreate(
soClient,
esClient,
newPolicies
);
}
}
async createPolicy(
newPolicy: NewPackagePolicy,
id: string,
@ -270,7 +304,8 @@ export class SyntheticsPrivateLocation {
async deleteMonitor(
config: HeartbeatConfig,
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
spaceId: string
) {
const soClient = savedObjectsClient;
const esClient = this.server.uptimeEsClient.baseESClient;
@ -296,7 +331,7 @@ export class SyntheticsPrivateLocation {
await this.server.fleet.packagePolicyService.delete(
soClient,
esClient,
[this.getPolicyId(config, location, request)],
[this.getPolicyId(config, location, spaceId)],
{
force: true,
}

628
x-pack/plugins/synthetics/server/synthetics_service/project_monitor_formatter.test.ts Normal file
View file

@ -0,0 +1,628 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { loggerMock } from '@kbn/logging-mocks';
import { savedObjectsClientMock } from '@kbn/core/server/mocks';
import {
INSUFFICIENT_FLEET_PERMISSIONS,
ProjectMonitorFormatter,
} from './project_monitor_formatter';
import { LocationStatus } from '../../common/runtime_types';
import { times } from 'lodash';
import { SyntheticsService } from './synthetics_service';
import { UptimeServerSetup } from '../legacy_uptime/lib/adapters';
import { encryptedSavedObjectsMock } from '@kbn/encrypted-saved-objects-plugin/server/mocks';
import { SyntheticsMonitorClient } from './synthetics_monitor/synthetics_monitor_client';
import { httpServerMock } from '@kbn/core-http-server-mocks';
import { Subject } from 'rxjs';
import { formatSecrets } from './utils';
import * as telemetryHooks from '../routes/telemetry/monitor_upgrade_sender';
const testMonitors = [
{
throttling: { download: 5, upload: 3, latency: 20 },
schedule: 3,
locations: [],
privateLocations: ['Test private location'],
params: { url: 'http://localhost:8080' },
playwrightOptions: {
userAgent:
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
viewport: { width: 375, height: 667 },
deviceScaleFactor: 2,
isMobile: true,
hasTouch: true,
headless: true,
},
name: 'check if title is present 10 0',
id: 'check if title is present 10 0',
tags: [],
content:
'UEsDBBQACAAIAAAAIQAAAAAAAAAAAAAAAAAQAAAAYmFzaWMuam91cm5leS50c2WQQU7DQAxF9znFV8QiUUOmXcCCUMQl2NdMnWbKJDMaO6Ilyt0JASQkNv9Z1teTZWNAIqwP5kU4iZGOug863u7uDXsSddbIddCOl0kMX6iPnsVoOAYxryTO1ucwpoGvtUrm+hiSYsLProIoxwp8iWwVM9oUeuTP/9V5k7UhofCscNhj2yx4xN2CzabElOHXWRxsx/YNroU69QwniImFB8Vui5vJzYcKxYRIJ66WTNQL5hL7p1WD9aYi9zQOtgPFGPNqecJ1sCj+tAB6J6erpj4FDcW3qh6TL5u1Mq/8yjn7BFBLBwhGDIWc4QAAAEkBAABQSwECLQMUAAgACAAAACEARgyFnOEAAABJAQAAEAAAAAAAAAAAACAApIEAAAAAYmFzaWMuam91cm5leS50c1BLBQYAAAAAAQABAD4AAAAfAQAAAAA=',
filter: { match: 'check if title is present 10 0' },
},
{
throttling: { download: 5, upload: 3, latency: 20 },
schedule: 3,
locations: [],
privateLocations: ['Test private location'],
params: { url: 'http://localhost:8080' },
playwrightOptions: {
userAgent:
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
viewport: { width: 375, height: 667 },
deviceScaleFactor: 2,
isMobile: true,
hasTouch: true,
headless: true,
},
name: 'check if title is present 10 1',
id: 'check if title is present 10 1',
tags: [],
content:
'UEsDBBQACAAIAAAAIQAAAAAAAAAAAAAAAAAQAAAAYmFzaWMuam91cm5leS50c2WQQU7DQAxF9znFV8QiUUOmXcCCUMQl2NdMnWbKJDMaO6Ilyt0JASQkNv9Z1teTZWNAIqwP5kU4iZGOug863u7uDXsSddbIddCOl0kMX6iPnsVoOAYxryTO1ucwpoGvtUrm+hiSYsLProIoxwp8iWwVM9oUeuTP/9V5k7UhofCscNhj2yx4xN2CzabElOHXWRxsx/YNroU69QwniImFB8Vui5vJzYcKxYRIJ66WTNQL5hL7p1WD9aYi9zQOtgPFGPNqecJ1sCj+tAB6J6erpj4FDcW3qh6TL5u1Mq/8yjn7BFBLBwhGDIWc4QAAAEkBAABQSwECLQMUAAgACAAAACEARgyFnOEAAABJAQAAEAAAAAAAAAAAACAApIEAAAAAYmFzaWMuam91cm5leS50c1BLBQYAAAAAAQABAD4AAAAfAQAAAAA=',
filter: { match: 'check if title is present 10 1' },
},
];
const privateLocations = times(1).map((n) => {
return {
id: `loc-${n}`,
label: 'Test private location',
geo: {
lat: 0,
lon: 0,
},
isServiceManaged: false,
agentPolicyId: `loc-${n}`,
concurrentMonitors: 1,
};
});
describe('ProjectMonitorFormatter', () => {
const mockEsClient = {
search: jest.fn(),
};
const logger = loggerMock.create();
const kibanaRequest = httpServerMock.createKibanaRequest();
const soClient = savedObjectsClientMock.create();
const serverMock: UptimeServerSetup = {
logger,
uptimeEsClient: mockEsClient,
authSavedObjectsClient: soClient,
config: {
service: {
username: 'dev',
password: '12345',
manifestUrl: 'http://localhost:8080/api/manifest',
},
},
spaces: {
spacesService: {
getSpaceId: jest.fn().mockReturnValue('test-space'),
},
},
} as unknown as UptimeServerSetup;
const syntheticsService = new SyntheticsService(serverMock);
syntheticsService.addConfig = jest.fn();
syntheticsService.editConfig = jest.fn();
syntheticsService.deleteConfigs = jest.fn();
const encryptedSavedObjectsClient = encryptedSavedObjectsMock.createStart().getClient();
const locations = times(3).map((n) => {
return {
id: `loc-${n}`,
label: `Location ${n}`,
url: `https://example.com/${n}`,
geo: {
lat: 0,
lon: 0,
},
isServiceManaged: true,
status: LocationStatus.GA,
};
});
const monitorClient = new SyntheticsMonitorClient(syntheticsService, serverMock);
it('should return errors', async () => {
const testSubject = new Subject();
testSubject.next = jest.fn();
const pushMonitorFormatter = new ProjectMonitorFormatter({
projectId: 'test-project',
spaceId: 'default-space',
keepStale: false,
locations,
privateLocations,
encryptedSavedObjectsClient,
savedObjectsClient: soClient,
monitors: testMonitors,
server: serverMock,
syntheticsMonitorClient: monitorClient,
request: kibanaRequest,
subject: testSubject,
});
pushMonitorFormatter.getProjectMonitorsForProject = jest.fn().mockResolvedValue([]);
await pushMonitorFormatter.configureAllProjectMonitors();
expect(testSubject.next).toHaveBeenNthCalledWith(
1,
'check if title is present 10 0: failed to create or update monitor'
);
expect(testSubject.next).toHaveBeenNthCalledWith(
2,
'check if title is present 10 1: failed to create or update monitor'
);
expect({
createdMonitors: pushMonitorFormatter.createdMonitors,
updatedMonitors: pushMonitorFormatter.updatedMonitors,
staleMonitors: pushMonitorFormatter.staleMonitors,
deletedMonitors: pushMonitorFormatter.deletedMonitors,
failedMonitors: pushMonitorFormatter.failedMonitors,
failedStaleMonitors: pushMonitorFormatter.failedStaleMonitors,
}).toStrictEqual({
createdMonitors: [],
deletedMonitors: [],
failedMonitors: [
{
details: "Cannot read properties of undefined (reading 'authz')",
id: 'check if title is present 10 0',
payload: testMonitors[0],
reason: 'Failed to create or update monitor',
},
{
details: "Cannot read properties of undefined (reading 'authz')",
id: 'check if title is present 10 1',
payload: testMonitors[1],
reason: 'Failed to create or update monitor',
},
],
failedStaleMonitors: [],
staleMonitors: [],
updatedMonitors: [],
});
});
it('throws fleet permission error', async () => {
const testSubject = new Subject();
serverMock.fleet = {
authz: {
fromRequest: jest
.fn()
.mockResolvedValue({ integrations: { writeIntegrationPolicies: false } }),
},
} as any;
const pushMonitorFormatter = new ProjectMonitorFormatter({
projectId: 'test-project',
spaceId: 'default-space',
keepStale: false,
locations,
privateLocations,
encryptedSavedObjectsClient,
savedObjectsClient: soClient,
monitors: testMonitors,
server: serverMock,
syntheticsMonitorClient: monitorClient,
request: kibanaRequest,
subject: testSubject,
});
pushMonitorFormatter.getProjectMonitorsForProject = jest.fn().mockResolvedValue([]);
await pushMonitorFormatter.configureAllProjectMonitors();
expect({
createdMonitors: pushMonitorFormatter.createdMonitors,
updatedMonitors: pushMonitorFormatter.updatedMonitors,
staleMonitors: pushMonitorFormatter.staleMonitors,
deletedMonitors: pushMonitorFormatter.deletedMonitors,
failedMonitors: pushMonitorFormatter.failedMonitors,
failedStaleMonitors: pushMonitorFormatter.failedStaleMonitors,
}).toStrictEqual({
createdMonitors: [],
deletedMonitors: [],
failedMonitors: [
{
details: INSUFFICIENT_FLEET_PERMISSIONS,
id: 'check if title is present 10 0',
payload: testMonitors[0],
reason: 'Failed to create or update monitor',
},
{
details: INSUFFICIENT_FLEET_PERMISSIONS,
id: 'check if title is present 10 1',
payload: testMonitors[1],
reason: 'Failed to create or update monitor',
},
],
failedStaleMonitors: [],
staleMonitors: [],
updatedMonitors: [],
});
});
it('catches errors from bulk edit method', async () => {
const testSubject = new Subject();
serverMock.fleet = {
authz: {
fromRequest: jest
.fn()
.mockResolvedValue({ integrations: { writeIntegrationPolicies: true } }),
},
} as any;
const pushMonitorFormatter = new ProjectMonitorFormatter({
projectId: 'test-project',
spaceId: 'default-space',
keepStale: false,
locations,
privateLocations,
encryptedSavedObjectsClient,
savedObjectsClient: soClient,
monitors: testMonitors,
server: serverMock,
syntheticsMonitorClient: monitorClient,
request: kibanaRequest,
subject: testSubject,
});
pushMonitorFormatter.getProjectMonitorsForProject = jest.fn().mockResolvedValue([]);
await pushMonitorFormatter.configureAllProjectMonitors();
expect({
createdMonitors: pushMonitorFormatter.createdMonitors,
updatedMonitors: pushMonitorFormatter.updatedMonitors,
staleMonitors: pushMonitorFormatter.staleMonitors,
deletedMonitors: pushMonitorFormatter.deletedMonitors,
failedMonitors: pushMonitorFormatter.failedMonitors,
failedStaleMonitors: pushMonitorFormatter.failedStaleMonitors,
}).toEqual({
createdMonitors: [],
updatedMonitors: [],
staleMonitors: [],
deletedMonitors: [],
failedMonitors: [
{
details: "Cannot read properties of undefined (reading 'buildPackagePolicyFromPackage')",
payload: payloadData,
reason: 'Failed to create 2 monitors',
},
],
failedStaleMonitors: [],
});
});
it('configures project monitors when there are errors', async () => {
const testSubject = new Subject();
serverMock.fleet = {
authz: {
fromRequest: jest
.fn()
.mockResolvedValue({ integrations: { writeIntegrationPolicies: true } }),
},
} as any;
soClient.bulkCreate = jest.fn().mockResolvedValue({ saved_objects: [] });
const pushMonitorFormatter = new ProjectMonitorFormatter({
projectId: 'test-project',
spaceId: 'default-space',
keepStale: false,
locations,
privateLocations,
encryptedSavedObjectsClient,
savedObjectsClient: soClient,
monitors: testMonitors,
server: serverMock,
syntheticsMonitorClient: monitorClient,
request: kibanaRequest,
subject: testSubject,
});
pushMonitorFormatter.getProjectMonitorsForProject = jest.fn().mockResolvedValue([]);
await pushMonitorFormatter.configureAllProjectMonitors();
expect({
createdMonitors: pushMonitorFormatter.createdMonitors,
updatedMonitors: pushMonitorFormatter.updatedMonitors,
staleMonitors: pushMonitorFormatter.staleMonitors,
deletedMonitors: pushMonitorFormatter.deletedMonitors,
failedMonitors: pushMonitorFormatter.failedMonitors,
failedStaleMonitors: pushMonitorFormatter.failedStaleMonitors,
}).toEqual({
createdMonitors: [],
updatedMonitors: [],
staleMonitors: [],
deletedMonitors: [],
failedMonitors: [
{
details: "Cannot read properties of undefined (reading 'buildPackagePolicyFromPackage')",
payload: payloadData,
reason: 'Failed to create 2 monitors',
},
],
failedStaleMonitors: [],
});
});
it('shows errors thrown by fleet api', async () => {
const testSubject = new Subject();
serverMock.fleet = {
authz: {
fromRequest: jest
.fn()
.mockResolvedValue({ integrations: { writeIntegrationPolicies: true } }),
},
packagePolicyService: {},
} as any;
soClient.bulkCreate = jest.fn().mockResolvedValue({ saved_objects: soResult });
const pushMonitorFormatter = new ProjectMonitorFormatter({
projectId: 'test-project',
spaceId: 'default-space',
keepStale: false,
locations,
privateLocations,
encryptedSavedObjectsClient,
savedObjectsClient: soClient,
monitors: testMonitors,
server: serverMock,
syntheticsMonitorClient: monitorClient,
request: kibanaRequest,
subject: testSubject,
});
pushMonitorFormatter.getProjectMonitorsForProject = jest.fn().mockResolvedValue([]);
await pushMonitorFormatter.configureAllProjectMonitors();
expect({
createdMonitors: pushMonitorFormatter.createdMonitors,
updatedMonitors: pushMonitorFormatter.updatedMonitors,
staleMonitors: pushMonitorFormatter.staleMonitors,
deletedMonitors: pushMonitorFormatter.deletedMonitors,
failedMonitors: pushMonitorFormatter.failedMonitors,
failedStaleMonitors: pushMonitorFormatter.failedStaleMonitors,
}).toEqual({
createdMonitors: [],
updatedMonitors: [],
staleMonitors: [],
deletedMonitors: [],
failedMonitors: [
{
details:
'this.server.fleet.packagePolicyService.buildPackagePolicyFromPackage is not a function',
reason: 'Failed to create 2 monitors',
payload: payloadData,
},
],
failedStaleMonitors: [],
});
});
it('creates project monitors when no errors', async () => {
const testSubject = new Subject();
serverMock.fleet = {
authz: {
fromRequest: jest
.fn()
.mockResolvedValue({ integrations: { writeIntegrationPolicies: true } }),
},
} as any;
soClient.bulkCreate = jest.fn().mockResolvedValue({ saved_objects: soResult });
monitorClient.addMonitors = jest.fn().mockReturnValue({});
const telemetrySpy = jest
.spyOn(telemetryHooks, 'sendTelemetryEvents')
.mockImplementation(jest.fn());
const pushMonitorFormatter = new ProjectMonitorFormatter({
projectId: 'test-project',
spaceId: 'default-space',
keepStale: false,
locations,
privateLocations,
encryptedSavedObjectsClient,
savedObjectsClient: soClient,
monitors: testMonitors,
server: serverMock,
syntheticsMonitorClient: monitorClient,
request: kibanaRequest,
subject: testSubject,
});
pushMonitorFormatter.getProjectMonitorsForProject = jest.fn().mockResolvedValue([]);
await pushMonitorFormatter.configureAllProjectMonitors();
expect(soClient.bulkCreate).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining(soData[0]),
expect.objectContaining(soData[1]),
])
);
expect(telemetrySpy).toHaveBeenCalledTimes(2);
expect({
createdMonitors: pushMonitorFormatter.createdMonitors,
updatedMonitors: pushMonitorFormatter.updatedMonitors,
staleMonitors: pushMonitorFormatter.staleMonitors,
deletedMonitors: pushMonitorFormatter.deletedMonitors,
failedMonitors: pushMonitorFormatter.failedMonitors,
failedStaleMonitors: pushMonitorFormatter.failedStaleMonitors,
}).toEqual({
createdMonitors: ['check if title is present 10 0', 'check if title is present 10 1'],
updatedMonitors: [],
staleMonitors: [],
deletedMonitors: [],
failedMonitors: [],
failedStaleMonitors: [],
});
});
});
const payloadData = [
{
__ui: {
is_zip_url_tls_enabled: false,
script_source: {
file_name: '',
is_generated_script: false,
},
},
config_id: '',
custom_heartbeat_id: 'check if title is present 10 0-test-project-default-space',
enabled: true,
'filter_journeys.match': 'check if title is present 10 0',
'filter_journeys.tags': [],
form_monitor_type: 'multistep',
ignore_https_errors: false,
journey_id: 'check if title is present 10 0',
locations: privateLocations,
name: 'check if title is present 10 0',
namespace: 'default_space',
origin: 'project',
original_space: 'default-space',
params: '{"url":"http://localhost:8080"}',
playwright_options:
'{"userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1","viewport":{"width":375,"height":667},"deviceScaleFactor":2,"isMobile":true,"hasTouch":true,"headless":true}',
playwright_text_assertion: '',
project_id: 'test-project',
schedule: {
number: '3',
unit: 'm',
},
screenshots: 'on',
'service.name': '',
'source.inline.script': '',
'source.project.content':
'UEsDBBQACAAIAAAAIQAAAAAAAAAAAAAAAAAQAAAAYmFzaWMuam91cm5leS50c2WQQU7DQAxF9znFV8QiUUOmXcCCUMQl2NdMnWbKJDMaO6Ilyt0JASQkNv9Z1teTZWNAIqwP5kU4iZGOug863u7uDXsSddbIddCOl0kMX6iPnsVoOAYxryTO1ucwpoGvtUrm+hiSYsLProIoxwp8iWwVM9oUeuTP/9V5k7UhofCscNhj2yx4xN2CzabElOHXWRxsx/YNroU69QwniImFB8Vui5vJzYcKxYRIJ66WTNQL5hL7p1WD9aYi9zQOtgPFGPNqecJ1sCj+tAB6J6erpj4FDcW3qh6TL5u1Mq/8yjn7BFBLBwhGDIWc4QAAAEkBAABQSwECLQMUAAgACAAAACEARgyFnOEAAABJAQAAEAAAAAAAAAAAACAApIEAAAAAYmFzaWMuam91cm5leS50c1BLBQYAAAAAAQABAD4AAAAfAQAAAAA=',
'source.zip_url.folder': '',
'source.zip_url.password': '',
'source.zip_url.proxy_url': '',
'source.zip_url.url': '',
'source.zip_url.ssl.certificate': undefined,
'source.zip_url.username': '',
'ssl.certificate': '',
'ssl.certificate_authorities': '',
'ssl.key': '',
'ssl.key_passphrase': '',
'ssl.supported_protocols': ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'],
'ssl.verification_mode': 'full',
synthetics_args: [],
tags: [],
'throttling.config': '5d/3u/20l',
'throttling.download_speed': '5',
'throttling.is_enabled': true,
'throttling.latency': '20',
'throttling.upload_speed': '3',
timeout: null,
type: 'browser',
'url.port': null,
urls: '',
},
{
__ui: {
is_zip_url_tls_enabled: false,
script_source: {
file_name: '',
is_generated_script: false,
},
},
config_id: '',
custom_heartbeat_id: 'check if title is present 10 1-test-project-default-space',
enabled: true,
'filter_journeys.match': 'check if title is present 10 1',
'filter_journeys.tags': [],
form_monitor_type: 'multistep',
ignore_https_errors: false,
journey_id: 'check if title is present 10 1',
locations: privateLocations,
name: 'check if title is present 10 1',
namespace: 'default_space',
origin: 'project',
original_space: 'default-space',
params: '{"url":"http://localhost:8080"}',
playwright_options:
'{"userAgent":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1","viewport":{"width":375,"height":667},"deviceScaleFactor":2,"isMobile":true,"hasTouch":true,"headless":true}',
playwright_text_assertion: '',
project_id: 'test-project',
schedule: {
number: '3',
unit: 'm',
},
screenshots: 'on',
'service.name': '',
'source.inline.script': '',
'source.project.content':
'UEsDBBQACAAIAAAAIQAAAAAAAAAAAAAAAAAQAAAAYmFzaWMuam91cm5leS50c2WQQU7DQAxF9znFV8QiUUOmXcCCUMQl2NdMnWbKJDMaO6Ilyt0JASQkNv9Z1teTZWNAIqwP5kU4iZGOug863u7uDXsSddbIddCOl0kMX6iPnsVoOAYxryTO1ucwpoGvtUrm+hiSYsLProIoxwp8iWwVM9oUeuTP/9V5k7UhofCscNhj2yx4xN2CzabElOHXWRxsx/YNroU69QwniImFB8Vui5vJzYcKxYRIJ66WTNQL5hL7p1WD9aYi9zQOtgPFGPNqecJ1sCj+tAB6J6erpj4FDcW3qh6TL5u1Mq/8yjn7BFBLBwhGDIWc4QAAAEkBAABQSwECLQMUAAgACAAAACEARgyFnOEAAABJAQAAEAAAAAAAAAAAACAApIEAAAAAYmFzaWMuam91cm5leS50c1BLBQYAAAAAAQABAD4AAAAfAQAAAAA=',
'source.zip_url.folder': '',
'source.zip_url.password': '',
'source.zip_url.proxy_url': '',
'source.zip_url.url': '',
'source.zip_url.username': '',
'ssl.certificate': '',
'ssl.certificate_authorities': '',
'ssl.key': '',
'ssl.key_passphrase': '',
'ssl.supported_protocols': ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'],
'ssl.verification_mode': 'full',
synthetics_args: [],
tags: [],
'throttling.config': '5d/3u/20l',
'throttling.download_speed': '5',
'throttling.is_enabled': true,
'throttling.latency': '20',
'throttling.upload_speed': '3',
timeout: null,
type: 'browser',
'url.port': null,
urls: '',
},
];
const soData = [
{
attributes: formatSecrets({
...payloadData[0],
revision: 1,
} as any),
type: 'synthetics-monitor',
},
{
attributes: formatSecrets({
...payloadData[1],
revision: 1,
} as any),
type: 'synthetics-monitor',
},
];
const soResult = soData.map((so) => ({ id: 'test-id', ...so }));

223
x-pack/plugins/synthetics/server/synthetics_service/project_monitor_formatter.ts
View file

@ -13,6 +13,8 @@ import {
SavedObjectsFindResult,
} from '@kbn/core/server';
import { EncryptedSavedObjectsClient } from '@kbn/encrypted-saved-objects-plugin/server';
import { deleteMonitor } from '../routes/monitor_cruds/delete_monitor';
import { syncNewMonitorBulk } from '../routes/monitor_cruds/bulk_cruds/add_monitor_bulk';
import { SyntheticsMonitorClient } from './synthetics_monitor/synthetics_monitor_client';
import {
BrowserFields,
@ -22,6 +24,7 @@ import {
ServiceLocationErrors,
ProjectBrowserMonitor,
Locations,
PrivateLocation,
} from '../../common/runtime_types';
import {
syntheticsMonitorType,
@ -29,9 +32,7 @@ import {
} from '../legacy_uptime/lib/saved_objects/synthetics_monitor';
import { normalizeProjectMonitor } from './normalizers/browser';
import { formatSecrets, normalizeSecrets } from './utils/secrets';
import { syncNewMonitor } from '../routes/monitor_cruds/add_monitor';
import { syncEditedMonitor } from '../routes/monitor_cruds/edit_monitor';
import { deleteMonitor } from '../routes/monitor_cruds/delete_monitor';
import { validateProjectMonitor } from '../routes/monitor_cruds/monitor_validation';
import type { UptimeServerSetup } from '../legacy_uptime/lib/adapters/framework';
@ -41,14 +42,17 @@ interface StaleMonitor {
savedObjectId: string;
}
type StaleMonitorMap = Record<string, StaleMonitor>;
type FailedMonitors = Array<{ id: string; reason: string; details: string; payload?: object }>;
type FailedMonitors = Array<{ id?: string; reason: string; details: string; payload?: object }>;
export const INSUFFICIENT_FLEET_PERMISSIONS =
'Insufficient permissions. In order to configure private locations, you must have Fleet and Integrations write permissions. To resolve, please generate a new API key with a user who has Fleet and Integrations write permissions.';
export class ProjectMonitorFormatter {
private projectId: string;
private spaceId: string;
private keepStale: boolean;
private locations: Locations;
private privateLocations: Locations;
private privateLocations: PrivateLocation[];
private savedObjectsClient: SavedObjectsClientContract;
private encryptedSavedObjectsClient: EncryptedSavedObjectsClient;
private staleMonitorsMap: StaleMonitorMap = {};
@ -65,6 +69,8 @@ export class ProjectMonitorFormatter {
private request: KibanaRequest;
private subject?: Subject<unknown>;
private writeIntegrationPoliciesPermissions?: boolean;
constructor({
locations,
privateLocations,
@ -80,7 +86,7 @@ export class ProjectMonitorFormatter {
subject,
}: {
locations: Locations;
privateLocations: Locations;
privateLocations: PrivateLocation[];
keepStale: boolean;
savedObjectsClient: SavedObjectsClientContract;
encryptedSavedObjectsClient: EncryptedSavedObjectsClient;
@ -108,32 +114,65 @@ export class ProjectMonitorFormatter {
}
public configureAllProjectMonitors = async () => {
this.staleMonitorsMap = await this.getAllProjectMonitorsForProject();
const existingMonitors = await this.getProjectMonitorsForProject();
this.staleMonitorsMap = await this.getStaleMonitorsMap(existingMonitors);
const normalizedNewMonitors: BrowserFields[] = [];
const normalizedUpdateMonitors: BrowserFields[] = [];
for (const monitor of this.monitors) {
await this.configureProjectMonitor({ monitor });
await new Promise((resolve) => setTimeout(resolve, 100));
const previousMonitor = existingMonitors.find(
(monitorObj) =>
(monitorObj.attributes as BrowserFields)[ConfigKey.JOURNEY_ID] === monitor.id
);
const normM = await this.validateProjectMonitor({
monitor,
});
if (normM) {
if (previousMonitor) {
this.updatedMonitors.push(monitor.id);
if (this.staleMonitorsMap[monitor.id]) {
this.staleMonitorsMap[monitor.id].stale = false;
}
normalizedUpdateMonitors.push(normM);
} else {
normalizedNewMonitors.push(normM);
}
}
}
await this.createMonitorsBulk(normalizedNewMonitors);
await this.updateMonitorBulk(normalizedUpdateMonitors);
await this.handleStaleMonitors();
};
private configureProjectMonitor = async ({ monitor }: { monitor: ProjectBrowserMonitor }) => {
validatePermissions = async ({ monitor }: { monitor: ProjectBrowserMonitor }) => {
if (this.writeIntegrationPoliciesPermissions || (monitor.privateLocations ?? []).length === 0) {
return;
}
const {
integrations: { writeIntegrationPolicies },
} = await this.server.fleet.authz.fromRequest(this.request);
this.writeIntegrationPoliciesPermissions = writeIntegrationPolicies;
if (!writeIntegrationPolicies) {
throw new Error(INSUFFICIENT_FLEET_PERMISSIONS);
}
};
validateProjectMonitor = async ({ monitor }: { monitor: ProjectBrowserMonitor }) => {
try {
const {
integrations: { writeIntegrationPolicies },
} = await this.server.fleet.authz.fromRequest(this.request);
await this.validatePermissions({ monitor });
if (monitor.privateLocations?.length && !writeIntegrationPolicies) {
throw new Error(
'Insufficient permissions. In order to configure private locations, you must have Fleet and Integrations write permissions. To resolve, please generate a new API key with a user who has Fleet and Integrations write permissions.'
);
}
// check to see if monitor already exists
const normalizedMonitor = normalizeProjectMonitor({
monitor,
locations: this.locations,
privateLocations: this.privateLocations,
monitor,
projectId: this.projectId,
namespace: this.spaceId,
});
@ -154,20 +193,7 @@ export class ProjectMonitorFormatter {
return null;
}
const previousMonitor = await this.getExistingMonitor(monitor.id);
if (previousMonitor) {
await this.updateMonitor(previousMonitor, normalizedMonitor);
this.updatedMonitors.push(monitor.id);
if (this.staleMonitorsMap[monitor.id]) {
this.staleMonitorsMap[monitor.id].stale = false;
}
this.handleStreamingMessage({ message: `${monitor.id}: monitor updated successfully` });
} else {
await this.createMonitor(normalizedMonitor);
this.createdMonitors.push(monitor.id);
this.handleStreamingMessage({ message: `${monitor.id}: monitor created successfully` });
}
return normalizedMonitor;
} catch (e) {
this.server.logger.error(e);
this.failedMonitors.push({
@ -183,36 +209,42 @@ export class ProjectMonitorFormatter {
}
};
private getAllProjectMonitorsForProject = async (): Promise<StaleMonitorMap> => {
private getStaleMonitorsMap = async (
existingMonitors: Array<SavedObjectsFindResult<EncryptedSyntheticsMonitor>>
): Promise<StaleMonitorMap> => {
const staleMonitors: StaleMonitorMap = {};
let page = 1;
let totalMonitors = 0;
do {
const { total, saved_objects: savedObjects } = await this.getProjectMonitorsForProject(page);
savedObjects.forEach((savedObject) => {
const journeyId = (savedObject.attributes as BrowserFields)[ConfigKey.JOURNEY_ID];
if (journeyId) {
staleMonitors[journeyId] = {
stale: true,
savedObjectId: savedObject.id,
journeyId,
};
}
});
page++;
totalMonitors = total;
} while (Object.keys(staleMonitors).length < totalMonitors);
existingMonitors.forEach((savedObject) => {
const journeyId = (savedObject.attributes as BrowserFields)[ConfigKey.JOURNEY_ID];
if (journeyId) {
staleMonitors[journeyId] = {
stale: true,
savedObjectId: savedObject.id,
journeyId,
};
}
});
return staleMonitors;
};
private getProjectMonitorsForProject = async (page: number) => {
return await this.savedObjectsClient.find<EncryptedSyntheticsMonitor>({
public getProjectMonitorsForProject = async () => {
const finder = this.savedObjectsClient.createPointInTimeFinder({
type: syntheticsMonitorType,
page,
perPage: 500,
perPage: 1000,
filter: this.projectFilter,
});
const hits: Array<SavedObjectsFindResult<EncryptedSyntheticsMonitor>> = [];
for await (const result of finder.find()) {
hits.push(
...(result.saved_objects as Array<SavedObjectsFindResult<EncryptedSyntheticsMonitor>>)
);
}
await finder.close();
return hits;
};
private getExistingMonitor = async (
@ -228,15 +260,75 @@ export class ProjectMonitorFormatter {
return savedObjects?.[0];
};
private createMonitor = async (normalizedMonitor: BrowserFields) => {
await syncNewMonitor({
normalizedMonitor,
monitor: normalizedMonitor,
server: this.server,
syntheticsMonitorClient: this.syntheticsMonitorClient,
savedObjectsClient: this.savedObjectsClient,
request: this.request,
});
private createMonitorsBulk = async (monitors: BrowserFields[]) => {
try {
if (monitors.length > 0) {
const { newMonitors } = await syncNewMonitorBulk({
normalizedMonitors: monitors,
server: this.server,
syntheticsMonitorClient: this.syntheticsMonitorClient,
soClient: this.savedObjectsClient,
request: this.request,
privateLocations: this.privateLocations,
spaceId: this.spaceId,
});
if (newMonitors && newMonitors.length === monitors.length) {
this.createdMonitors.push(...monitors.map((monitor) => monitor[ConfigKey.JOURNEY_ID]!));
this.handleStreamingMessage({
message: `${monitors.length} monitor${
monitors.length > 1 ? 's' : ''
} created successfully.`,
});
} else {
this.failedMonitors.push({
reason: `Failed to create ${monitors.length} monitors`,
details: 'Failed to create monitors',
payload: monitors,
});
this.handleStreamingMessage({
message: `Failed to create ${monitors.length} monitors`,
});
}
}
} catch (e) {
this.server.logger.error(e);
this.failedMonitors.push({
reason: `Failed to create ${monitors.length} monitors`,
details: e.message,
payload: monitors,
});
this.handleStreamingMessage({
message: `Failed to create ${monitors.length} monitors`,
});
}
};
private updateMonitorBulk = async (monitors: BrowserFields[]) => {
try {
for (const monitor of monitors) {
const previousMonitor = await this.getExistingMonitor(monitor[ConfigKey.JOURNEY_ID]!);
await this.updateMonitor(previousMonitor, monitor);
}
if (monitors.length > 0) {
this.handleStreamingMessage({
message: `${monitors.length} monitor${
monitors.length > 1 ? 's' : ''
} updated successfully.`,
});
}
} catch (e) {
this.server.logger.error(e);
this.failedMonitors.push({
reason: 'Failed to update monitors',
details: e.message,
payload: monitors,
});
this.handleStreamingMessage({
message: `Failed to update ${monitors.length} monitors`,
});
}
};
private updateMonitor = async (
@ -275,6 +367,7 @@ export class ProjectMonitorFormatter {
syntheticsMonitorClient: this.syntheticsMonitorClient,
savedObjectsClient: this.savedObjectsClient,
request: this.request,
spaceId: this.spaceId,
});
return { editedMonitor, errors: [] };
}

2
x-pack/plugins/synthetics/server/synthetics_service/service_api_client.ts
View file

@ -80,7 +80,7 @@ export class ServiceAPIClient {
}
async post(data: ServiceData) {
return this.callAPI('POST', data);
return this.callAPI('PUT', data);
}
async put(data: ServiceData) {

32
x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.test.ts
View file

@ -13,6 +13,7 @@ import times from 'lodash/times';
import {
LocationStatus,
MonitorFields,
PrivateLocation,
SyntheticsMonitorWithId,
} from '../../../common/runtime_types';
@ -64,6 +65,20 @@ describe('SyntheticsMonitorClient', () => {
};
});
const privateLocations: PrivateLocation[] = times(1).map((n) => {
return {
id: `loc-${n}`,
label: 'Test private location',
geo: {
lat: 0,
lon: 0,
},
isServiceManaged: false,
agentPolicyId: `loc-${n}`,
concurrentMonitors: 1,
};
});
const monitor = {
type: 'http',
enabled: true,
@ -87,12 +102,18 @@ describe('SyntheticsMonitorClient', () => {
const id = 'test-id-1';
const client = new SyntheticsMonitorClient(syntheticsService, serverMock);
client.privateLocationAPI.createMonitor = jest.fn();
client.privateLocationAPI.createMonitors = jest.fn();
await client.addMonitor(monitor, id, mockRequest, savedObjectsClientMock);
await client.addMonitors(
[{ monitor, id }],
mockRequest,
savedObjectsClientMock,
privateLocations,
'test-space'
);
expect(syntheticsService.addConfig).toHaveBeenCalledTimes(1);
expect(client.privateLocationAPI.createMonitor).toHaveBeenCalledTimes(1);
expect(client.privateLocationAPI.createMonitors).toHaveBeenCalledTimes(1);
});
it('should edit a monitor', async () => {
@ -102,7 +123,7 @@ describe('SyntheticsMonitorClient', () => {
const client = new SyntheticsMonitorClient(syntheticsService, serverMock);
client.privateLocationAPI.editMonitor = jest.fn();
await client.editMonitor(monitor, id, mockRequest, savedObjectsClientMock);
await client.editMonitor(monitor, id, mockRequest, savedObjectsClientMock, 'test-space');
expect(syntheticsService.editConfig).toHaveBeenCalledTimes(1);
expect(client.privateLocationAPI.editMonitor).toHaveBeenCalledTimes(1);
@ -117,7 +138,8 @@ describe('SyntheticsMonitorClient', () => {
await client.deleteMonitor(
monitor as unknown as SyntheticsMonitorWithId,
mockRequest,
savedObjectsClientMock
savedObjectsClientMock,
'test-space'
);
expect(syntheticsService.deleteConfigs).toHaveBeenCalledTimes(1);

65
x-pack/plugins/synthetics/server/synthetics_service/synthetics_monitor/synthetics_monitor_client.ts
View file

@ -14,6 +14,7 @@ import {
MonitorFields,
SyntheticsMonitorWithId,
HeartbeatConfig,
PrivateLocation,
} from '../../../common/runtime_types';
export class SyntheticsMonitorClient {
@ -26,36 +27,61 @@ export class SyntheticsMonitorClient {
this.privateLocationAPI = new SyntheticsPrivateLocation(server);
}
async addMonitor(
monitor: MonitorFields,
id: string,
async addMonitors(
monitors: Array<{ monitor: MonitorFields; id: string }>,
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
allPrivateLocations: PrivateLocation[],
spaceId: string
) {
await this.syntheticsService.setupIndexTemplates();
const privateConfigs: HeartbeatConfig[] = [];
const publicConfigs: HeartbeatConfig[] = [];
const config = formatHeartbeatRequest({
monitor,
monitorId: id,
customHeartbeatId: monitor[ConfigKey.CUSTOM_HEARTBEAT_ID],
});
for (const monitorObj of monitors) {
const { monitor, id } = monitorObj;
const config = formatHeartbeatRequest({
monitor,
monitorId: id,
customHeartbeatId: monitor[ConfigKey.CUSTOM_HEARTBEAT_ID],
});
const { privateLocations, publicLocations } = this.parseLocations(config);
const { privateLocations, publicLocations } = this.parseLocations(config);
if (privateLocations.length > 0) {
privateConfigs.push(config);
}
if (privateLocations.length > 0) {
await this.privateLocationAPI.createMonitor(config, request, savedObjectsClient);
if (publicLocations.length > 0) {
publicConfigs.push(config);
}
}
if (publicLocations.length > 0) {
return await this.syntheticsService.addConfig(config);
let newPolicies;
if (privateConfigs.length > 0) {
newPolicies = await this.privateLocationAPI.createMonitors(
privateConfigs,
request,
savedObjectsClient,
allPrivateLocations,
spaceId
);
}
let syncErrors;
if (publicConfigs.length > 0) {
syncErrors = await this.syntheticsService.addConfig(publicConfigs);
}
return { newPolicies, syncErrors };
}
async editMonitor(
editedMonitor: MonitorFields,
id: string,
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
spaceId: string
) {
const editedConfig = formatHeartbeatRequest({
monitor: editedMonitor,
@ -65,7 +91,7 @@ export class SyntheticsMonitorClient {
const { publicLocations } = this.parseLocations(editedConfig);
await this.privateLocationAPI.editMonitor(editedConfig, request, savedObjectsClient);
await this.privateLocationAPI.editMonitor(editedConfig, request, savedObjectsClient, spaceId);
if (publicLocations.length > 0) {
return await this.syntheticsService.editConfig(editedConfig);
@ -77,9 +103,10 @@ export class SyntheticsMonitorClient {
async deleteMonitor(
monitor: SyntheticsMonitorWithId,
request: KibanaRequest,
savedObjectsClient: SavedObjectsClientContract
savedObjectsClient: SavedObjectsClientContract,
spaceId: string
) {
await this.privateLocationAPI.deleteMonitor(monitor, request, savedObjectsClient);
await this.privateLocationAPI.deleteMonitor(monitor, request, savedObjectsClient, spaceId);
return await this.syntheticsService.deleteConfigs([monitor]);
}

4
x-pack/plugins/synthetics/server/synthetics_service/synthetics_service.ts
View file

@ -257,8 +257,8 @@ export class SyntheticsService {
};
}
async addConfig(config: HeartbeatConfig) {
const monitors = this.formatConfigs([config]);
async addConfig(config: HeartbeatConfig | HeartbeatConfig[]) {
const monitors = this.formatConfigs(Array.isArray(config) ? config : [config]);
this.apiKey = await this.getApiKey();

15
x-pack/test/api_integration/apis/uptime/rest/add_monitor_project.ts
View file

@ -19,7 +19,7 @@ import { PrivateLocationTestService } from './services/private_location_test_ser
import { comparePolicies, getTestProjectSyntheticsPolicy } from './sample_data/test_policy';
export default function ({ getService }: FtrProviderContext) {
describe('[PUT] /api/uptime/service/monitors', function () {
describe('AddProjectMonitors', function () {
this.tags('skipCloud');
const supertest = getService('supertest');
@ -125,6 +125,7 @@ export default function ({ getService }: FtrProviderContext) {
);
expect(messages).to.have.length(2);
expect(messages[0]).eql('1 monitor updated successfully.');
expect(messages[1].createdMonitors).eql([]);
expect(messages[1].failedMonitors).eql([]);
expect(messages[1].updatedMonitors).eql(
@ -234,6 +235,7 @@ export default function ({ getService }: FtrProviderContext) {
);
expect(messages).to.have.length(2);
expect(messages[0]).eql('1 monitor updated successfully.');
expect(messages[1].createdMonitors).eql([]);
expect(messages[1].failedMonitors).eql([]);
expect(messages[1].deletedMonitors).eql([]);
@ -281,6 +283,7 @@ export default function ({ getService }: FtrProviderContext) {
keep_stale: false,
})
);
expect(messages).to.have.length(3);
// expect monitor to have been deleted
@ -650,6 +653,7 @@ export default function ({ getService }: FtrProviderContext) {
JSON.stringify(projectMonitors)
);
expect(messages).to.have.length(2);
expect(messages[0]).eql('1 monitor updated successfully.');
expect(messages[1].updatedMonitors).eql([projectMonitors.monitors[0].id]);
// ensure that monitor can still be decrypted
@ -745,8 +749,8 @@ export default function ({ getService }: FtrProviderContext) {
);
expect(messages).to.have.length(3);
expect(messages[0]).to.eql(`${testMonitors[0].id}: monitor created successfully`);
expect(messages[1]).to.eql('test-id-2: failed to create or update monitor');
expect(messages[0]).to.eql('test-id-2: failed to create or update monitor');
expect(messages[1]).to.eql(`1 monitor created successfully.`);
expect(messages[2]).to.eql({
createdMonitors: [testMonitors[0].id],
updatedMonitors: [],
@ -946,10 +950,9 @@ export default function ({ getService }: FtrProviderContext) {
monitors: testMonitors,
})
);
expect(messages).to.have.length(3);
expect(messages).to.have.length(2);
expect(messages).to.eql([
`${testMonitors[0].id}: monitor created successfully`,
'test-id-2: monitor created successfully',
`2 monitors created successfully.`,
{
createdMonitors: [testMonitors[0].id, 'test-id-2'],
updatedMonitors: [],