[Uptimes] Push configs to service (#120069) (#120458)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/uptime/common/types/index.ts

@@ -33,9 +33,10 @@ export interface MonitorIdParam {
 
 export type SyntheticsMonitorSavedObject = SimpleSavedObject<{
   name: string;
-  runOnce: boolean;
+  runOnce?: boolean;
   urls?: string[];
   tags?: string[];
+  locations: string[];
   schedule: string;
   type: 'http' | 'tcp' | 'icmp' | 'browser';
   source?: {
@@ -59,4 +60,4 @@ export interface ManifestLocation {
   status: string;
 }
 
-export type ServiceLocations = Array<{ id: string; label: string; geo: LocationGeo }>;
+export type ServiceLocations = Array<{ id: string; label: string; geo: LocationGeo; url: string }>;

x-pack/plugins/uptime/server/lib/adapters/framework/adapter_types.ts

@@ -26,6 +26,7 @@ import { SecurityPluginStart } from '../../../../../security/server';
 import { CloudSetup } from '../../../../../cloud/server';
 import { FleetStartContract } from '../../../../../fleet/server';
 import { UptimeConfig } from '../../../../common/config';
+import { SyntheticsService } from '../../synthetics_service/synthetics_service';
 
 export type UMElasticsearchQueryFn<P, R = any> = (
   params: {
@@ -47,6 +48,7 @@ export interface UptimeServerSetup {
   security: SecurityPluginStart;
   savedObjectsClient: SavedObjectsClientContract;
   encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
+  syntheticsService: SyntheticsService;
 }
 
 export interface UptimeCorePluginsSetup {

x-pack/plugins/uptime/server/lib/synthetics_service/get_api_key.test.ts

@@ -50,7 +50,7 @@ describe('getAPIKeyTest', function () {
             cluster: ['monitor', 'read_ilm', 'read_pipeline'],
             index: [
               {
-                names: ['synthetics-*'],
+                names: ['synthetics-*', 'heartbeat-*'],
                 privileges: ['view_index_metadata', 'create_doc', 'auto_configure'],
               },
             ],

x-pack/plugins/uptime/server/lib/synthetics_service/get_api_key.ts

@@ -28,10 +28,15 @@ export const getAPIKeyForSyntheticsService = async ({
     includedHiddenTypes: [syntheticsServiceApiKey.name],
   });
 
-  const apiKey = await getSyntheticsServiceAPIKey(encryptedClient);
-  if (apiKey) {
-    return apiKey;
+  try {
+    const apiKey = await getSyntheticsServiceAPIKey(encryptedClient);
+    if (apiKey) {
+      return apiKey;
+    }
+  } catch (err) {
+    // TODO: figure out how to handle decryption errors
   }
+
   return await generateAndSaveAPIKey({ request, security, savedObjectsClient });
 };
 
@@ -61,7 +66,7 @@ export const generateAndSaveAPIKey = async ({
         cluster: ['monitor', 'read_ilm', 'read_pipeline'],
         index: [
           {
-            names: ['synthetics-*'],
+            names: ['synthetics-*', 'heartbeat-*'],
             privileges: ['view_index_metadata', 'create_doc', 'auto_configure'],
           },
         ],

x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.test.ts

@@ -27,13 +27,7 @@ describe('getServiceLocations', function () {
   });
   it('should return parsed locations', async () => {
     const locations = await getServiceLocations({
-      config: {
-        unsafe: {
-          service: {
-            manifestUrl: 'http://local.dev',
-          },
-        },
-      },
+      manifestUrl: 'http://local.dev',
     });
 
     expect(locations).toEqual([
@@ -44,6 +38,7 @@ describe('getServiceLocations', function () {
         },
         id: 'us_central',
         label: 'US Central',
+        url: 'https://local.dev',
       },
     ]);
   });

x-pack/plugins/uptime/server/lib/synthetics_service/get_service_locations.ts

@@ -6,20 +6,19 @@
  */
 
 import axios from 'axios';
-import { UptimeConfig } from '../../../common/config';
 import { ManifestLocation, ServiceLocations } from '../../../common/types';
 
-export async function getServiceLocations({ config }: { config: UptimeConfig }) {
-  const manifestURL = config.unsafe.service.manifestUrl;
+export async function getServiceLocations({ manifestUrl }: { manifestUrl: string }) {
   const locations: ServiceLocations = [];
   try {
-    const { data } = await axios.get<Record<string, ManifestLocation>>(manifestURL);
+    const { data } = await axios.get<Record<string, ManifestLocation>>(manifestUrl);
 
     Object.entries(data.locations).forEach(([locationId, location]) => {
       locations.push({
         id: locationId,
         label: location.geo.name,
         geo: location.geo.location,
+        url: location.url,
       });
     });
 

x-pack/plugins/uptime/server/lib/synthetics_service/service_api_client.ts

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import axios from 'axios';
+import { forkJoin, from as rxjsFrom, Observable, of } from 'rxjs';
+import { catchError, tap } from 'rxjs/operators';
+import { ServiceLocations, SyntheticsMonitorSavedObject } from '../../../common/types';
+import { getServiceLocations } from './get_service_locations';
+import { Logger } from '../../../../../../src/core/server';
+
+const TEST_SERVICE_USERNAME = 'localKibanaIntegrationTestsUser';
+
+export type MonitorConfigs = Array<
+  SyntheticsMonitorSavedObject['attributes'] & {
+    id: string;
+    source?: {
+      inline: {
+        script: string;
+      };
+    };
+  }
+>;
+
+export interface ServiceData {
+  monitors: MonitorConfigs;
+  output: {
+    hosts: string[];
+    api_key: string;
+  };
+}
+
+export class ServiceAPIClient {
+  private readonly username: string;
+  private readonly authorization: string;
+  private locations: ServiceLocations;
+  private logger: Logger;
+
+  constructor(manifestUrl: string, username: string, password: string, logger: Logger) {
+    this.username = username;
+    this.authorization = 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64');
+    this.logger = logger;
+    this.locations = [];
+
+    getServiceLocations({ manifestUrl }).then((result) => {
+      this.locations = result;
+    });
+  }
+
+  async post(data: ServiceData) {
+    return this.callAPI('POST', data);
+  }
+
+  async put(data: ServiceData) {
+    return this.callAPI('POST', data);
+  }
+
+  async delete(data: ServiceData) {
+    return this.callAPI('DELETE', data);
+  }
+
+  async callAPI(method: 'POST' | 'PUT' | 'DELETE', { monitors: allMonitors, output }: ServiceData) {
+    if (this.username === TEST_SERVICE_USERNAME) {
+      // we don't want to call service while local integration tests are running
+      return;
+    }
+
+    const callServiceEndpoint = (monitors: ServiceData['monitors'], url: string) => {
+      return axios({
+        method,
+        url: url + '/monitors',
+        data: { monitors, output },
+        headers: {
+          Authorization: this.authorization,
+        },
+      });
+    };
+
+    const pushErrors: Array<{ locationId: string; error: Error }> = [];
+
+    const promises: Array<Observable<unknown>> = [];
+
+    this.locations.forEach(({ id, url }) => {
+      const locMonitors = allMonitors.filter(
+        ({ locations }) => !locations || locations?.includes(id)
+      );
+      if (locMonitors.length > 0) {
+        promises.push(
+          rxjsFrom(callServiceEndpoint(locMonitors, url)).pipe(
+            tap((result) => {
+              this.logger.debug(result.data);
+            }),
+            catchError((err) => {
+              pushErrors.push({ locationId: id, error: err });
+              this.logger.error(err);
+              // we don't want to throw an unhandled exception here
+              return of(true);
+            })
+          )
+        );
+      }
+    });
+
+    await forkJoin(promises).toPromise();
+
+    return pushErrors;
+  }
+}

x-pack/plugins/uptime/server/lib/synthetics_service/synthetics_service.ts

@@ -7,7 +7,8 @@
 
 /* eslint-disable max-classes-per-file */
 
-import axios from 'axios';
+import { ValuesType } from 'utility-types';
+
 import {
   CoreStart,
   KibanaRequest,
@@ -27,6 +28,7 @@ import { SyntheticsMonitorSavedObject } from '../../../common/types';
 import { syntheticsMonitorType } from '../saved_objects/synthetics_monitor';
 import { getEsHosts } from './get_es_hosts';
 import { UptimeConfig } from '../../../common/config';
+import { MonitorConfigs, ServiceAPIClient } from './service_api_client';
 
 const SYNTHETICS_SERVICE_SYNC_MONITORS_TASK_TYPE =
   'UPTIME:SyntheticsService:Sync-Saved-Monitor-Objects';
@@ -35,6 +37,7 @@ const SYNTHETICS_SERVICE_SYNC_MONITORS_TASK_ID = 'UPTIME:SyntheticsService:sync-
 export class SyntheticsService {
   private logger: Logger;
   private readonly server: UptimeServerSetup;
+  private apiClient: ServiceAPIClient;
 
   private readonly config: UptimeConfig;
   private readonly esHosts: string[];
@@ -46,6 +49,10 @@ export class SyntheticsService {
     this.server = server;
     this.config = server.config;
 
+    const { manifestUrl, username, password } = this.config.unsafe.service;
+
+    this.apiClient = new ServiceAPIClient(manifestUrl, username, password, logger);
+
     this.esHosts = getEsHosts({ config: this.config, cloud: server.cloud });
   }
 
@@ -101,7 +108,7 @@ export class SyntheticsService {
             async run() {
               const { state } = taskInstance;
 
-              // TODO: Push API Key and Monitor Configs to service here
+              await service.pushConfigs();
 
               return { state };
             },
@@ -120,7 +127,7 @@ export class SyntheticsService {
         id: SYNTHETICS_SERVICE_SYNC_MONITORS_TASK_ID,
         taskType: SYNTHETICS_SERVICE_SYNC_MONITORS_TASK_TYPE,
         schedule: {
-          interval: '5m',
+          interval: '1m',
         },
         params: {},
         state: {},
@@ -137,7 +144,7 @@ export class SyntheticsService {
       });
   }
 
-  async pushConfigs(request: KibanaRequest) {
+  async getOutput(request?: KibanaRequest) {
     if (!this.apiKey) {
       try {
         this.apiKey = await getAPIKeyForSyntheticsService({ server: this.server, request });
@@ -152,43 +159,92 @@ export class SyntheticsService {
       throw error;
     }
 
-    const monitors = await this.getMonitorConfigs();
+    return {
+      hosts: this.esHosts,
+      api_key: `${this.apiKey.id}:${this.apiKey.apiKey}`,
+    };
+  }
+
+  async pushConfigs(request?: KibanaRequest, configs?: MonitorConfigs) {
+    const monitors = this.formatConfigs(configs || (await this.getMonitorConfigs()));
     const data = {
       monitors,
-      output: {
-        hosts: this.esHosts,
-        api_key: `${this.apiKey.id}:${this.apiKey.apiKey}`,
-      },
+      output: await this.getOutput(request),
     };
 
-    const { url, username, password } = this.config.unsafe.service;
-
     try {
-      await axios({
-        method: 'POST',
-        url: url + '/monitors',
-        data,
-        headers: {
-          Authorization: 'Basic ' + Buffer.from(`${username}:${password}`).toString('base64'),
-        },
-      });
+      return await this.apiClient.post(data);
     } catch (e) {
       this.logger.error(e);
+      throw e;
     }
   }
 
+  async deleteConfigs(request: KibanaRequest, configs: MonitorConfigs) {
+    const data = {
+      monitors: configs,
+      output: await this.getOutput(request),
+    };
+    return await this.apiClient.delete(data);
+  }
+
   async getMonitorConfigs() {
     const savedObjectsClient = this.server.savedObjectsClient;
-    const monitorsSavedObjects = await savedObjectsClient.find<SyntheticsMonitorSavedObject>({
+    const monitorsSavedObjects = await savedObjectsClient.find<
+      SyntheticsMonitorSavedObject['attributes']
+    >({
       type: syntheticsMonitorType,
     });
 
     const savedObjectsList = monitorsSavedObjects.saved_objects;
-    return savedObjectsList.map(({ attributes, id }) => ({
+    return savedObjectsList.map<ValuesType<MonitorConfigs>>(({ attributes, id }) => ({
       ...attributes,
       id,
     }));
   }
+
+  formatConfigs(configs: MonitorConfigs) {
+    // TODO: Move to dedicated formatter class
+    function parseSchedule(schedule: any) {
+      if (schedule?.number) {
+        return `@every ${schedule.number}${schedule.unit}`;
+      }
+      return schedule;
+    }
+
+    function parseUrl(urls?: string | string[]) {
+      if (!urls) {
+        return undefined;
+      }
+      if (urls instanceof Array) {
+        return urls;
+      }
+      return [urls];
+    }
+
+    function parseInlineSource(monAttrs: any) {
+      if (monAttrs['source.inline.script']) {
+        return {
+          inline: {
+            script: monAttrs['source.inline.script'],
+          },
+        };
+      }
+    }
+    return configs.map((monAttrs) => {
+      const { id, schedule, type, name, locations, tags, urls } = monAttrs;
+      return {
+        id,
+        type,
+        name,
+        locations,
+        tags,
+        source: parseInlineSource(monAttrs),
+        urls: parseUrl(urls),
+        schedule: parseSchedule(schedule),
+      };
+    });
+  }
 }
 
 class APIKeyMissingError extends Error {

x-pack/plugins/uptime/server/plugin.ts

@@ -27,6 +27,7 @@ import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_fie
 import { Dataset } from '../../rule_registry/server';
 import { UptimeConfig } from '../common/config';
 import { SyntheticsService } from './lib/synthetics_service/synthetics_service';
+import { syntheticsServiceApiKey } from './lib/saved_objects/service_api_key';
 
 export type UptimeRuleRegistry = ReturnType<Plugin['setup']>['ruleRegistry'];
 
@@ -89,9 +90,16 @@ export class Plugin implements PluginType {
   }
 
   public start(coreStart: CoreStart, plugins: UptimeCorePluginsStart) {
-    this.savedObjectsClient = new SavedObjectsClient(
-      coreStart.savedObjects.createInternalRepository()
-    );
+    if (this.server?.config?.unsafe?.service.enabled) {
+      this.savedObjectsClient = new SavedObjectsClient(
+        coreStart.savedObjects.createInternalRepository([syntheticsServiceApiKey.name])
+      );
+    } else {
+      this.savedObjectsClient = new SavedObjectsClient(
+        coreStart.savedObjects.createInternalRepository()
+      );
+    }
+
     if (this.server) {
       this.server.security = plugins.security;
       this.server.fleet = plugins.fleet;
@@ -102,6 +110,9 @@ export class Plugin implements PluginType {
     if (this.server?.config?.unsafe?.service.enabled) {
       this.syntheticService?.init(coreStart);
       this.syntheticService?.scheduleSyncTask(plugins.taskManager);
+      if (this.server && this.syntheticService) {
+        this.server.syntheticsService = this.syntheticService;
+      }
     }
   }
 

x-pack/plugins/uptime/server/rest_api/synthetics_service/add_monitor.ts

@@ -16,11 +16,21 @@ export const addSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({
   validate: {
     body: schema.any(),
   },
-  handler: async ({ request, savedObjectsClient }): Promise<any> => {
-    const monitor = request.body as SyntheticsMonitorSavedObject;
+  handler: async ({ request, savedObjectsClient, server }): Promise<any> => {
+    const monitor = request.body as SyntheticsMonitorSavedObject['attributes'];
 
     const newMonitor = await savedObjectsClient.create(syntheticsMonitorType, monitor);
-    // TODO: call to service sync
+
+    const { syntheticsService } = server;
+
+    const errors = await syntheticsService.pushConfigs(request, [
+      { ...newMonitor.attributes, id: newMonitor.id },
+    ]);
+
+    if (errors) {
+      return errors;
+    }
+
     return newMonitor;
   },
 });

x-pack/plugins/uptime/server/rest_api/synthetics_service/delete_monitor.ts

@@ -9,6 +9,7 @@ import { SavedObjectsErrorHelpers } from '../../../../../../src/core/server';
 import { UMRestApiRouteFactory } from '../types';
 import { API_URLS } from '../../../common/constants';
 import { syntheticsMonitorType } from '../../lib/saved_objects/synthetics_monitor';
+import { SyntheticsMonitorSavedObject } from '../../../common/types';
 
 export const deleteSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({
   method: 'DELETE',
@@ -18,17 +19,30 @@ export const deleteSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({
       monitorId: schema.string(),
     }),
   },
-  handler: async ({ request, savedObjectsClient }): Promise<any> => {
+  handler: async ({ request, savedObjectsClient, server }): Promise<any> => {
     const { monitorId } = request.params;
 
+    const { syntheticsService } = server;
+
     try {
+      const monitor = await savedObjectsClient.get<SyntheticsMonitorSavedObject['attributes']>(
+        syntheticsMonitorType,
+        monitorId
+      );
+
       await savedObjectsClient.delete(syntheticsMonitorType, monitorId);
-      // TODO: call to service sync
+      const errors = await syntheticsService.deleteConfigs(request, [
+        { ...monitor.attributes, id: monitorId },
+      ]);
+      if (errors) {
+        return errors;
+      }
       return monitorId;
     } catch (getErr) {
       if (SavedObjectsErrorHelpers.isNotFoundError(getErr)) {
         return 'Not found';
       }
+      throw getErr;
     }
   },
 });

x-pack/plugins/uptime/server/rest_api/synthetics_service/edit_monitor.ts

@@ -19,13 +19,26 @@ export const editSyntheticsMonitorRoute: UMRestApiRouteFactory = () => ({
     }),
     body: schema.any(),
   },
-  handler: async ({ request, savedObjectsClient }): Promise<any> => {
+  handler: async ({ request, savedObjectsClient, server }): Promise<any> => {
     const monitor = request.body as SyntheticsMonitorSavedObject['attributes'];
 
     const { monitorId } = request.params;
 
+    const { syntheticsService } = server;
+
     const editMonitor = await savedObjectsClient.update(syntheticsMonitorType, monitorId, monitor);
-    // TODO: call to service sync
+
+    const errors = await syntheticsService.pushConfigs(request, [
+      {
+        ...(editMonitor.attributes as SyntheticsMonitorSavedObject['attributes']),
+        id: editMonitor.id,
+      },
+    ]);
+
+    if (errors) {
+      return errors;
+    }
+
     return editMonitor;
   },
 });

x-pack/plugins/uptime/server/rest_api/synthetics_service/get_service_locations.ts

@@ -13,5 +13,6 @@ export const getServiceLocationsRoute: UMRestApiRouteFactory = () => ({
   method: 'GET',
   path: API_URLS.SERVICE_LOCATIONS,
   validate: {},
-  handler: async ({ server }): Promise<any> => getServiceLocations({ config: server.config }),
+  handler: async ({ server }): Promise<any> =>
+    getServiceLocations({ manifestUrl: server.config.service.manifestUrl }),
 });

x-pack/test/api_integration/config.ts

@@ -38,7 +38,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi
         '--xpack.uptime.unsafe.service.enabled=true',
         '--xpack.uptime.unsafe.service.password=test',
         '--xpack.uptime.unsafe.service.manifestUrl=http://test.com',
-        '--xpack.uptime.unsafe.service.username=user',
+        '--xpack.uptime.unsafe.service.username=localKibanaIntegrationTestsUser',
         `--xpack.securitySolution.enableExperimental=${JSON.stringify(['ruleRegistryEnabled'])}`,
       ],
     },