[ML] Fixing saved object sync in a serverless environment when apis are missing (#156585)

Temporarily fixes https://github.com/elastic/kibana/issues/156500

In a serverless environment various elasticsearch apis may be missing
depending on the project. We should allow these errors and ensure that
saved objects related to the missing api are not synced.

In the near future we should hopefully be able to determine which apis
are missing before attempting to call them.

x-pack/plugins/ml/server/saved_objects/checks.ts

@@ -20,9 +20,8 @@ import type {
   MlSavedObjectType,
 } from '../../common/types/saved_objects';
 
-import type { DataFrameAnalyticsConfig } from '../../common/types/data_frame_analytics';
 import type { ResolveMlCapabilities } from '../../common/types/capabilities';
-import { getJobDetailsFromTrainedModel } from './util';
+import { getJobDetailsFromTrainedModel, getJobsAndModels } from './util';
 
 export interface JobSavedObjectStatus {
   jobId: string;
@@ -79,7 +78,7 @@ export function checksFactory(
   mlSavedObjectService: MLSavedObjectService
 ) {
   async function checkStatus(): Promise<StatusResponse> {
-    const [
+    const {
       jobObjects,
       allJobObjects,
       modelObjects,
@@ -88,18 +87,7 @@ export function checksFactory(
       datafeeds,
       dfaJobs,
       models,
-    ] = await Promise.all([
-      mlSavedObjectService.getAllJobObjects(undefined, false),
-      mlSavedObjectService.getAllJobObjectsForAllSpaces(),
-      mlSavedObjectService.getAllTrainedModelObjects(false),
-      mlSavedObjectService.getAllTrainedModelObjectsForAllSpaces(),
-      client.asInternalUser.ml.getJobs(),
-      client.asInternalUser.ml.getDatafeeds(),
-      client.asInternalUser.ml.getDataFrameAnalytics() as unknown as {
-        data_frame_analytics: DataFrameAnalyticsConfig[];
-      },
-      client.asInternalUser.ml.getTrainedModels(),
-    ]);
+    } = await getJobsAndModels(client, mlSavedObjectService);
 
     const jobSavedObjectsStatus: JobSavedObjectStatus[] = jobObjects.map(
       ({ attributes, namespaces }) => {
@@ -111,10 +99,10 @@ export function checksFactory(
         let datafeedExists: boolean | undefined;
 
         if (type === 'anomaly-detector') {
-          jobExists = adJobs.jobs.some((j) => j.job_id === jobId);
-          datafeedExists = datafeeds.datafeeds.some((d) => d.job_id === jobId);
+          jobExists = adJobs.some((j) => j.job_id === jobId);
+          datafeedExists = datafeeds.some((d) => d.job_id === jobId);
         } else {
-          jobExists = dfaJobs.data_frame_analytics.some((j) => j.id === jobId);
+          jobExists = dfaJobs.some((j) => j.id === jobId);
         }
 
         return {
@@ -130,12 +118,12 @@ export function checksFactory(
       }
     );
 
-    const dfaJobsCreateTimeMap = dfaJobs.data_frame_analytics.reduce((acc, cur) => {
+    const dfaJobsCreateTimeMap = dfaJobs.reduce((acc, cur) => {
       acc.set(cur.id, cur.create_time!);
       return acc;
     }, new Map<string, number>());
 
-    const modelJobExits = models.trained_model_configs.reduce((acc, cur) => {
+    const modelJobExits = models.reduce((acc, cur) => {
       const job = getJobDetailsFromTrainedModel(cur);
       if (job === null) {
         return acc;
@@ -152,7 +140,7 @@ export function checksFactory(
 
     const modelSavedObjectsStatus: TrainedModelSavedObjectStatus[] = modelObjects.map(
       ({ attributes: { job, model_id: modelId }, namespaces }) => {
-        const trainedModelExists = models.trained_model_configs.some((m) => m.model_id === modelId);
+        const trainedModelExists = models.some((m) => m.model_id === modelId);
         const dfaJobExists = modelJobExits.get(modelId) ?? null;
 
         return {
@@ -194,14 +182,14 @@ export function checksFactory(
     );
     const modelObjectIds = new Set(modelSavedObjectsStatus.map(({ modelId }) => modelId));
 
-    const anomalyDetectorsStatus = adJobs.jobs
+    const anomalyDetectorsStatus = adJobs
       .filter(({ job_id: jobId }) => {
         // only list jobs which are in the current space (adObjectIds)
         // or are not in any spaces (nonSpaceADObjectIds)
         return adObjectIds.has(jobId) === true || nonSpaceADObjectIds.has(jobId) === false;
       })
       .map(({ job_id: jobId }) => {
-        const datafeedId = datafeeds.datafeeds.find((df) => df.job_id === jobId)?.datafeed_id;
+        const datafeedId = datafeeds.find((df) => df.job_id === jobId)?.datafeed_id;
         return {
           jobId,
           datafeedId: datafeedId ?? null,
@@ -211,7 +199,7 @@ export function checksFactory(
         };
       });
 
-    const dataFrameAnalyticsStatus = dfaJobs.data_frame_analytics
+    const dataFrameAnalyticsStatus = dfaJobs
       .filter(({ id: jobId }) => {
         // only list jobs which are in the current space (dfaObjectIds)
         // or are not in any spaces (nonSpaceDFAObjectIds)
@@ -227,7 +215,7 @@ export function checksFactory(
         };
       });
 
-    const modelsStatus = models.trained_model_configs
+    const modelsStatus = models
       .filter(({ model_id: modelId }) => {
         // only list jobs which are in the current space (adObjectIds)
         // or are not in any spaces (nonSpaceADObjectIds)

x-pack/plugins/ml/server/saved_objects/util.ts

@@ -6,10 +6,14 @@
  */
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import { SavedObjectsServiceStart, KibanaRequest } from '@kbn/core/server';
-import { SavedObjectsClient } from '@kbn/core/server';
+import {
+  type SavedObjectsServiceStart,
+  type KibanaRequest,
+  type IScopedClusterClient,
+  SavedObjectsClient,
+} from '@kbn/core/server';
+import type { TrainedModelJob, MLSavedObjectService } from './service';
 import { ML_JOB_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects';
-import type { TrainedModelJob } from './service';
 
 export function savedObjectClientsFactory(
   getSavedObjectsStart: () => SavedObjectsServiceStart | null
@@ -57,3 +61,98 @@ export function getJobDetailsFromTrainedModel(
   const createTime: number = model.metadata.analytics_config.create_time;
   return { job_id: jobId, create_time: createTime };
 }
+
+/*
+ * Function for calling elasticsearch APIs for retrieving ML jobs and models.
+ * The elasticsearch api may be missing in a serverless environment, in which case
+ * we return null.
+ */
+
+function mlFunctionsFactory(client: IScopedClusterClient) {
+  return {
+    async getJobs() {
+      try {
+        return client.asInternalUser.ml.getJobs();
+      } catch (error) {
+        return null;
+      }
+    },
+    async getDatafeeds() {
+      try {
+        return client.asInternalUser.ml.getDatafeeds();
+      } catch (error) {
+        return null;
+      }
+    },
+    async getTrainedModels() {
+      try {
+        return client.asInternalUser.ml.getTrainedModels();
+      } catch (error) {
+        return null;
+      }
+    },
+    async getDataFrameAnalytics() {
+      try {
+        return client.asInternalUser.ml.getDataFrameAnalytics();
+      } catch (error) {
+        return null;
+      }
+    },
+  };
+}
+
+/*
+ * Function for retrieving lists of jobs, models and saved objects.
+ * If any of the elasticsearch APIs are missing, it returns empty arrays
+ * so that the sync process does not create or delete any saved objects.
+ */
+
+export async function getJobsAndModels(
+  client: IScopedClusterClient,
+  mlSavedObjectService: MLSavedObjectService
+) {
+  const { getJobs, getDatafeeds, getTrainedModels, getDataFrameAnalytics } =
+    mlFunctionsFactory(client);
+
+  const [
+    jobObjects,
+    allJobObjects,
+    modelObjects,
+    allModelObjects,
+    adJobs,
+    datafeeds,
+    dfaJobs,
+    models,
+  ] = await Promise.all([
+    mlSavedObjectService.getAllJobObjects(undefined, false),
+    mlSavedObjectService.getAllJobObjectsForAllSpaces(),
+    mlSavedObjectService.getAllTrainedModelObjects(false),
+    mlSavedObjectService.getAllTrainedModelObjectsForAllSpaces(),
+    getJobs(),
+    getDatafeeds(),
+    getDataFrameAnalytics(),
+    getTrainedModels(),
+  ]);
+
+  const adJobObjects =
+    adJobs !== null ? jobObjects.filter((j) => j.attributes.type === 'anomaly-detector') : [];
+  const adAllJobObjects =
+    adJobs !== null ? allJobObjects.filter((j) => j.attributes.type === 'anomaly-detector') : [];
+  const dfaJobObjects =
+    dfaJobs !== null ? jobObjects.filter((j) => j.attributes.type === 'data-frame-analytics') : [];
+  const dfaAllJobObjects =
+    dfaJobs !== null
+      ? allJobObjects.filter((j) => j.attributes.type === 'data-frame-analytics')
+      : [];
+
+  return {
+    jobObjects: [...adJobObjects, ...dfaJobObjects],
+    allJobObjects: [...adAllJobObjects, ...dfaAllJobObjects],
+    modelObjects: models === null ? [] : modelObjects,
+    allModelObjects: models === null ? [] : allModelObjects,
+    adJobs: adJobs === null ? [] : adJobs.jobs,
+    datafeeds: datafeeds === null ? [] : datafeeds.datafeeds,
+    dfaJobs: dfaJobs === null ? [] : dfaJobs.data_frame_analytics,
+    models: models === null ? [] : models.trained_model_configs,
+  };
+}