[ML] Additional job spaces initialization (#83127)

* [ML] Additional job spaces initialization

* adding logs test

* updating integrations

* updating test text

* fixing logs jobs error

* fix bug with duplicate ids

* updating initialization log text

* fixing initialization text

* adding metrics overrides

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -77,6 +77,7 @@ async function createAnomalyDetectionJob({
     prefix: `${APM_ML_JOB_GROUP}-${snakeCase(environment)}-${randomToken}-`,
     groups: [APM_ML_JOB_GROUP],
     indexPatternName,
+    applyToAllSpaces: true,
     query: {
       bool: {
         filter: [

x-pack/plugins/ml/common/types/saved_objects.ts

@@ -6,3 +6,12 @@
 
 export type JobType = 'anomaly-detector' | 'data-frame-analytics';
 export const ML_SAVED_OBJECT_TYPE = 'ml-job';
+
+type Result = Record<string, { success: boolean; error?: any }>;
+
+export interface RepairSavedObjectResponse {
+  savedObjectsCreated: Result;
+  savedObjectsDeleted: Result;
+  datafeedsAdded: Result;
+  datafeedsRemoved: Result;
+}

x-pack/plugins/ml/server/lib/route_guard.ts

@@ -13,6 +13,7 @@ import {
   SavedObjectsClientContract,
 } from 'kibana/server';
 import { SpacesPluginSetup } from '../../../spaces/server';
+import type { SecurityPluginSetup } from '../../../security/server';
 
 import { jobSavedObjectServiceFactory, JobSavedObjectService } from '../saved_objects';
 import { MlLicense } from '../../common/license';
@@ -36,6 +37,7 @@ export class RouteGuard {
   private _getMlSavedObjectClient: GetMlSavedObjectClient;
   private _getInternalSavedObjectClient: GetInternalSavedObjectClient;
   private _spacesPlugin: SpacesPluginSetup | undefined;
+  private _authorization: SecurityPluginSetup['authz'] | undefined;
   private _isMlReady: () => Promise<void>;
 
   constructor(
@@ -43,12 +45,14 @@ export class RouteGuard {
     getSavedObject: GetMlSavedObjectClient,
     getInternalSavedObject: GetInternalSavedObjectClient,
     spacesPlugin: SpacesPluginSetup | undefined,
+    authorization: SecurityPluginSetup['authz'] | undefined,
     isMlReady: () => Promise<void>
   ) {
     this._mlLicense = mlLicense;
     this._getMlSavedObjectClient = getSavedObject;
     this._getInternalSavedObjectClient = getInternalSavedObject;
     this._spacesPlugin = spacesPlugin;
+    this._authorization = authorization;
     this._isMlReady = isMlReady;
   }
 
@@ -81,6 +85,7 @@ export class RouteGuard {
         mlSavedObjectClient,
         internalSavedObjectsClient,
         this._spacesPlugin !== undefined,
+        this._authorization,
         this._isMlReady
       );
       const client = context.core.elasticsearch.client;

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.test.ts

@@ -8,6 +8,7 @@ import { SavedObjectsClientContract, KibanaRequest, IScopedClusterClient } from
 import { Module } from '../../../common/types/modules';
 import { DataRecognizer } from '../data_recognizer';
 import type { MlClient } from '../../lib/ml_client';
+import { JobSavedObjectService } from '../../saved_objects';
 
 const callAs = () => Promise.resolve({ body: {} });
 
@@ -26,6 +27,7 @@ describe('ML - data recognizer', () => {
       find: jest.fn(),
       bulkCreate: jest.fn(),
     } as unknown) as SavedObjectsClientContract,
+    {} as JobSavedObjectService,
     { headers: { authorization: '' } } as KibanaRequest
   );
 

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts

@@ -44,6 +44,7 @@ import { jobServiceProvider } from '../job_service';
 import { resultsServiceProvider } from '../results_service';
 import { JobExistResult, JobStat } from '../../../common/types/data_recognizer';
 import { MlJobsStatsResponse } from '../job_service/jobs';
+import { JobSavedObjectService } from '../../saved_objects';
 
 const ML_DIR = 'ml';
 const KIBANA_DIR = 'kibana';
@@ -108,6 +109,8 @@ export class DataRecognizer {
   private _client: IScopedClusterClient;
   private _mlClient: MlClient;
   private _savedObjectsClient: SavedObjectsClientContract;
+  private _jobSavedObjectService: JobSavedObjectService;
+  private _request: KibanaRequest;
 
   private _authorizationHeader: object;
   private _modulesDir = `${__dirname}/modules`;
@@ -127,11 +130,14 @@ export class DataRecognizer {
     mlClusterClient: IScopedClusterClient,
     mlClient: MlClient,
     savedObjectsClient: SavedObjectsClientContract,
+    jobSavedObjectService: JobSavedObjectService,
     request: KibanaRequest
   ) {
     this._client = mlClusterClient;
     this._mlClient = mlClient;
     this._savedObjectsClient = savedObjectsClient;
+    this._jobSavedObjectService = jobSavedObjectService;
+    this._request = request;
     this._authorizationHeader = getAuthorizationHeader(request);
     this._jobsService = jobServiceProvider(mlClusterClient, mlClient);
     this._resultsService = resultsServiceProvider(mlClient);
@@ -394,7 +400,8 @@ export class DataRecognizer {
     end?: number,
     jobOverrides?: JobOverride | JobOverride[],
     datafeedOverrides?: DatafeedOverride | DatafeedOverride[],
-    estimateModelMemory: boolean = true
+    estimateModelMemory: boolean = true,
+    applyToAllSpaces: boolean = false
   ) {
     // load the config from disk
     const moduleConfig = await this.getModule(moduleId, jobPrefix);
@@ -458,7 +465,7 @@ export class DataRecognizer {
       if (useDedicatedIndex === true) {
         moduleConfig.jobs.forEach((job) => (job.config.results_index_name = job.id));
       }
-      saveResults.jobs = await this.saveJobs(moduleConfig.jobs);
+      saveResults.jobs = await this.saveJobs(moduleConfig.jobs, applyToAllSpaces);
     }
 
     // create the datafeeds
@@ -699,8 +706,8 @@ export class DataRecognizer {
   // save the jobs.
   // if any fail (e.g. it already exists), catch the error and mark the result
   // as success: false
-  async saveJobs(jobs: ModuleJob[]): Promise<JobResponse[]> {
-    return await Promise.all(
+  async saveJobs(jobs: ModuleJob[], applyToAllSpaces: boolean = false): Promise<JobResponse[]> {
+    const resp = await Promise.all(
       jobs.map(async (job) => {
         const jobId = job.id;
         try {
@@ -712,6 +719,19 @@ export class DataRecognizer {
         }
       })
     );
+    if (applyToAllSpaces === true) {
+      const canCreateGlobalJobs = await this._jobSavedObjectService.canCreateGlobalJobs(
+        this._request
+      );
+      if (canCreateGlobalJobs === true) {
+        await this._jobSavedObjectService.assignJobsToSpaces(
+          'anomaly-detector',
+          jobs.map((j) => j.id),
+          ['*']
+        );
+      }
+    }
+    return resp;
   }
 
   async saveJob(job: ModuleJob) {

x-pack/plugins/ml/server/plugin.ts

@@ -16,6 +16,7 @@ import {
   IClusterClient,
   SavedObjectsServiceStart,
 } from 'kibana/server';
+import type { SecurityPluginSetup } from '../../security/server';
 import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
 import { SpacesPluginSetup } from '../../spaces/server';
 import { PluginsSetup, RouteInitialization } from './types';
@@ -68,6 +69,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
   private clusterClient: IClusterClient | null = null;
   private savedObjectsStart: SavedObjectsServiceStart | null = null;
   private spacesPlugin: SpacesPluginSetup | undefined;
+  private security: SecurityPluginSetup | undefined;
   private isMlReady: Promise<void>;
   private setMlReady: () => void = () => {};
 
@@ -80,6 +82,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
 
   public setup(coreSetup: CoreSetup, plugins: PluginsSetup): MlPluginSetup {
     this.spacesPlugin = plugins.spaces;
+    this.security = plugins.security;
     const { admin, user, apmUser } = getPluginPrivileges();
 
     plugins.features.registerKibanaFeature({
@@ -140,6 +143,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
         getMlSavedObjectsClient,
         getInternalSavedObjectsClient,
         plugins.spaces,
+        plugins.security?.authz,
         () => this.isMlReady
       ),
       mlLicense: this.mlLicense,
@@ -185,6 +189,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
         this.mlLicense,
         plugins.spaces,
         plugins.cloud,
+        plugins.security?.authz,
         resolveMlCapabilities,
         () => this.clusterClient,
         () => getInternalSavedObjectsClient(),
@@ -202,6 +207,7 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
     // and create them if needed.
     const { initializeJobs } = jobSavedObjectsInitializationFactory(
       coreStart,
+      this.security,
       this.spacesPlugin !== undefined
     );
     initializeJobs().finally(() => {

x-pack/plugins/ml/server/routes/modules.ts

@@ -18,15 +18,23 @@ import {
 } from './schemas/modules';
 import { RouteInitialization } from '../types';
 import type { MlClient } from '../lib/ml_client';
+import type { JobSavedObjectService } from '../saved_objects';
 
 function recognize(
   client: IScopedClusterClient,
   mlClient: MlClient,
   savedObjectsClient: SavedObjectsClientContract,
+  jobSavedObjectService: JobSavedObjectService,
   request: KibanaRequest,
   indexPatternTitle: string
 ) {
-  const dr = new DataRecognizer(client, mlClient, savedObjectsClient, request);
+  const dr = new DataRecognizer(
+    client,
+    mlClient,
+    savedObjectsClient,
+    jobSavedObjectService,
+    request
+  );
   return dr.findMatches(indexPatternTitle);
 }
 
@@ -34,10 +42,17 @@ function getModule(
   client: IScopedClusterClient,
   mlClient: MlClient,
   savedObjectsClient: SavedObjectsClientContract,
+  jobSavedObjectService: JobSavedObjectService,
   request: KibanaRequest,
   moduleId: string
 ) {
-  const dr = new DataRecognizer(client, mlClient, savedObjectsClient, request);
+  const dr = new DataRecognizer(
+    client,
+    mlClient,
+    savedObjectsClient,
+    jobSavedObjectService,
+    request
+  );
   if (moduleId === undefined) {
     return dr.listModules();
   } else {
@@ -49,6 +64,7 @@ function setup(
   client: IScopedClusterClient,
   mlClient: MlClient,
   savedObjectsClient: SavedObjectsClientContract,
+  jobSavedObjectService: JobSavedObjectService,
   request: KibanaRequest,
   moduleId: string,
   prefix?: string,
@@ -61,9 +77,16 @@ function setup(
   end?: number,
   jobOverrides?: JobOverride | JobOverride[],
   datafeedOverrides?: DatafeedOverride | DatafeedOverride[],
-  estimateModelMemory?: boolean
+  estimateModelMemory?: boolean,
+  applyToAllSpaces?: boolean
 ) {
-  const dr = new DataRecognizer(client, mlClient, savedObjectsClient, request);
+  const dr = new DataRecognizer(
+    client,
+    mlClient,
+    savedObjectsClient,
+    jobSavedObjectService,
+    request
+  );
   return dr.setup(
     moduleId,
     prefix,
@@ -76,7 +99,8 @@ function setup(
     end,
     jobOverrides,
     datafeedOverrides,
-    estimateModelMemory
+    estimateModelMemory,
+    applyToAllSpaces
   );
 }
 
@@ -84,10 +108,17 @@ function dataRecognizerJobsExist(
   client: IScopedClusterClient,
   mlClient: MlClient,
   savedObjectsClient: SavedObjectsClientContract,
+  jobSavedObjectService: JobSavedObjectService,
   request: KibanaRequest,
   moduleId: string
 ) {
-  const dr = new DataRecognizer(client, mlClient, savedObjectsClient, request);
+  const dr = new DataRecognizer(
+    client,
+    mlClient,
+    savedObjectsClient,
+    jobSavedObjectService,
+    request
+  );
   return dr.dataRecognizerJobsExist(moduleId);
 }
 
@@ -132,22 +163,25 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) {
         tags: ['access:ml:canCreateJob'],
       },
     },
-    routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response, context }) => {
-      try {
-        const { indexPatternTitle } = request.params;
-        const results = await recognize(
-          client,
-          mlClient,
-          context.core.savedObjects.client,
-          request,
-          indexPatternTitle
-        );
+    routeGuard.fullLicenseAPIGuard(
+      async ({ client, mlClient, request, response, context, jobSavedObjectService }) => {
+        try {
+          const { indexPatternTitle } = request.params;
+          const results = await recognize(
+            client,
+            mlClient,
+            context.core.savedObjects.client,
+            jobSavedObjectService,
+            request,
+            indexPatternTitle
+          );
 
-        return response.ok({ body: results });
-      } catch (e) {
-        return response.customError(wrapError(e));
+          return response.ok({ body: results });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
       }
-    })
+    )
   );
 
   /**
@@ -268,27 +302,30 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) {
         tags: ['access:ml:canGetJobs'],
       },
     },
-    routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response, context }) => {
-      try {
-        let { moduleId } = request.params;
-        if (moduleId === '') {
-          // if the endpoint is called with a trailing /
-          // the moduleId will be an empty string.
-          moduleId = undefined;
-        }
-        const results = await getModule(
-          client,
-          mlClient,
-          context.core.savedObjects.client,
-          request,
-          moduleId
-        );
+    routeGuard.fullLicenseAPIGuard(
+      async ({ client, mlClient, request, response, context, jobSavedObjectService }) => {
+        try {
+          let { moduleId } = request.params;
+          if (moduleId === '') {
+            // if the endpoint is called with a trailing /
+            // the moduleId will be an empty string.
+            moduleId = undefined;
+          }
+          const results = await getModule(
+            client,
+            mlClient,
+            context.core.savedObjects.client,
+            jobSavedObjectService,
+            request,
+            moduleId
+          );
 
-        return response.ok({ body: results });
-      } catch (e) {
-        return response.customError(wrapError(e));
+          return response.ok({ body: results });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
       }
-    })
+    )
   );
 
   /**
@@ -442,49 +479,53 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) {
         tags: ['access:ml:canCreateJob'],
       },
     },
-    routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response, context }) => {
-      try {
-        const { moduleId } = request.params;
+    routeGuard.fullLicenseAPIGuard(
+      async ({ client, mlClient, request, response, context, jobSavedObjectService }) => {
+        try {
+          const { moduleId } = request.params;
 
-        const {
-          prefix,
-          groups,
-          indexPatternName,
-          query,
-          useDedicatedIndex,
-          startDatafeed,
-          start,
-          end,
-          jobOverrides,
-          datafeedOverrides,
-          estimateModelMemory,
-        } = request.body as TypeOf<typeof setupModuleBodySchema>;
+          const {
+            prefix,
+            groups,
+            indexPatternName,
+            query,
+            useDedicatedIndex,
+            startDatafeed,
+            start,
+            end,
+            jobOverrides,
+            datafeedOverrides,
+            estimateModelMemory,
+            applyToAllSpaces,
+          } = request.body as TypeOf<typeof setupModuleBodySchema>;
 
-        const result = await setup(
-          client,
-          mlClient,
+          const result = await setup(
+            client,
+            mlClient,
+            context.core.savedObjects.client,
+            jobSavedObjectService,
+            request,
+            moduleId,
+            prefix,
+            groups,
+            indexPatternName,
+            query,
+            useDedicatedIndex,
+            startDatafeed,
+            start,
+            end,
+            jobOverrides,
+            datafeedOverrides,
+            estimateModelMemory,
+            applyToAllSpaces
+          );
 
-          context.core.savedObjects.client,
-          request,
-          moduleId,
-          prefix,
-          groups,
-          indexPatternName,
-          query,
-          useDedicatedIndex,
-          startDatafeed,
-          start,
-          end,
-          jobOverrides,
-          datafeedOverrides,
-          estimateModelMemory
-        );
-
-        return response.ok({ body: result });
-      } catch (e) {
-        return response.customError(wrapError(e));
+          return response.ok({ body: result });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
       }
-    })
+    )
   );
 
   /**
@@ -549,22 +590,24 @@ export function dataRecognizer({ router, routeGuard }: RouteInitialization) {
         tags: ['access:ml:canGetJobs'],
       },
     },
-    routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response, context }) => {
-      try {
-        const { moduleId } = request.params;
-        const result = await dataRecognizerJobsExist(
-          client,
-          mlClient,
+    routeGuard.fullLicenseAPIGuard(
+      async ({ client, mlClient, request, response, context, jobSavedObjectService }) => {
+        try {
+          const { moduleId } = request.params;
+          const result = await dataRecognizerJobsExist(
+            client,
+            mlClient,
+            context.core.savedObjects.client,
+            jobSavedObjectService,
+            request,
+            moduleId
+          );
 
-          context.core.savedObjects.client,
-          request,
-          moduleId
-        );
-
-        return response.ok({ body: result });
-      } catch (e) {
-        return response.customError(wrapError(e));
+          return response.ok({ body: result });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
       }
-    })
+    )
   );
 }

x-pack/plugins/ml/server/routes/schemas/modules.ts

@@ -69,6 +69,11 @@ export const setupModuleBodySchema = schema.object({
    * should be made by checking the cardinality of fields in the job configurations (optional).
    */
   estimateModelMemory: schema.maybe(schema.boolean()),
+
+  /**
+   * Add each job created to the * space (optional)
+   */
+  applyToAllSpaces: schema.maybe(schema.boolean()),
 });
 
 export const optionalModuleIdParamSchema = schema.object({

x-pack/plugins/ml/server/saved_objects/authorization.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { KibanaRequest } from 'kibana/server';
+import type { SecurityPluginSetup } from '../../../security/server';
+
+export function authorizationProvider(authorization: SecurityPluginSetup['authz']) {
+  async function authorizationCheck(request: KibanaRequest) {
+    const checkPrivilegesWithRequest = authorization.checkPrivilegesWithRequest(request);
+    // Checking privileges "dynamically" will check against the current space, if spaces are enabled.
+    // If spaces are disabled, then this will check privileges globally instead.
+    // SO, if spaces are disabled, then you don't technically need to perform this check, but I included it here
+    // for completeness.
+    const checkPrivilegesDynamicallyWithRequest = authorization.checkPrivilegesDynamicallyWithRequest(
+      request
+    );
+    const createMLJobAuthorizationAction = authorization.actions.savedObject.get(
+      'ml-job',
+      'create'
+    );
+    const canCreateGlobally = (
+      await checkPrivilegesWithRequest.globally({
+        kibana: [createMLJobAuthorizationAction],
+      })
+    ).hasAllRequested;
+    const canCreateAtSpace = (
+      await checkPrivilegesDynamicallyWithRequest({ kibana: [createMLJobAuthorizationAction] })
+    ).hasAllRequested;
+    return {
+      canCreateGlobally,
+      canCreateAtSpace,
+    };
+  }
+
+  return { authorizationCheck };
+}

x-pack/plugins/ml/server/saved_objects/initialization/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { jobSavedObjectsInitializationFactory } from './initialization';

x-pack/plugins/ml/server/saved_objects/initialization/initialization.ts

@@ -5,11 +5,13 @@
  */
 
 import { IScopedClusterClient, CoreStart, SavedObjectsClientContract } from 'kibana/server';
-import { savedObjectClientsFactory } from './util';
-import { repairFactory } from './repair';
-import { jobSavedObjectServiceFactory, JobObject } from './service';
-import { mlLog } from '../lib/log';
-import { ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects';
+import { savedObjectClientsFactory } from '../util';
+import { repairFactory } from '../repair';
+import { jobSavedObjectServiceFactory, JobObject } from '../service';
+import { mlLog } from '../../lib/log';
+import { ML_SAVED_OBJECT_TYPE } from '../../../common/types/saved_objects';
+import { createJobSpaceOverrides } from './space_overrides';
+import type { SecurityPluginSetup } from '../../../../security/server';
 
 /**
  * Creates initializeJobs function which is used to check whether
@@ -17,7 +19,11 @@ import { ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects';
  *
  * @param core: CoreStart
  */
-export function jobSavedObjectsInitializationFactory(core: CoreStart, spacesEnabled: boolean) {
+export function jobSavedObjectsInitializationFactory(
+  core: CoreStart,
+  security: SecurityPluginSetup | undefined,
+  spacesEnabled: boolean
+) {
   const client = (core.elasticsearch.client as unknown) as IScopedClusterClient;
 
   /**
@@ -35,22 +41,26 @@ export function jobSavedObjectsInitializationFactory(core: CoreStart, spacesEnab
         return;
       }
 
-      const jobSavedObjectService = jobSavedObjectServiceFactory(
-        savedObjectsClient,
-        savedObjectsClient,
-        spacesEnabled,
-        () => Promise.resolve() // pretend isMlReady, to allow us to initialize the saved objects
-      );
-
       if ((await _needsInitializing(savedObjectsClient)) === false) {
         // ml job saved objects have already been initialized
         return;
       }
 
+      const jobSavedObjectService = jobSavedObjectServiceFactory(
+        savedObjectsClient,
+        savedObjectsClient,
+        spacesEnabled,
+        security?.authz,
+        () => Promise.resolve() // pretend isMlReady, to allow us to initialize the saved objects
+      );
+
       mlLog.info('Initializing job saved objects');
+      // create space overrides for specific jobs
+      const jobSpaceOverrides = await createJobSpaceOverrides(client);
+      // initialize jobs
       const { initSavedObjects } = repairFactory(client, jobSavedObjectService);
-      const { jobs } = await initSavedObjects();
-      mlLog.info(`${jobs.length} job saved objects initialized for * space`);
+      const { jobs } = await initSavedObjects(false, jobSpaceOverrides);
+      mlLog.info(`${jobs.length} job saved objects initialized`);
     } catch (error) {
       mlLog.error(`Error Initializing jobs ${JSON.stringify(error)}`);
     }

x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { createJobSpaceOverrides } from './space_overrides';

x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/logs.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import RE2 from 're2';
+import { mlLog } from '../../../lib/log';
+import { Job } from '../../../../common/types/anomaly_detection_jobs';
+
+const GROUP = 'logs-ui';
+const MODULE_PREFIX = 'kibana-logs-ui';
+const SOURCES = ['default', 'internal-stack-monitoring'];
+const JOB_IDS = ['log-entry-rate', 'log-entry-categories-count'];
+
+// jobs created by the logs plugin will be in the logs-ui group
+// they contain the a space name in the job id, and so the id can be parsed
+// and the job assigned to the correct space.
+export async function logJobsSpaces({
+  asInternalUser,
+}: IScopedClusterClient): Promise<Array<{ id: string; space: string }>> {
+  try {
+    const { body } = await asInternalUser.ml.getJobs<{ jobs: Job[] }>({
+      job_id: GROUP,
+    });
+    if (body.jobs.length === 0) {
+      return [];
+    }
+
+    const findLogJobSpace = findLogJobSpaceFactory();
+    return body.jobs
+      .map((j) => ({ id: j.job_id, space: findLogJobSpace(j.job_id) }))
+      .filter((j) => j.space !== null) as Array<{ id: string; space: string }>;
+  } catch ({ body }) {
+    if (body.status !== 404) {
+      // 404s are expected if there are no logs-ui jobs
+      mlLog.error(`Error Initializing Logs job ${JSON.stringify(body)}`);
+    }
+  }
+  return [];
+}
+
+function findLogJobSpaceFactory() {
+  const reg = new RE2(`${MODULE_PREFIX}-(.+)-(${SOURCES.join('|')})-(${JOB_IDS.join('|')})`);
+
+  return (jobId: string) => {
+    const result = reg.exec(jobId);
+    if (result === null) {
+      return null;
+    }
+    return result[1] ?? null;
+  };
+}

x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/metrics.ts

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import RE2 from 're2';
+import { mlLog } from '../../../lib/log';
+import { Job } from '../../../../common/types/anomaly_detection_jobs';
+
+const GROUP = 'metrics';
+const MODULE_PREFIX = 'kibana-metrics-ui';
+const SOURCES = ['default', 'internal-stack-monitoring'];
+const JOB_IDS = [
+  'k8s_memory_usage',
+  'k8s_network_in',
+  'k8s_network_out',
+  'hosts_memory_usage',
+  'hosts_network_in',
+  'hosts_network_out',
+];
+
+// jobs created by the logs plugin will be in the metrics group
+// they contain the a space name in the job id, and so the id can be parsed
+// and the job assigned to the correct space.
+export async function metricsJobsSpaces({
+  asInternalUser,
+}: IScopedClusterClient): Promise<Array<{ id: string; space: string }>> {
+  try {
+    const { body } = await asInternalUser.ml.getJobs<{ jobs: Job[] }>({
+      job_id: GROUP,
+    });
+    if (body.jobs.length === 0) {
+      return [];
+    }
+
+    const findMetricJobSpace = findMetricsJobSpaceFactory();
+    return body.jobs
+      .map((j) => ({ id: j.job_id, space: findMetricJobSpace(j.job_id) }))
+      .filter((j) => j.space !== null) as Array<{ id: string; space: string }>;
+  } catch ({ body }) {
+    if (body.status !== 404) {
+      // 404s are expected if there are no metrics jobs
+      mlLog.error(`Error Initializing Metrics job ${JSON.stringify(body)}`);
+    }
+  }
+  return [];
+}
+
+function findMetricsJobSpaceFactory() {
+  const reg = new RE2(`${MODULE_PREFIX}-(.+)-(${SOURCES.join('|')})-(${JOB_IDS.join('|')})`);
+
+  return (jobId: string) => {
+    const result = reg.exec(jobId);
+    if (result === null) {
+      return null;
+    }
+    return result[1] ?? null;
+  };
+}

x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/space_overrides.test.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import { createJobSpaceOverrides } from './space_overrides';
+
+const jobs = [
+  {
+    job_id: 'kibana-logs-ui-default-default-log-entry-rate',
+  },
+  {
+    job_id: 'kibana-logs-ui-other_space-default-log-entry-rate',
+  },
+  {
+    job_id: 'kibana-logs-ui-other_space-default-log-entry-categories-count',
+  },
+  {
+    job_id: 'kibana-logs-ui-other_space-internal-stack-monitoring-log-entry-rate',
+  },
+  {
+    job_id: 'kibana-logs-ui-other_space-dinosaur-log-entry-rate', // shouldn't match
+  },
+  {
+    job_id: 'kibana-logs-ui-other_space-default-dinosaur', // shouldn't match
+  },
+  {
+    job_id: 'kibana-metrics-ui-default-default-k8s_memory_usage',
+  },
+  {
+    job_id: 'kibana-metrics-ui-other_space-default-hosts_network_in',
+  },
+];
+
+const result = {
+  overrides: {
+    'anomaly-detector': {
+      'kibana-logs-ui-default-default-log-entry-rate': ['default'],
+      'kibana-logs-ui-other_space-default-log-entry-rate': ['other_space'],
+      'kibana-logs-ui-other_space-default-log-entry-categories-count': ['other_space'],
+      'kibana-logs-ui-other_space-internal-stack-monitoring-log-entry-rate': ['other_space'],
+      'kibana-metrics-ui-default-default-k8s_memory_usage': ['default'],
+      'kibana-metrics-ui-other_space-default-hosts_network_in': ['other_space'],
+    },
+    'data-frame-analytics': {},
+  },
+};
+
+const callAs = {
+  ml: {
+    getJobs: jest.fn(() =>
+      Promise.resolve({
+        body: { jobs },
+      })
+    ),
+  },
+};
+
+const mlClusterClient = ({
+  asInternalUser: callAs,
+} as unknown) as IScopedClusterClient;
+
+describe('ML - job initialization', () => {
+  describe('createJobSpaceOverrides', () => {
+    it('should apply job overrides correctly', async () => {
+      const overrides = await createJobSpaceOverrides(mlClusterClient);
+      expect(overrides).toEqual(result);
+    });
+  });
+});

x-pack/plugins/ml/server/saved_objects/initialization/space_overrides/space_overrides.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import type { JobSpaceOverrides } from '../../repair';
+import { logJobsSpaces } from './logs';
+import { metricsJobsSpaces } from './metrics';
+
+// create a list of jobs and specific spaces to place them in
+// when the are being initialized.
+export async function createJobSpaceOverrides(
+  clusterClient: IScopedClusterClient
+): Promise<JobSpaceOverrides> {
+  const spaceOverrides: JobSpaceOverrides = {
+    overrides: {
+      'anomaly-detector': {},
+      'data-frame-analytics': {},
+    },
+  };
+  (await logJobsSpaces(clusterClient)).forEach(
+    (o) => (spaceOverrides.overrides['anomaly-detector'][o.id] = [o.space])
+  );
+  (await metricsJobsSpaces(clusterClient)).forEach(
+    (o) => (spaceOverrides.overrides['anomaly-detector'][o.id] = [o.space])
+  );
+  return spaceOverrides;
+}

x-pack/plugins/ml/server/saved_objects/repair.ts

@@ -7,11 +7,17 @@
 import Boom from '@hapi/boom';
 import { IScopedClusterClient } from 'kibana/server';
 import type { JobObject, JobSavedObjectService } from './service';
-import { JobType } from '../../common/types/saved_objects';
+import { JobType, RepairSavedObjectResponse } from '../../common/types/saved_objects';
 import { checksFactory } from './checks';
 
 import { Datafeed } from '../../common/types/anomaly_detection_jobs';
 
+export interface JobSpaceOverrides {
+  overrides: {
+    [type in JobType]: { [jobId: string]: string[] };
+  };
+}
+
 export function repairFactory(
   client: IScopedClusterClient,
   jobSavedObjectService: JobSavedObjectService
@@ -19,13 +25,7 @@ export function repairFactory(
   const { checkStatus } = checksFactory(client, jobSavedObjectService);
 
   async function repairJobs(simulate: boolean = false) {
-    type Result = Record<string, { success: boolean; error?: any }>;
-    const results: {
-      savedObjectsCreated: Result;
-      savedObjectsDeleted: Result;
-      datafeedsAdded: Result;
-      datafeedsRemoved: Result;
-    } = {
+    const results: RepairSavedObjectResponse = {
       savedObjectsCreated: {},
       savedObjectsDeleted: {},
       datafeedsAdded: {},
@@ -173,14 +173,14 @@ export function repairFactory(
     return results;
   }
 
-  async function initSavedObjects(simulate: boolean = false, namespaces: string[] = ['*']) {
+  async function initSavedObjects(simulate: boolean = false, spaceOverrides?: JobSpaceOverrides) {
     const results: { jobs: Array<{ id: string; type: string }>; success: boolean; error?: any } = {
       jobs: [],
       success: true,
     };
     const status = await checkStatus();
 
-    const jobs: JobObject[] = [];
+    const jobs: Array<{ job: JobObject; namespaces: string[] }> = [];
     const types: JobType[] = ['anomaly-detector', 'data-frame-analytics'];
 
     types.forEach((type) => {
@@ -190,22 +190,28 @@ export function repairFactory(
             results.jobs.push({ id: job.jobId, type });
           } else {
             jobs.push({
-              job_id: job.jobId,
-              datafeed_id: job.datafeedId ?? null,
-              type,
+              job: {
+                job_id: job.jobId,
+                datafeed_id: job.datafeedId ?? null,
+                type,
+              },
+              // allow some jobs to be assigned to specific spaces when initializing
+              namespaces: spaceOverrides?.overrides[type][job.jobId] ?? ['*'],
             });
           }
         }
       });
     });
+
     try {
-      const createResults = await jobSavedObjectService.bulkCreateJobs(jobs, namespaces);
+      const createResults = await jobSavedObjectService.bulkCreateJobs(jobs);
       createResults.saved_objects.forEach(({ attributes }) => {
         results.jobs.push({
           id: attributes.job_id,
           type: attributes.type,
         });
       });
+      return { jobs: jobs.map((j) => j.job.job_id) };
     } catch (error) {
       results.success = false;
       results.error = Boom.boomify(error).output;

x-pack/plugins/ml/server/saved_objects/service.ts

@@ -5,9 +5,11 @@
  */
 
 import RE2 from 're2';
-import { SavedObjectsClientContract, SavedObjectsFindOptions } from 'kibana/server';
+import { KibanaRequest, SavedObjectsClientContract, SavedObjectsFindOptions } from 'kibana/server';
+import type { SecurityPluginSetup } from '../../../security/server';
 import { JobType, ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects';
 import { MLJobNotFound } from '../lib/ml_client';
+import { authorizationProvider } from './authorization';
 
 export interface JobObject {
   job_id: string;
@@ -22,6 +24,7 @@ export function jobSavedObjectServiceFactory(
   savedObjectsClient: SavedObjectsClientContract,
   internalSavedObjectsClient: SavedObjectsClientContract,
   spacesEnabled: boolean,
+  authorization: SecurityPluginSetup['authz'] | undefined,
   isMlReady: () => Promise<void>
 ) {
   async function _getJobObjects(
@@ -58,29 +61,33 @@ export function jobSavedObjectServiceFactory(
 
   async function _createJob(jobType: JobType, jobId: string, datafeedId?: string) {
     await isMlReady();
-    await savedObjectsClient.create<JobObject>(
-      ML_SAVED_OBJECT_TYPE,
-      {
-        job_id: jobId,
-        datafeed_id: datafeedId ?? null,
-        type: jobType,
-      },
-      { id: jobId, overwrite: true }
-    );
+    const job: JobObject = {
+      job_id: jobId,
+      datafeed_id: datafeedId ?? null,
+      type: jobType,
+    };
+    await savedObjectsClient.create<JobObject>(ML_SAVED_OBJECT_TYPE, job, {
+      id: savedObjectId(job),
+      overwrite: true,
+    });
   }
 
-  async function _bulkCreateJobs(jobs: JobObject[], namespaces?: string[]) {
+  async function _bulkCreateJobs(jobs: Array<{ job: JobObject; namespaces: string[] }>) {
     await isMlReady();
     return await savedObjectsClient.bulkCreate<JobObject>(
       jobs.map((j) => ({
         type: ML_SAVED_OBJECT_TYPE,
-        id: j.job_id,
-        attributes: j,
-        initialNamespaces: namespaces,
+        id: savedObjectId(j.job),
+        attributes: j.job,
+        initialNamespaces: j.namespaces,
       }))
     );
   }
 
+  function savedObjectId(job: JobObject) {
+    return `${job.type}-${job.job_id}`;
+  }
+
   async function _deleteJob(jobType: JobType, jobId: string) {
     const jobs = await _getJobObjects(jobType, jobId);
     const job = jobs[0];
@@ -107,8 +114,8 @@ export function jobSavedObjectServiceFactory(
     await _deleteJob('data-frame-analytics', jobId);
   }
 
-  async function bulkCreateJobs(jobs: JobObject[], namespaces?: string[]) {
-    return await _bulkCreateJobs(jobs, namespaces);
+  async function bulkCreateJobs(jobs: Array<{ job: JobObject; namespaces: string[] }>) {
+    return await _bulkCreateJobs(jobs);
   }
 
   async function getAllJobObjects(jobType?: JobType, currentSpaceOnly: boolean = true) {
@@ -279,6 +286,14 @@ export function jobSavedObjectServiceFactory(
     return results;
   }
 
+  async function canCreateGlobalJobs(request: KibanaRequest) {
+    if (authorization === undefined) {
+      return true;
+    }
+    const { authorizationCheck } = authorizationProvider(authorization);
+    return (await authorizationCheck(request)).canCreateGlobally;
+  }
+
   return {
     getAllJobObjects,
     createAnomalyDetectionJob,
@@ -295,6 +310,7 @@ export function jobSavedObjectServiceFactory(
     removeJobsFromSpaces,
     bulkCreateJobs,
     getAllJobObjectsForAllSpaces,
+    canCreateGlobalJobs,
   };
 }
 

x-pack/plugins/ml/server/shared_services/providers/modules.ts

@@ -10,6 +10,7 @@ import { DataRecognizer } from '../../models/data_recognizer';
 import { GetGuards } from '../shared_services';
 import { moduleIdParamSchema, setupModuleBodySchema } from '../../routes/schemas/modules';
 import { MlClient } from '../../lib/ml_client';
+import { JobSavedObjectService } from '../../saved_objects';
 
 export type ModuleSetupPayload = TypeOf<typeof moduleIdParamSchema> &
   TypeOf<typeof setupModuleBodySchema>;
@@ -34,8 +35,14 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider {
           return await getGuards(request, savedObjectsClient)
             .isFullLicense()
             .hasMlCapabilities(['canGetJobs'])
-            .ok(async ({ scopedClient, mlClient }) => {
-              const dr = dataRecognizerFactory(scopedClient, mlClient, savedObjectsClient, request);
+            .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => {
+              const dr = dataRecognizerFactory(
+                scopedClient,
+                mlClient,
+                savedObjectsClient,
+                jobSavedObjectService,
+                request
+              );
               return dr.findMatches(...args);
             });
         },
@@ -43,8 +50,14 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider {
           return await getGuards(request, savedObjectsClient)
             .isFullLicense()
             .hasMlCapabilities(['canGetJobs'])
-            .ok(async ({ scopedClient, mlClient }) => {
-              const dr = dataRecognizerFactory(scopedClient, mlClient, savedObjectsClient, request);
+            .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => {
+              const dr = dataRecognizerFactory(
+                scopedClient,
+                mlClient,
+                savedObjectsClient,
+                jobSavedObjectService,
+                request
+              );
               return dr.getModule(moduleId);
             });
         },
@@ -52,8 +65,14 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider {
           return await getGuards(request, savedObjectsClient)
             .isFullLicense()
             .hasMlCapabilities(['canGetJobs'])
-            .ok(async ({ scopedClient, mlClient }) => {
-              const dr = dataRecognizerFactory(scopedClient, mlClient, savedObjectsClient, request);
+            .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => {
+              const dr = dataRecognizerFactory(
+                scopedClient,
+                mlClient,
+                savedObjectsClient,
+                jobSavedObjectService,
+                request
+              );
               return dr.listModules();
             });
         },
@@ -61,8 +80,14 @@ export function getModulesProvider(getGuards: GetGuards): ModulesProvider {
           return await getGuards(request, savedObjectsClient)
             .isFullLicense()
             .hasMlCapabilities(['canCreateJob'])
-            .ok(async ({ scopedClient, mlClient }) => {
-              const dr = dataRecognizerFactory(scopedClient, mlClient, savedObjectsClient, request);
+            .ok(async ({ scopedClient, mlClient, jobSavedObjectService }) => {
+              const dr = dataRecognizerFactory(
+                scopedClient,
+                mlClient,
+                savedObjectsClient,
+                jobSavedObjectService,
+                request
+              );
               return dr.setup(
                 payload.moduleId,
                 payload.prefix,
@@ -88,7 +113,8 @@ function dataRecognizerFactory(
   client: IScopedClusterClient,
   mlClient: MlClient,
   savedObjectsClient: SavedObjectsClientContract,
+  jobSavedObjectService: JobSavedObjectService,
   request: KibanaRequest
 ) {
-  return new DataRecognizer(client, mlClient, savedObjectsClient, request);
+  return new DataRecognizer(client, mlClient, savedObjectsClient, jobSavedObjectService, request);
 }

x-pack/plugins/ml/server/shared_services/shared_services.ts

@@ -12,7 +12,8 @@ import { SpacesPluginSetup } from '../../../spaces/server';
 import { KibanaRequest } from '../../.././../../src/core/server/http';
 import { MlLicense } from '../../common/license';
 
-import { CloudSetup } from '../../../cloud/server';
+import type { CloudSetup } from '../../../cloud/server';
+import type { SecurityPluginSetup } from '../../../security/server';
 import { licenseChecks } from './license_checks';
 import { MlSystemProvider, getMlSystemProvider } from './providers/system';
 import { JobServiceProvider, getJobServiceProvider } from './providers/job_service';
@@ -26,7 +27,7 @@ import { ResolveMlCapabilities, MlCapabilitiesKey } from '../../common/types/cap
 import { hasMlCapabilitiesProvider, HasMlCapabilities } from '../lib/capabilities';
 import { MLClusterClientUninitialized } from './errors';
 import { MlClient, getMlClient } from '../lib/ml_client';
-import { jobSavedObjectServiceFactory } from '../saved_objects';
+import { jobSavedObjectServiceFactory, JobSavedObjectService } from '../saved_objects';
 
 export type SharedServices = JobServiceProvider &
   AnomalyDetectorsProvider &
@@ -53,6 +54,7 @@ export interface SharedServicesChecks {
 interface OkParams {
   scopedClient: IScopedClusterClient;
   mlClient: MlClient;
+  jobSavedObjectService: JobSavedObjectService;
 }
 
 type OkCallback = (okParams: OkParams) => any;
@@ -61,6 +63,7 @@ export function createSharedServices(
   mlLicense: MlLicense,
   spacesPlugin: SpacesPluginSetup | undefined,
   cloud: CloudSetup,
+  authorization: SecurityPluginSetup['authz'] | undefined,
   resolveMlCapabilities: ResolveMlCapabilities,
   getClusterClient: () => IClusterClient | null,
   getInternalSavedObjectsClient: () => SavedObjectsClientContract | null,
@@ -80,11 +83,14 @@ export function createSharedServices(
       getClusterClient,
       savedObjectsClient,
       internalSavedObjectsClient,
+      authorization,
       spacesPlugin !== undefined,
       isMlReady
     );
 
-    const { hasMlCapabilities, scopedClient, mlClient } = getRequestItems(request);
+    const { hasMlCapabilities, scopedClient, mlClient, jobSavedObjectService } = getRequestItems(
+      request
+    );
     const asyncGuards: Array<Promise<void>> = [];
 
     const guards: Guards = {
@@ -102,7 +108,7 @@ export function createSharedServices(
       },
       async ok(callback: OkCallback) {
         await Promise.all(asyncGuards);
-        return callback({ scopedClient, mlClient });
+        return callback({ scopedClient, mlClient, jobSavedObjectService });
       },
     };
     return guards;
@@ -122,6 +128,7 @@ function getRequestItemsProvider(
   getClusterClient: () => IClusterClient | null,
   savedObjectsClient: SavedObjectsClientContract,
   internalSavedObjectsClient: SavedObjectsClientContract,
+  authorization: SecurityPluginSetup['authz'] | undefined,
   spaceEnabled: boolean,
   isMlReady: () => Promise<void>
 ) {
@@ -138,6 +145,7 @@ function getRequestItemsProvider(
       savedObjectsClient,
       internalSavedObjectsClient,
       spaceEnabled,
+      authorization,
       isMlReady
     );
 
@@ -158,6 +166,6 @@ function getRequestItemsProvider(
       };
       mlClient = getMlClient(scopedClient, jobSavedObjectService);
     }
-    return { hasMlCapabilities, scopedClient, mlClient };
+    return { hasMlCapabilities, scopedClient, mlClient, jobSavedObjectService };
   };
 }

x-pack/plugins/security_solution/public/common/components/ml_popover/api.ts

@@ -83,6 +83,7 @@ export const setupMlJob = async ({
         indexPatternName,
         startDatafeed: false,
         useDedicatedIndex: true,
+        applyToAllSpaces: true,
       }),
       asSystemRequest: true,
     }

x-pack/plugins/uptime/public/state/api/ml_anomaly.ts

@@ -41,6 +41,7 @@ export const createMLJob = async ({
     startDatafeed: true,
     start: moment().subtract(2, 'w').valueOf(),
     indexPatternName: heartbeatIndices,
+    applyToAllSpaces: true,
     query: {
       bool: {
         filter: [