[ML] Api tests for ml/modules/jobs_exist/{moduleId} (#142378) (#142446)

* [ML] Api tests for ml/modules/jobs_exist/{moduleId}

* fixing title

* fixing delete index pattern function

(cherry picked from commit 656a48fee9)

Co-authored-by: James Gowdy <jgowdy@elastic.co>

x-pack/test/api_integration/apis/ml/modules/index.ts

@@ -38,5 +38,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./get_module'));
     loadTestFile(require.resolve('./recognize_module'));
     loadTestFile(require.resolve('./setup_module'));
+    loadTestFile(require.resolve('./jobs_exist'));
   });
 }

x-pack/test/api_integration/apis/ml/modules/jobs_exist.ts

@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+import { USER } from '../../../../functional/services/ml/security_common';
+import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api';
+
+export default ({ getService }: FtrProviderContext) => {
+  const esArchiver = getService('esArchiver');
+  const supertest = getService('supertestWithoutAuth');
+  const ml = getService('ml');
+
+  const idSpace1 = 'space1';
+  const sourceDataArchive = 'x-pack/test/functional/es_archives/ml/module_sample_logs';
+  const moduleInfo = {
+    moduleId: 'sample_data_weblogs',
+    jobIds: ['low_request_rate', 'response_code_rates', 'url_scanning'],
+    dataView: { name: 'ft_module_sample_logs', timeField: '@timestamp' },
+  };
+
+  async function runRequest(moduleId: string, expectedStatusCode: number, user: USER) {
+    const { body, status } = await supertest
+      .get(`/api/ml/modules/jobs_exist/${moduleId}`)
+      .auth(user, ml.securityCommon.getPasswordForUser(user))
+      .set(COMMON_REQUEST_HEADERS);
+
+    ml.api.assertResponseStatusCode(expectedStatusCode, status, body);
+    return body;
+  }
+
+  describe('GET ml/modules/jobs_exist/{moduleId}', function () {
+    before(async () => {
+      await ml.testResources.setKibanaTimeZoneToUTC();
+      await esArchiver.loadIfNeeded(sourceDataArchive);
+      // create data view in default space
+      await ml.testResources.createIndexPatternIfNeeded(
+        moduleInfo.dataView.name,
+        moduleInfo.dataView.timeField
+      );
+      // create data view in idSpace1
+      await ml.testResources.createIndexPatternIfNeeded(
+        moduleInfo.dataView.name,
+        moduleInfo.dataView.timeField,
+        idSpace1
+      );
+    });
+
+    afterEach(async () => {
+      await ml.api.cleanMlIndices();
+    });
+
+    after(async () => {
+      // delete all data views in all spaces
+      await ml.testResources.deleteIndexPatternByTitle(moduleInfo.dataView.name);
+      await ml.testResources.deleteIndexPatternByTitle(moduleInfo.dataView.name, idSpace1);
+    });
+
+    it('should find jobs installed by module without prefix', async () => {
+      const prefix = '';
+      await ml.api.setupModule(moduleInfo.moduleId, {
+        prefix,
+        indexPatternName: moduleInfo.dataView.name,
+        startDatafeed: false,
+        estimateModelMemory: false,
+      });
+      const { jobsExist, jobs } = await runRequest(moduleInfo.moduleId, 200, USER.ML_POWERUSER);
+
+      const expectedJobIds = moduleInfo.jobIds.map((j) => ({ id: `${prefix}${j}` }));
+      expect(jobsExist).to.eql(true, 'Expected jobsExist to be true');
+      expect(jobs).to.eql(expectedJobIds, `Expected jobs to be ${expectedJobIds}`);
+    });
+
+    it('should find jobs installed by module with prefix', async () => {
+      const prefix = 'pf1_';
+      await ml.api.setupModule(moduleInfo.moduleId, {
+        prefix,
+        indexPatternName: moduleInfo.dataView.name,
+        startDatafeed: false,
+        estimateModelMemory: false,
+      });
+      const { jobsExist, jobs } = await runRequest(moduleInfo.moduleId, 200, USER.ML_POWERUSER);
+
+      const expectedJobIds = moduleInfo.jobIds.map((j) => ({ id: `${prefix}${j}` }));
+      expect(jobsExist).to.eql(true, 'Expected jobsExist to be true');
+      expect(jobs).to.eql(expectedJobIds, `Expected jobs to be ${expectedJobIds}`);
+    });
+
+    it('should not find jobs installed into a different space', async () => {
+      const prefix = 'pf1_';
+      await ml.api.setupModule(
+        moduleInfo.moduleId,
+        {
+          prefix,
+          indexPatternName: moduleInfo.dataView.name,
+          startDatafeed: false,
+          estimateModelMemory: false,
+        },
+        idSpace1
+      );
+      const { jobsExist, jobs } = await runRequest(moduleInfo.moduleId, 200, USER.ML_POWERUSER);
+
+      expect(jobsExist).to.eql(false, 'Expected jobsExist to be false');
+      expect(jobs).to.eql(undefined, `Expected jobs to be undefined`);
+    });
+
+    it("should not find jobs for module which hasn't been installed", async () => {
+      const { jobsExist, jobs } = await runRequest('apache_ecs', 200, USER.ML_POWERUSER);
+
+      expect(jobsExist).to.eql(false, 'Expected jobsExist to be false');
+      expect(jobs).to.eql(undefined, `Expected jobs to be undefined`);
+    });
+  });
+};

x-pack/test/functional/services/ml/api.ts

@@ -8,6 +8,7 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import expect from '@kbn/expect';
 import { ProvidedType } from '@kbn/test';
+import type { TypeOf } from '@kbn/config-schema';
 import fs from 'fs';
 import { Calendar } from '@kbn/ml-plugin/server/models/calendar';
 import { Annotation } from '@kbn/ml-plugin/common/types/annotations';
@@ -17,6 +18,7 @@ import { DataFrameTaskStateType } from '@kbn/ml-plugin/common/types/data_frame_a
 import { DATA_FRAME_TASK_STATE } from '@kbn/ml-plugin/common/constants/data_frame_analytics';
 import { Datafeed, Job } from '@kbn/ml-plugin/common/types/anomaly_detection_jobs';
 import { JobType } from '@kbn/ml-plugin/common/types/saved_objects';
+import { setupModuleBodySchema } from '@kbn/ml-plugin/server/routes/schemas/modules';
 import {
   ML_ANNOTATIONS_INDEX_ALIAS_READ,
   ML_ANNOTATIONS_INDEX_ALIAS_WRITE,
@@ -1445,5 +1447,21 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) {
 
       log.debug('> Ingest pipeline deleted');
     },
+
+    async setupModule(
+      moduleId: string,
+      body: TypeOf<typeof setupModuleBodySchema>,
+      space?: string
+    ) {
+      log.debug(`Setting up module with ID: "${moduleId}"`);
+      const { body: module, status } = await kbnSupertest
+        .post(`${space ? `/s/${space}` : ''}/api/ml/modules/setup/${moduleId}`)
+        .set(COMMON_REQUEST_HEADERS)
+        .send(body);
+      this.assertResponseStatusCode(200, status, module);
+
+      log.debug('Module set up');
+      return module;
+    },
   };
 }

x-pack/test/functional/services/ml/test_resources.ts

@@ -54,15 +54,25 @@ export function MachineLearningTestResourcesProvider(
       await kibanaServer.uiSettings.unset('hideAnnouncements');
     },
 
-    async savedObjectExistsById(id: string, objectType: SavedObjectType): Promise<boolean> {
-      const response = await supertest.get(`/api/saved_objects/${objectType}/${id}`);
+    async savedObjectExistsById(
+      id: string,
+      objectType: SavedObjectType,
+      space?: string
+    ): Promise<boolean> {
+      const response = await supertest.get(
+        `${space ? `/s/${space}` : ''}/api/saved_objects/${objectType}/${id}`
+      );
       return response.status === 200;
     },
 
-    async savedObjectExistsByTitle(title: string, objectType: SavedObjectType): Promise<boolean> {
-      const id = await this.getSavedObjectIdByTitle(title, objectType);
+    async savedObjectExistsByTitle(
+      title: string,
+      objectType: SavedObjectType,
+      space?: string
+    ): Promise<boolean> {
+      const id = await this.getSavedObjectIdByTitle(title, objectType, space);
       if (id) {
-        return await this.savedObjectExistsById(id, objectType);
+        return await this.savedObjectExistsById(id, objectType, space);
       } else {
         return false;
       }
@@ -70,11 +80,14 @@ export function MachineLearningTestResourcesProvider(
 
     async getSavedObjectIdByTitle(
       title: string,
-      objectType: SavedObjectType
+      objectType: SavedObjectType,
+      space?: string
     ): Promise<string | undefined> {
       log.debug(`Searching for '${objectType}' with title '${title}'...`);
       const { body: findResponse, status } = await supertest
-        .get(`/api/saved_objects/_find?type=${objectType}&per_page=10000`)
+        .get(
+          `${space ? `/s/${space}` : ''}/api/saved_objects/_find?type=${objectType}&per_page=10000`
+        )
         .set(COMMON_REQUEST_HEADERS);
       mlApi.assertResponseStatusCode(200, status, findResponse);
 
@@ -104,8 +117,8 @@ export function MachineLearningTestResourcesProvider(
       return savedObjectIds;
     },
 
-    async getIndexPatternId(title: string): Promise<string | undefined> {
-      return this.getSavedObjectIdByTitle(title, SavedObjectType.INDEX_PATTERN);
+    async getIndexPatternId(title: string, space?: string): Promise<string | undefined> {
+      return this.getSavedObjectIdByTitle(title, SavedObjectType.INDEX_PATTERN, space);
     },
 
     async getSavedSearchId(title: string): Promise<string | undefined> {
@@ -120,7 +133,11 @@ export function MachineLearningTestResourcesProvider(
       return this.getSavedObjectIdByTitle(title, SavedObjectType.DASHBOARD);
     },
 
-    async createIndexPattern(title: string, timeFieldName?: string): Promise<string> {
+    async createIndexPattern(
+      title: string,
+      timeFieldName?: string,
+      space?: string
+    ): Promise<string> {
       log.debug(
         `Creating index pattern with title '${title}'${
           timeFieldName !== undefined ? ` and time field '${timeFieldName}'` : ''
@@ -128,12 +145,12 @@ export function MachineLearningTestResourcesProvider(
       );
 
       const { body: createResponse, status } = await supertest
-        .post(`/api/saved_objects/${SavedObjectType.INDEX_PATTERN}`)
+        .post(`${space ? `/s/${space}` : ''}/api/saved_objects/${SavedObjectType.INDEX_PATTERN}`)
         .set(COMMON_REQUEST_HEADERS)
         .send({ attributes: { title, timeFieldName } });
       mlApi.assertResponseStatusCode(200, status, createResponse);
 
-      await this.assertIndexPatternExistByTitle(title);
+      await this.assertIndexPatternExistByTitle(title, space);
 
       log.debug(` > Created with id '${createResponse.id}'`);
       return createResponse.id;
@@ -152,13 +169,17 @@ export function MachineLearningTestResourcesProvider(
       return createResponse;
     },
 
-    async createIndexPatternIfNeeded(title: string, timeFieldName?: string): Promise<string> {
-      const indexPatternId = await this.getIndexPatternId(title);
+    async createIndexPatternIfNeeded(
+      title: string,
+      timeFieldName?: string,
+      space?: string
+    ): Promise<string> {
+      const indexPatternId = await this.getIndexPatternId(title, space);
       if (indexPatternId !== undefined) {
         log.debug(`Index pattern with title '${title}' already exists. Nothing to create.`);
         return indexPatternId;
       } else {
-        return await this.createIndexPattern(title, timeFieldName);
+        return await this.createIndexPattern(title, timeFieldName, space);
       }
     },
 
@@ -301,7 +322,12 @@ export function MachineLearningTestResourcesProvider(
       );
     },
 
-    async deleteSavedObjectById(id: string, objectType: SavedObjectType, force: boolean = false) {
+    async deleteSavedObjectById(
+      id: string,
+      objectType: SavedObjectType,
+      force: boolean = false,
+      space?: string
+    ) {
       log.debug(`Deleting ${objectType} with id '${id}'...`);
 
       if ((await this.savedObjectExistsById(id, objectType)) === false) {
@@ -309,31 +335,31 @@ export function MachineLearningTestResourcesProvider(
         return;
       } else {
         const { body, status } = await supertest
-          .delete(`/api/saved_objects/${objectType}/${id}`)
+          .delete(`${space ? `/s/${space}` : ''}/api/saved_objects/${objectType}/${id}`)
           .set(COMMON_REQUEST_HEADERS)
           .query({ force });
         mlApi.assertResponseStatusCode(200, status, body);
 
-        await this.assertSavedObjectNotExistsById(id, objectType);
+        await this.assertSavedObjectNotExistsById(id, objectType, space);
 
         log.debug(` > Deleted ${objectType} with id '${id}'`);
       }
     },
 
-    async deleteIndexPatternByTitle(title: string) {
+    async deleteIndexPatternByTitle(title: string, space?: string) {
       log.debug(`Deleting index pattern with title '${title}'...`);
 
-      const indexPatternId = await this.getIndexPatternId(title);
+      const indexPatternId = await this.getIndexPatternId(title, space);
       if (indexPatternId === undefined) {
         log.debug(`Index pattern with title '${title}' does not exists. Nothing to delete.`);
         return;
       } else {
-        await this.deleteIndexPatternById(indexPatternId);
+        await this.deleteIndexPatternById(indexPatternId, space);
       }
     },
 
-    async deleteIndexPatternById(id: string) {
-      await this.deleteSavedObjectById(id, SavedObjectType.INDEX_PATTERN);
+    async deleteIndexPatternById(id: string, space?: string) {
+      await this.deleteSavedObjectById(id, SavedObjectType.INDEX_PATTERN, false, space);
     },
 
     async deleteSavedSearchByTitle(title: string) {
@@ -396,12 +422,16 @@ export function MachineLearningTestResourcesProvider(
       }
     },
 
-    async assertSavedObjectExistsByTitle(title: string, objectType: SavedObjectType) {
+    async assertSavedObjectExistsByTitle(
+      title: string,
+      objectType: SavedObjectType,
+      space?: string
+    ) {
       await retry.waitForWithTimeout(
         `${objectType} with title '${title}' to exist`,
         5 * 1000,
         async () => {
-          if ((await this.savedObjectExistsByTitle(title, objectType)) === true) {
+          if ((await this.savedObjectExistsByTitle(title, objectType, space)) === true) {
             return true;
           } else {
             throw new Error(`${objectType} with title '${title}' should exist.`);
@@ -438,12 +468,12 @@ export function MachineLearningTestResourcesProvider(
       );
     },
 
-    async assertSavedObjectNotExistsById(id: string, objectType: SavedObjectType) {
+    async assertSavedObjectNotExistsById(id: string, objectType: SavedObjectType, space?: string) {
       await retry.waitForWithTimeout(
         `${objectType} with id '${id}' not to exist`,
         5 * 1000,
         async () => {
-          if ((await this.savedObjectExistsById(id, objectType)) === false) {
+          if ((await this.savedObjectExistsById(id, objectType, space)) === false) {
             return true;
           } else {
             throw new Error(`${objectType} with id '${id}' should not exist.`);
@@ -452,8 +482,8 @@ export function MachineLearningTestResourcesProvider(
       );
     },
 
-    async assertIndexPatternExistByTitle(title: string) {
-      await this.assertSavedObjectExistsByTitle(title, SavedObjectType.INDEX_PATTERN);
+    async assertIndexPatternExistByTitle(title: string, space?: string) {
+      await this.assertSavedObjectExistsByTitle(title, SavedObjectType.INDEX_PATTERN, space);
     },
 
     async assertIndexPatternExistById(id: string) {