[Rollup] Fix flaky test when deleting Jobs (#33231) (#33254)

x-pack/plugins/rollup/public/search/rollup_search_strategy.js

@@ -18,7 +18,7 @@ function getAllFetchParams(searchRequests, Promise) {
   });
 }
 
-async function serializeAllFetchParams(fetchParams, searchRequests) {
+function serializeAllFetchParams(fetchParams, searchRequests) {
   const searchRequestsWithFetchParams = [];
   const failedSearchRequests = [];
 

x-pack/plugins/rollup/server/lib/jobs_compatibility.js

@@ -53,12 +53,12 @@ export function mergeJobConfigurations(jobs = []) {
         const aggName = agg.agg;
         const aggDoesntExist = !allAggs[aggName];
         const fieldDoesntExist = allAggs[aggName] && !allAggs[aggName][fieldName];
-        const aggIsntDateHistogram = aggName !== 'date_histogram';
+        const isDateHistogramAgg = aggName === 'date_histogram';
 
         // If we currently don't have this aggregation, add it.
         // Special case for date histogram, since there can only be one
         // date histogram field.
-        if(aggDoesntExist || (fieldDoesntExist && aggIsntDateHistogram)) {
+        if(aggDoesntExist || (fieldDoesntExist && !isDateHistogramAgg)) {
           allAggs[aggName] = allAggs[aggName] || {};
           allAggs[aggName][fieldName] = { ...agg };
         }

x-pack/test/api_integration/apis/management/rollup/constants.js

@@ -5,11 +5,12 @@
  */
 
 export const API_BASE_PATH = '/api/rollup';
+export const INDEX_PATTERNS_EXTENSION_BASE_PATH = '/api/index_patterns/rollup';
 export const ROLLUP_INDEX_NAME = 'rollup_index';
 export const INDEX_TO_ROLLUP_MAPPINGS = {
   properties: {
-    testTotalField: { 'type': 'long'  },
-    testTagField: { 'type': 'keyword'  },
-    testCreatedField: { 'type': 'date'  },
+    testTotalField: { type: 'long'  },
+    testTagField: { type: 'keyword'  },
+    testCreatedField: { type: 'date'  },
   }
 };

x-pack/test/api_integration/apis/management/rollup/index.js

@@ -7,5 +7,7 @@
 export default function ({ loadTestFile }) {
   describe('rollup', () => {
     loadTestFile(require.resolve('./rollup'));
+    loadTestFile(require.resolve('./index_patterns_extensions'));
+    loadTestFile(require.resolve('./rollup_search'));
   });
 }

x-pack/test/api_integration/apis/management/rollup/index_patterns_extensions.js

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+import querystring from 'querystring';
+
+import { registerHelpers } from './rollup.test_helpers';
+import { INDEX_TO_ROLLUP_MAPPINGS, INDEX_PATTERNS_EXTENSION_BASE_PATH } from './constants';
+import { getRandomString } from './lib';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const es = getService('es');
+
+  const {
+    createIndexWithMappings,
+    getJobPayload,
+    createJob,
+    cleanUp,
+  } = registerHelpers({ supertest, es });
+
+  describe('index patterns extension', () => {
+    describe('Fields for wildcards', () => {
+      const BASE_URI = `${INDEX_PATTERNS_EXTENSION_BASE_PATH}/_fields_for_wildcard`;
+
+      describe('query params validation', () => {
+        let uri;
+        let body;
+        let params;
+
+        it('"pattern" is required', async () => {
+          uri = `${BASE_URI}`;
+          ({ body } = await supertest.get(uri).expect(400));
+          expect(body.message).to.contain('"pattern" is required');
+        });
+
+        it('"params" is required', async () => {
+          params = { pattern: 'foo' };
+          uri = `${BASE_URI}?${querystring.stringify(params)}`;
+          ({ body } = await supertest.get(uri).expect(400));
+          expect(body.message).to.contain('"params" is required');
+        });
+
+        it('"params" must be an object', async () => {
+          params = { pattern: 'foo', params: 'bar' };
+          uri = `${BASE_URI}?${querystring.stringify(params)}`;
+          ({ body } = await supertest.get(uri).expect(400));
+          expect(body.message).to.contain('"params" must be an object');
+        });
+
+        it('"params" must be an object that only accepts a "rollup_index" property', async () => {
+          params = { pattern: 'foo', params: JSON.stringify({ someProp: 'bar' }) };
+          uri = `${BASE_URI}?${querystring.stringify(params)}`;
+          ({ body } = await supertest.get(uri).expect(400));
+          expect(body.message).to.contain('"someProp" is not allowed');
+        });
+
+        it('"meta_fields" must be an Array', async () => {
+          params = {
+            pattern: 'foo',
+            params: JSON.stringify({ rollup_index: 'bar' }),
+            meta_fields: 'stringValue'
+          };
+          uri = `${BASE_URI}?${querystring.stringify(params)}`;
+          ({ body } = await supertest.get(uri).expect(400));
+          expect(body.message).to.contain('"meta_fields" must be an array');
+        });
+
+        it('should return 404 the rollup index to query does not exist', async () => {
+          uri = `${BASE_URI}?${querystring.stringify({ pattern: 'foo', params: JSON.stringify({ rollup_index: 'bar' }) })}`;
+          ({ body } = await supertest.get(uri).expect(404));
+          expect(body.message).to.contain('no such index [bar]');
+        });
+      });
+
+      it('should return the correct fields', async () => {
+        // Create a Rollup job on an index with the INDEX_TO_ROLLUP_MAPPINGS
+        const indexName = await createIndexWithMappings();
+        const rollupIndex = getRandomString();
+        const payload = getJobPayload(indexName, undefined, rollupIndex);
+        await createJob(payload);
+
+        // Query for wildcard
+        const params = {
+          pattern: indexName,
+          params: JSON.stringify({ rollup_index: rollupIndex })
+        };
+        const uri = `${BASE_URI}?${querystring.stringify(params)}`;
+        const { body } = await supertest.get(uri).expect(200);
+
+        // Verify that the fields for wildcard correspond to our declared mappings
+        const propertiesWithMappings = Object.keys(INDEX_TO_ROLLUP_MAPPINGS.properties);
+        const fieldsForWildcard = body.fields.map(field => field.name);
+        expect(fieldsForWildcard.sort()).eql(propertiesWithMappings.sort());
+
+        // Cleanup
+        await cleanUp();
+      });
+    });
+  });
+}

x-pack/test/api_integration/apis/management/rollup/lib/index.js

@@ -11,3 +11,7 @@ export {
 export {
   getRandomString,
 } from './random';
+
+export {
+  wait
+} from './utils';

x-pack/test/api_integration/apis/management/rollup/lib/utils.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+export const wait = (time = 1000) => (
+  new Promise((resolve) => (
+    setTimeout(resolve, time)
+  ))
+);

x-pack/test/api_integration/apis/management/rollup/rollup.js

@@ -23,10 +23,10 @@ export default function ({ getService }) {
     cleanUp,
   } = registerHelpers({ supertest, es });
 
-  describe('Rollup jobs', () => {
+  describe('jobs', () => {
     after(() => cleanUp());
 
-    describe('Rollup indices', () => {
+    describe('indices', () => {
       it('should return an empty object when there are no rollup indices', async () => {
         const uri = `${API_BASE_PATH}/indices`;
 
@@ -74,7 +74,7 @@ export default function ({ getService }) {
       });
     });
 
-    describe('Crud', () => {
+    describe('crud', () => {
       describe('list', () => {
         it('should return an empty array when there are no jobs', async () => {
           const { body } = await supertest
@@ -206,7 +206,7 @@ export default function ({ getService }) {
       });
     });
 
-    describe('Actions', () => {
+    describe('actions', () => {
       describe('start', () => {
         let job;
 

x-pack/test/api_integration/apis/management/rollup/rollup.test_helpers.js

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { initElasticsearchIndicesHelpers, getRandomString } from './lib';
+import { initElasticsearchIndicesHelpers, getRandomString, wait } from './lib';
 import { API_BASE_PATH, ROLLUP_INDEX_NAME, INDEX_TO_ROLLUP_MAPPINGS } from './constants';
 
 const jobsCreated = [];
@@ -17,11 +17,11 @@ export const registerHelpers = ({ supertest, es }) => {
     return createIndex(indexName, { mappings });
   };
 
-  const getJobPayload = (indexName, id = getRandomString()) => ({
+  const getJobPayload = (indexName, id = getRandomString(), rollupIndex = ROLLUP_INDEX_NAME) => ({
     job: {
       id,
       index_pattern: indexName,
-      rollup_index: ROLLUP_INDEX_NAME,
+      rollup_index: rollupIndex,
       cron: '0 0 0 ? * 7',
       page_size: 1000,
       groups: {
@@ -86,9 +86,29 @@ export const registerHelpers = ({ supertest, es }) => {
       .send({ jobIds });
   };
 
+  const loadJobs = () => supertest.get(`${API_BASE_PATH}/jobs`);
+
+  const waitForJobsToStop = (attempt = 0) => (
+    loadJobs()
+      .then(async ({ body: { jobs } }) => {
+        const jobBeingStopped = jobs.filter(job => job.status.job_state !== 'stopped' && job.status.job_state !== 'started');
+
+        if (!jobBeingStopped.length) {
+          return;
+        }
+
+        if (attempt < 3 && jobBeingStopped.length) {
+          await wait(500);
+          return waitForJobsToStop(++attempt);
+        }
+
+        throw new Error('Error while waiting for Rollup Jobs to stop');
+      }));
+
   const stopAllJobStarted = (jobIds = jobsStarted, attempt = 0) => (
     stopJob(jobIds)
-      .then(() => supertest.get(`${API_BASE_PATH}/jobs`))
+      .then(waitForJobsToStop)
+      .then(loadJobs)
       .then(({ body: { jobs } }) => {
         // We make sure that there are no more jobs started
         // as trying to delete a job that is started will throw an exception
@@ -109,7 +129,7 @@ export const registerHelpers = ({ supertest, es }) => {
           throw response;
         }
       })
-      .then(() => supertest.get(`${API_BASE_PATH}/jobs`))
+      .then(loadJobs)
       .then(({ body: { jobs } }) => {
         if (jobs.length && attempt < 3) {
           // There are still some jobs left to delete.
@@ -135,7 +155,7 @@ export const registerHelpers = ({ supertest, es }) => {
   const cleanUp = () => (
     Promise.all([
       deleteAllIndices(),
-      stopAllJobStarted().then(() => deleteJobsCreated()),
+      stopAllJobStarted().then(deleteJobsCreated),
       deleteIndicesGeneratedByJobs(),
     ]).catch(err => {
       console.log('ERROR cleaning up!');

x-pack/test/api_integration/apis/management/rollup/rollup_search.js

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+
+import { registerHelpers } from './rollup.test_helpers';
+import { API_BASE_PATH } from './constants';
+import { getRandomString } from './lib';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const es = getService('es');
+
+  const {
+    createIndexWithMappings,
+    getJobPayload,
+    createJob,
+    cleanUp,
+  } = registerHelpers({ supertest, es });
+
+  describe('search', () => {
+    const URI = `${API_BASE_PATH}/search`;
+
+    it('return a 404 if the rollup index does not exist', async () => {
+      const { body } = await supertest
+        .post(URI)
+        .set('kbn-xsrf', 'xxx')
+        .send([{ index: 'unknown', query: {} } ])
+        .expect(404);
+
+      expect(body.message).to.contain('no such index [unknown]');
+    });
+
+    it('should return a 200 when searching on existing rollup index', async () => {
+      // Create a Rollup job on an index with the INDEX_TO_ROLLUP_MAPPINGS
+      const indexName = await createIndexWithMappings();
+      const rollupIndex = getRandomString();
+      await createJob(getJobPayload(indexName, undefined, rollupIndex));
+
+      const { body } = await supertest
+        .post(URI)
+        .set('kbn-xsrf', 'xxx')
+        .send([{ index: rollupIndex, query: { size: 0 } } ])
+        .expect(200);
+
+      // make sure total hits is an integer and not an object
+      expect(body[0].hits.total).to.equal(0);
+
+      await cleanUp();
+    });
+  });
+}