[7.x] Hybrid index pattern test (#43498) (#49176)

test/functional/page_objects/settings_page.js

@@ -239,8 +239,8 @@ export function SettingsPageProvider({ getService, getPageObjects }) {
     async setScriptedFieldLanguageFilter(language) {
       await find.clickByCssSelector(
         'select[data-test-subj="scriptedFieldLanguageFilterDropdown"] > option[label="' +
-            language +
-            '"]'
+        language +
+        '"]'
       );
     }
 
@@ -287,9 +287,14 @@ export function SettingsPageProvider({ getService, getPageObjects }) {
       await indexLink.click();
     }
 
+    async getIndexPatternList() {
+      await testSubjects.existOrFail('indexPatternTable', { timeout: 5000 });
+      return await find.allByCssSelector('[data-test-subj="indexPatternTable"] .euiTable a');
+    }
+
     async isIndexPatternListEmpty() {
       await testSubjects.existOrFail('indexPatternTable', { timeout: 5000 });
-      const indexPatternList = await find.allByCssSelector('[data-test-subj="indexPatternTable"] .euiTable a');
+      const indexPatternList = await this.getIndexPatternList();
       return indexPatternList.length === 0;
     }
 
@@ -300,13 +305,16 @@ export function SettingsPageProvider({ getService, getPageObjects }) {
       }
     }
 
-    async createIndexPattern(indexPatternName, timefield = '@timestamp') {
+    async createIndexPattern(indexPatternName, timefield = '@timestamp', isStandardIndexPattern = true) {
       await retry.try(async () => {
         await this.navigateTo();
         await PageObjects.header.waitUntilLoadingHasFinished();
         await this.clickKibanaIndexPatterns();
         await PageObjects.header.waitUntilLoadingHasFinished();
         await this.clickOptionalAddNewButton();
+        if (!isStandardIndexPattern) {
+          await this.clickCreateNewRollupButton();
+        }
         await PageObjects.header.waitUntilLoadingHasFinished();
         await retry.try(async () => {
           await this.setIndexPatternField({ indexPatternName });
@@ -340,6 +348,10 @@ export function SettingsPageProvider({ getService, getPageObjects }) {
       }
     }
 
+    async clickCreateNewRollupButton() {
+      await testSubjects.click('createRollupIndexPatternButton');
+    }
+
     async getIndexPatternIdFromUrl() {
       const currentUrl = await browser.getCurrentUrl();
       const indexPatternId = currentUrl.match(/.*\/(.*)/)[1];

x-pack/legacy/plugins/index_management/public/sections/home/index_list/index_table/index_table.js

@@ -337,6 +337,7 @@ export class IndexTable extends Component {
       const { name } = index;
       return (
         <EuiTableRow
+          data-test-subj="indexTableRow"
           isSelected={this.isItemSelected(name) || name === detailPanelIndexName}
           isSelectable
           key={`${name}-row`}
@@ -383,7 +384,8 @@ export class IndexTable extends Component {
     return (
       <EuiFlexItem key={name} grow={false}>
         <EuiSwitch
-          id={`checkboxToggles-{name}`}
+          id={`checkboxToggles-${name}`}
+          data-test-subj={`checkboxToggles-${name}`}
           checked={toggleNameToVisibleMap[name]}
           onChange={event => toggleChanged(name, event.target.checked)}
           label={label}

x-pack/test/functional/apps/rollup_job/hybrid_index_helper.js

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+export default function mockRolledUpData(jobName, targetIndexName, day) {
+  return {
+    index: `${targetIndexName}`,
+    body: {
+      '_rollup.version': 2,
+      '@timestamp.date_histogram.time_zone': 'UTC',
+      '@timestamp.date_histogram.timestamp': day.toISOString(),
+      '@timestamp.date_histogram.interval': '1000ms',
+      '@timestamp.date_histogram._count': 1,
+      '_rollup.id': jobName
+    }
+  };
+}
+
+//This function just adds some stub indices that includes a timestamp and an arbritary metric. This is fine since we are not actually testing
+//rollup functionality.
+export function mockIndices(day, prepend) {
+  return {
+    index: `${prepend}-${day.format('MM-DD-YYYY')}`,
+    body: {
+      '@timestamp': day.toISOString(),
+      foo_metric: 1
+    }
+  };
+}

x-pack/test/functional/apps/rollup_job/hybrid_index_pattern.js

@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+
+import datemath from '@elastic/datemath';
+import expect from '@kbn/expect';
+import mockRolledUpData, { mockIndices } from './hybrid_index_helper';
+
+export default function ({ getService, getPageObjects }) {
+
+  const es = getService('es');
+  const esArchiver = getService('esArchiver');
+  const retry = getService('retry');
+  const PageObjects = getPageObjects(['common', 'settings']);
+
+  describe('hybrid index pattern', function () {
+    //Since rollups can only be created once with the same name (even if you delete it),
+    //we add the Date.now() to avoid name collision if you run the tests locally back to back.
+    const rollupJobName = `hybrid-index-pattern-test-rollup-job-${Date.now()}`;
+    const rollupTargetIndexName = `rollup-target-data`;
+    const regularIndexPrefix = `regular-index`;
+    const rollupSourceIndexPrefix = `rollup-source-data`;
+    const rollupIndexPatternName = `${regularIndexPrefix}*,${rollupTargetIndexName}`;
+    const now = new Date();
+    const pastDates = [
+      datemath.parse('now-1d', { forceNow: now }),
+      datemath.parse('now-2d', { forceNow: now }),
+      datemath.parse('now-3d', { forceNow: now }),
+    ];
+
+    it('create hybrid index pattern', async () => {
+      //Create data for rollup job to recognize.
+      //Index past data to be used in the test.
+      await pastDates.map(async (day) => {
+        await es.index(mockIndices(day, rollupSourceIndexPrefix));
+      });
+
+      await retry.waitForWithTimeout('waiting for 3 records to be loaded into elasticsearch.', 10000, async () => {
+        const response = await es.indices.get({
+          index: `${rollupSourceIndexPrefix}*`,
+          allow_no_indices: false
+        });
+        return Object.keys(response).length === 3;
+      });
+
+      await retry.try(async () => {
+        //Create a rollup for kibana to recognize
+        await es.transport.request({
+          path: `/_rollup/job/${rollupJobName}`,
+          method: 'PUT',
+          body: {
+            'index_pattern': `${rollupSourceIndexPrefix}*`,
+            'rollup_index': rollupTargetIndexName,
+            'cron': '*/10 * * * * ?',
+            'groups': {
+              'date_histogram': {
+                'fixed_interval': '1000ms',
+                'field': '@timestamp',
+                'time_zone': 'UTC'
+              }
+            },
+            'timeout': '20s',
+            'page_size': 1000
+          }
+        });
+      });
+
+
+      await pastDates.map(async (day) => {
+        await es.index(mockRolledUpData(rollupJobName, rollupTargetIndexName, day));
+      });
+
+      //Index live data to be used in the test.
+      await es.index(mockIndices(datemath.parse('now', { forceNow: now }), regularIndexPrefix));
+
+      await PageObjects.common.navigateToApp('settings');
+      await PageObjects.settings.createIndexPattern(rollupIndexPatternName, '@timestamp', false);
+
+
+      await PageObjects.settings.clickKibanaIndexPatterns();
+      const indexPattern = (await PageObjects.settings.getIndexPatternList()).pop();
+      const indexPatternText = await indexPattern.getVisibleText();
+      expect(indexPatternText).to.contain(rollupIndexPatternName);
+      expect(indexPatternText).to.contain('Rollup');
+    });
+
+    after(async () => {
+      // Delete the rollup job.
+      await es.transport.request({
+        path: `/_rollup/job/${rollupJobName}`,
+        method: 'DELETE',
+      });
+
+      await es.indices.delete({ index: rollupTargetIndexName });
+      await es.indices.delete({ index: `${regularIndexPrefix}*` });
+      await es.indices.delete({ index: `${rollupSourceIndexPrefix}*` });
+      await esArchiver.load('empty_kibana');
+    });
+  });
+}

x-pack/test/functional/apps/rollup_job/index.js

@@ -5,10 +5,10 @@
  */
 
 export default function ({ loadTestFile }) {
-  // FLAKY: https://github.com/elastic/kibana/issues/43559
-  describe.skip('rollup job', function () {
+  describe('rollup job', function () {
     this.tags('ciGroup1');
 
     loadTestFile(require.resolve('./rollup_jobs'));
+    loadTestFile(require.resolve('./hybrid_index_pattern'));
   });
 }

x-pack/test/functional/apps/rollup_job/rollup_jobs.js

@@ -4,64 +4,64 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+
+import datemath from '@elastic/datemath';
 import expect from '@kbn/expect';
-import { indexBy } from 'lodash';
+import { mockIndices } from './hybrid_index_helper';
 
 export default function ({ getService, getPageObjects }) {
+
+  const es = getService('es');
   const esArchiver = getService('esArchiver');
-  const log = getService('log');
-  const PageObjects = getPageObjects(['security', 'rollup', 'common', 'header']);
+  const PageObjects = getPageObjects(['rollup', 'common']);
 
   describe('rollup job', function () {
-    this.tags('smoke');
-    before(async () => {
-      // init data
-      await esArchiver.loadIfNeeded('logstash_functional');
-      await esArchiver.load('canvas/default');
-      await PageObjects.common.navigateToApp('rollupJob');
-    });
+    //Since rollups can only be created once with the same name (even if you delete it),
+    //we add the Date.now() to avoid name collision.
+    const rollupJobName = 'rollup-to-be-' + Date.now();
+    const targetIndexName = 'rollup-to-be';
+    const rollupSourceIndexPattern = 'to-be*';
+    const rollupSourceDataPrepend = 'to-be';
+    //make sure all dates have the same concept of "now"
+    const now = new Date();
+    const pastDates = [
+      datemath.parse('now-1d', { forceNow: now }),
+      datemath.parse('now-2d', { forceNow: now }),
+      datemath.parse('now-3d', { forceNow: now }),
+    ];
 
-    after(async () => await esArchiver.unload('logstash_functional'));
-
-    it('create and save a new job', async () => {
-      const jobName = 'Testjob1';
-      const indexPattern = '.kibana*';
-      const indexName = 'rollup_index';
+    it('create new rollup job', async () => {
       const interval = '1000ms';
 
-      await PageObjects.rollup.createNewRollUpJob();
-      await PageObjects.rollup.verifyStepIsActive(1);
-      await PageObjects.rollup.addRoleNameandIndexPattern(jobName, indexPattern);
-      await PageObjects.rollup.verifyIndexPatternAccepted();
-      await PageObjects.rollup.setIndexName(indexName);
-      await PageObjects.rollup.moveToNextStep();
+      pastDates.map(async (day) => {
+        await es.index(mockIndices(day, rollupSourceDataPrepend));
+      });
 
-      //now navigate to histogram
-      await PageObjects.rollup.verifyStepIsActive(2);
-      await PageObjects.rollup.setJobInterval(interval);
-      await PageObjects.rollup.moveToNextStep();
+      await PageObjects.common.navigateToApp('rollupJob');
+      await PageObjects.rollup.createNewRollUpJob(rollupJobName, rollupSourceIndexPattern, targetIndexName,
+        interval, ' ', true, { time: '*/10 * * * * ?', cron: true });
 
-      //Terms (optional)
-      await PageObjects.rollup.verifyStepIsActive(3);
-      await PageObjects.rollup.moveToNextStep();
+      const jobList = await PageObjects.rollup.getJobList();
+      expect(jobList.length).to.be(1);
 
-      //Histogram(optional)
-      await PageObjects.rollup.verifyStepIsActive(4);
-      await PageObjects.rollup.moveToNextStep();
+    });
 
-      //Metrics(optional)
-      await PageObjects.rollup.verifyStepIsActive(5);
-      await PageObjects.rollup.moveToNextStep();
+    after(async () => {
+      //Stop the running rollup job.
+      await es.transport.request({
+        path: `/_rollup/job/${rollupJobName}/_stop?wait_for_completion=true`,
+        method: 'POST',
+      });
+      // Delete the rollup job.
+      await es.transport.request({
+        path: `/_rollup/job/${rollupJobName}`,
+        method: 'DELETE',
+      });
 
-      //saveJob and verify the name in the list
-      await PageObjects.rollup.verifyStepIsActive(6);
-      await PageObjects.rollup.saveJob();
-
-      // verify jobListTitle
-      const jobList = indexBy(await PageObjects.rollup.getJobList(), 'jobName');
-      log.debug(JSON.stringify(jobList));
-      log.debug(Object.keys(jobList));
-      expect(Object.keys(jobList)).to.have.length(1);
+      //Delete all data indices that were created.
+      await es.indices.delete({ index: targetIndexName });
+      await es.indices.delete({ index: rollupSourceIndexPattern });
+      await esArchiver.load('empty_kibana');
     });
   });
 }

x-pack/test/functional/page_objects/index_management_page.ts

@@ -7,15 +7,55 @@
 import { FtrProviderContext } from '../ftr_provider_context';
 
 export function IndexManagementPageProvider({ getService }: FtrProviderContext) {
+  const find = getService('find');
   const testSubjects = getService('testSubjects');
 
   return {
     async sectionHeadingText() {
       return await testSubjects.getVisibleText('appTitle');
     },
+    async reloadIndices() {
+      await testSubjects.click('reloadIndicesButton');
+    },
     async reloadIndicesButton() {
       return await testSubjects.find('reloadIndicesButton');
     },
+    async toggleRollupIndices() {
+      await testSubjects.click('checkboxToggles-rollupToggle');
+    },
+
+    async getIndexList() {
+      const table = await find.byCssSelector('table');
+      const $ = await table.parseDomContent();
+      return $.findTestSubjects('indexTableRow')
+        .toArray()
+        .map(row => {
+          return {
+            indexName: $(row)
+              .findTestSubject('indexTableIndexNameLink')
+              .text(),
+            indexHealth: $(row)
+              .findTestSubject('indexTableCell-health')
+              .text(),
+            indexStatus: $(row)
+              .findTestSubject('indexTableCell-status')
+              .text(),
+            indexPrimary: $(row)
+              .findTestSubject('indexTableCell-primary')
+              .text(),
+            indexReplicas: $(row)
+              .findTestSubject('indexTableCell-replica')
+              .text(),
+            indexDocuments: $(row)
+              .findTestSubject('indexTableCell-documents')
+              .text()
+              .replace('documents', ''),
+            indexSize: $(row)
+              .findTestSubject('indexTableCell-size')
+              .text(),
+          };
+        });
+    },
     async changeTabs(tab: 'indicesTab' | 'templatesTab') {
       return await testSubjects.click(tab);
     },

x-pack/test/functional/page_objects/rollup_page.js

@@ -10,24 +10,69 @@ import { map as mapAsync } from 'bluebird';
 export function RollupPageProvider({ getService, getPageObjects }) {
   const testSubjects = getService('testSubjects');
   const log = getService('log');
+  const find = getService('find');
   const PageObjects = getPageObjects(['header', 'common']);
 
 
   class RollupJobPage {
-    async createNewRollUpJob() {
+    async createNewRollUpJob(jobName, indexPattern, indexName, interval, delay = '1d',
+      startImmediately = false, scheduledTime = { time: 'minute', cron: true }) {
+      let stepNum = 1;
+      //Step 1
       await testSubjects.click('createRollupJobButton');
+      await this.verifyStepIsActive(stepNum);
+      await this.addRollupNameandIndexPattern(jobName, indexPattern);
+      await this.verifyIndexPatternAccepted();
+      await this.setIndexName(indexName);
+      await this.setScheduleTime(scheduledTime.time, scheduledTime.cron);
+      await this.setRollupDelay(delay);
+      stepNum = await this.moveToNextStep(stepNum);
+
+      //Step 2: Histogram
+      await this.verifyStepIsActive(stepNum);
+      await this.setJobInterval(interval);
+      stepNum = await this.moveToNextStep(stepNum);
+
+      //Step 3: Terms (optional)
+      await this.verifyStepIsActive(stepNum);
+      stepNum = await this.moveToNextStep();
+
+      //Step 4: Histogram(optional)
+      await this.verifyStepIsActive(stepNum);
+      stepNum = await this.moveToNextStep();
+
+      //Step 5: Metrics(optional)
+      await this.verifyStepIsActive(stepNum);
+      stepNum = await this.moveToNextStep();
+
+      //Step 6: saveJob and verify the name in the list
+      await this.verifyStepIsActive(stepNum);
+      await this.saveJob(startImmediately);
     }
 
     async verifyStepIsActive(stepNumber) {
       await testSubjects.exists(`createRollupStep${stepNumber}--active`);
     }
 
-    async addRoleNameandIndexPattern(name, indexPattern) {
+    async setScheduleTime(time, cron) {
+      if (cron) {
+        await testSubjects.click('rollupShowAdvancedCronLink');
+        await testSubjects.setValue('rollupAdvancedCron', time);
+      }
+      // TODO: Add handling for if Cron is false to go through clicking options.
+    }
+
+    async addRollupNameandIndexPattern(name, indexPattern) {
       log.debug(`Adding name ${name} to form`);
       await testSubjects.setValue('rollupJobName', name);
       await testSubjects.setValue('rollupIndexPattern', indexPattern);
     }
 
+    async setRollupDelay(time) {
+      log.debug(`Setting rollup delay to "${time}"`);
+      await testSubjects.setValue('rollupDelay', time);
+    }
+
     async verifyIndexPatternAccepted() {
       const span = await testSubjects.find('fieldIndexPatternSuccessMessage');
       const message = await span.findByCssSelector('p');
@@ -39,15 +84,20 @@ export function RollupPageProvider({ getService, getPageObjects }) {
       await testSubjects.setValue('rollupIndexName', name);
     }
 
-    async moveToNextStep() {
+    async moveToNextStep(stepNum) {
       await testSubjects.click('rollupJobNextButton');
+      return stepNum + 1;
     }
 
     async setJobInterval(time) {
       await testSubjects.setValue('rollupJobInterval', time);
     }
 
-    async saveJob() {
+    async saveJob(startImmediately) {
+      if (startImmediately) {
+        const checkbox = await find.byCssSelector('.euiCheckbox');
+        await checkbox.click();
+      }
       await testSubjects.click('rollupJobSaveButton');
       await PageObjects.header.waitUntilLoadingHasFinished();
     }