Add mechanism for dashboard snapshots (#15463)

* Add mechanism for dashboard snapshots

* Adjust wait for render function since it needs to be 2, not gt 0.

Should be obsolete when the new render stuff is complete.

* resize images using new library so comparisons work across different screen resolutions

* use jimp comparison and see if expanding to expanded panel mode helps when comparing across browser/os

* Try to ensure window size

* Experiment with a smaller window, see if screenshot dimensions change

Update screenshot for new window dimensions

* Try cover + quality, see what the diffs look like.

* Stop trying to get TSVB to pass, try area charts

There is a timezone bug with tsvb:
https://github.com/elastic/kibana/issues/15501

* gah, cover didn't work, check resize

* bump render counter to 6, as it should be.

As it turns out, the visualization was not done re-rendering to
maximized mode

* Bump threshold for comparison

* reduce down to a single test run

* Don't use an environment variable to detect updateBaselines cmd line flag

package.json

@@ -263,6 +263,7 @@
     "istanbul-instrumenter-loader": "3.0.0",
     "jest": "21.2.1",
     "jest-cli": "21.2.1",
+    "jimp": "0.2.28",
     "jsdom": "9.9.1",
     "karma": "1.7.0",
     "karma-chrome-launcher": "2.1.1",

src/functional_test_runner/cli.js

@@ -16,6 +16,7 @@ cmd
   .option('--verbose', 'Log everything', false)
   .option('--quiet', 'Only log errors', false)
   .option('--silent', 'Log nothing', false)
+  .option('--updateBaselines', 'Replace baseline screenshots with whatever is generated from the test', false)
   .option('--debug', 'Run in debug mode', false)
   .parse(process.argv);
 
@@ -35,7 +36,8 @@ const functionalTestRunner = createFunctionalTestRunner({
     mochaOpts: {
       bail: cmd.bail,
       grep: cmd.grep,
-    }
+    },
+    updateBaselines: cmd.updateBaselines
   }
 });
 

src/functional_test_runner/lib/config/schema.js

@@ -62,6 +62,8 @@ export const schema = Joi.object().keys({
     ui: Joi.string().default('bdd'),
   }).default(),
 
+  updateBaselines: Joi.boolean().default(false),
+
   junit: Joi.object().keys({
     enabled: Joi.boolean().default(!!process.env.CI),
     reportName: Joi.string(),

src/functional_test_runner/lib/mocha/load_test_files.js

@@ -12,7 +12,7 @@ import { decorateMochaUi } from './decorate_mocha_ui';
  *  @param  {String} path
  *  @return {undefined} - mutates mocha, no return value
  */
-export const loadTestFiles = (mocha, log, lifecycle, providers, paths) => {
+export const loadTestFiles = (mocha, log, lifecycle, providers, paths, updateBaselines) => {
   const innerLoadTestFile = (path) => {
     if (typeof path !== 'string' || !isAbsolute(path)) {
       throw new TypeError('loadTestFile() only accepts absolute paths');
@@ -46,6 +46,7 @@ export const loadTestFiles = (mocha, log, lifecycle, providers, paths) => {
         getService: providers.getService,
         getPageObject: providers.getPageObject,
         getPageObjects: providers.getPageObjects,
+        updateBaselines,
       });
 
       if (returnVal && typeof returnVal.then === 'function') {

src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -27,6 +27,6 @@ export async function setupMocha(lifecycle, log, config, providers) {
     await lifecycle.trigger('beforeEachTest');
   });
 
-  loadTestFiles(mocha, log, lifecycle, providers, config.get('testFiles'));
+  loadTestFiles(mocha, log, lifecycle, providers, config.get('testFiles'), config.get('updateBaselines'));
   return mocha;
 }

test/functional/apps/dashboard/_dashboard_snapshots.js

@@ -0,0 +1,68 @@
+import expect from 'expect.js';
+
+import { AREA_CHART_VIS_NAME } from '../../page_objects/dashboard_page';
+
+
+export default function ({ getService, getPageObjects, updateBaselines }) {
+  const dashboardVisualizations = getService('dashboardVisualizations');
+  const PageObjects = getPageObjects(['dashboard', 'header', 'visualize']);
+  const screenshot = getService('screenshots');
+  const remote = getService('remote');
+
+  describe('dashboard snapshots', function describeIndexTests() {
+    before(async function () {
+      await PageObjects.dashboard.initTests();
+      await PageObjects.dashboard.preserveCrossAppState();
+      await remote.setWindowSize(1000, 500);
+    });
+
+    after(async function () {
+      // avoids any 'Object with id x not found' errors when switching tests.
+      await PageObjects.header.clickVisualize();
+      await PageObjects.visualize.gotoLandingPage();
+      await PageObjects.header.clickDashboard();
+      await PageObjects.dashboard.gotoDashboardLandingPage();
+    });
+
+    // This one won't work because of https://github.com/elastic/kibana/issues/15501.  See if we can get it to work
+    // once TSVB has timezone support.
+    it.skip('compare TSVB snapshot', async () => {
+      await PageObjects.dashboard.gotoDashboardLandingPage();
+      await PageObjects.dashboard.clickNewDashboard();
+      await PageObjects.dashboard.setTimepickerInDataRange();
+      await dashboardVisualizations.createAndAddTSVBVisualization('TSVB');
+      await PageObjects.dashboard.saveDashboard('tsvb');
+      await PageObjects.header.clickToastOK();
+
+      await PageObjects.dashboard.clickFullScreenMode();
+      await PageObjects.dashboard.toggleExpandPanel();
+
+      await PageObjects.dashboard.waitForRenderCounter(2);
+      const percentSimilar = await screenshot.compareAgainstBaseline('tsvb_dashboard', updateBaselines);
+
+      await PageObjects.dashboard.clickExitFullScreenLogoButton();
+
+      expect(percentSimilar).to.be(0);
+    });
+
+    it('compare area chart snapshot', async () => {
+      await PageObjects.dashboard.gotoDashboardLandingPage();
+      await PageObjects.dashboard.clickNewDashboard();
+      await PageObjects.dashboard.setTimepickerInDataRange();
+      await PageObjects.dashboard.addVisualizations([AREA_CHART_VIS_NAME]);
+      await PageObjects.dashboard.saveDashboard('area');
+      await PageObjects.header.clickToastOK();
+
+      await PageObjects.dashboard.clickFullScreenMode();
+      await PageObjects.dashboard.toggleExpandPanel();
+
+      await PageObjects.dashboard.waitForRenderCounter(6);
+      const percentSimilar = await screenshot.compareAgainstBaseline('area_chart', updateBaselines);
+
+      await PageObjects.dashboard.clickExitFullScreenLogoButton();
+
+      // Testing some OS/browser differnces were shown to cause .009 percent difference.
+      expect(percentSimilar).to.be.lessThan(0.05);
+    });
+  });
+}

test/functional/apps/dashboard/index.js

@@ -5,6 +5,7 @@ export default function ({ getService, loadTestFile }) {
     before(() => remote.setWindowSize(1200, 900));
     loadTestFile(require.resolve('./_bwc_shared_urls'));
     loadTestFile(require.resolve('./_dashboard_queries'));
+    loadTestFile(require.resolve('./_dashboard_snapshots'));
     loadTestFile(require.resolve('./_dashboard_grid'));
     loadTestFile(require.resolve('./_panel_controls'));
     loadTestFile(require.resolve('./_view_edit'));

test/functional/services/lib/compare_pngs.js

@@ -0,0 +1,37 @@
+import Jimp from 'jimp';
+
+export async function comparePngs(actualPath, expectedPath, diffPath, log) {
+  log.debug(`comparePngs: ${actualPath} vs ${expectedPath}`);
+  const actual = (await Jimp.read(actualPath)).clone();
+  const expected = (await Jimp.read(expectedPath)).clone();
+
+  if (actual.bitmap.width !== expected.bitmap.width || actual.bitmap.height !== expected.bitmap.height) {
+    console.log('expected height ' + expected.bitmap.height + ' and width ' + expected.bitmap.width);
+    console.log('actual height ' + actual.bitmap.height + ' and width ' + actual.bitmap.width);
+
+    const width = Math.min(actual.bitmap.width, expected.bitmap.width);
+    const height = Math.min(actual.bitmap.height, expected.bitmap.height);
+    actual.resize(width, height);//, Jimp.HORIZONTAL_ALIGN_LEFT | Jimp.VERTICAL_ALIGN_TOP);
+    expected.resize(width, height);//, Jimp.HORIZONTAL_ALIGN_LEFT | Jimp.VERTICAL_ALIGN_TOP);
+  }
+
+  actual.quality(60);
+  expected.quality(60);
+
+  log.debug(`calculating diff pixels...`);
+  // Note that this threshold value only affects color comparison from pixel to pixel. It won't have
+  // any affect when comparing neighboring pixels - so slight shifts, font variations, or "blurry-ness"
+  // will still show up as diffs, but upping this will not help that.  Instead we keep the threshold low, and expect
+  // some the diffCount to be lower than our own threshold value.
+  const THRESHOLD = .1;
+  const { image, percent } = Jimp.diff(actual, expected, THRESHOLD);
+  log.debug(`percentSimilar: ${percent}`);
+  if (percent > 0) {
+    image.write(diffPath);
+
+    // For debugging purposes it'll help to see the resized images and how they compare.
+    actual.write(actualPath.substring(0, actualPath.length - 4) + '-resized.png');
+    expected.write(expectedPath.substring(0, expectedPath.length - 4) + '-resized.png');
+  }
+  return percent;
+}

test/functional/services/screenshots.js

@@ -1,9 +1,12 @@
 import { resolve, dirname } from 'path';
-import { writeFile } from 'fs';
-
-import { fromNode as fcb } from 'bluebird';
+import { writeFile, readFileSync } from 'fs';
+import { fromNode as fcb, promisify } from 'bluebird';
 import mkdirp from 'mkdirp';
 import del from 'del';
+import { comparePngs } from './lib/compare_pngs';
+
+const mkdirAsync = promisify(mkdirp);
+const writeFileAsync = promisify(writeFile);
 
 export async function ScreenshotsProvider({ getService }) {
   const log = getService('log');
@@ -13,9 +16,35 @@ export async function ScreenshotsProvider({ getService }) {
 
   const SESSION_DIRECTORY = resolve(config.get('screenshots.directory'), 'session');
   const FAILURE_DIRECTORY = resolve(config.get('screenshots.directory'), 'failure');
+  const BASELINE_DIRECTORY = resolve(config.get('screenshots.directory'), 'baseline');
   await del([SESSION_DIRECTORY, FAILURE_DIRECTORY]);
 
   class Screenshots {
+
+    /**
+     *
+     * @param name {string} name of the file to use for comparison
+     * @param updateBaselines {boolean} optional, pass true to update the baseline snapshot.
+     * @return {Promise.<number>} Percentage difference between the baseline and the current snapshot.
+     */
+    async compareAgainstBaseline(name, updateBaselines) {
+      log.debug('compareAgainstBaseline');
+      const sessionPath = resolve(SESSION_DIRECTORY, `${name}.png`);
+      await this._take(sessionPath);
+
+      const baselinePath = resolve(BASELINE_DIRECTORY, `${name}.png`);
+      const failurePath = resolve(FAILURE_DIRECTORY, `${name}.png`);
+
+      if (updateBaselines) {
+        log.debug('Updating baseline snapshot');
+        await writeFileAsync(baselinePath, readFileSync(sessionPath));
+        return 0;
+      } else {
+        await mkdirAsync(FAILURE_DIRECTORY);
+        return await comparePngs(sessionPath, baselinePath, failurePath, log);
+      }
+    }
+
     async take(name) {
       return await this._take(resolve(SESSION_DIRECTORY, `${name}.png`));
     }