Add visual regression screenshot gallery.

- Add 'test:visualRegression' grunt task.
- Run 'test:visualRegression' as part of npm script 'test'.
- Add 'clean:screenshots task'.
- Clean screenshots/session when funtional tests are run.

.gitignore

@@ -15,6 +15,7 @@ target
 /test/screenshots/diff
 /test/screenshots/failure
 /test/screenshots/session
+/test/screenshots/visual_regression_gallery.html
 /esvm
 .htpasswd
 .eslintcache

package.json

@@ -38,7 +38,7 @@
     "Tim Sullivan <tim@elastic.co>"
   ],
   "scripts": {
-    "test": "grunt test",
+    "test": "grunt test; grunt test:visualRegression",
     "test:dev": "grunt test:dev",
     "test:quick": "grunt test:quick",
     "test:browser": "grunt test:browser",
@@ -59,8 +59,7 @@
     "makelogs": "makelogs",
     "mocha": "mocha",
     "mocha:debug": "mocha --debug-brk",
-    "sterilize": "grunt sterilize",
-    "compareScreenshots": "node utilities/compareScreenshots"
+    "sterilize": "grunt sterilize"
   },
   "repository": {
     "type": "git",
@@ -153,6 +152,7 @@
     "auto-release-sinon": "1.0.3",
     "babel-eslint": "4.1.8",
     "chokidar": "1.4.3",
+    "dot": "1.0.3",
     "elasticdump": "2.1.1",
     "eslint": "1.10.3",
     "eslint-plugin-mocha": "1.1.0",
@@ -191,7 +191,7 @@
     "nock": "2.10.0",
     "npm": "2.11.0",
     "portscanner": "1.0.0",
-    "simple-git": "1.8.0",
+    "simple-git": "1.37.0",
     "sinon": "1.17.2",
     "source-map": "0.4.4",
     "source-map-support": "0.4.0",

tasks/config/clean.js

@@ -3,6 +3,7 @@ module.exports = function (grunt) {
   return {
     build: 'build',
     target: 'target',
+    screenshots: 'test/screenshots/session',
     testsFromModules: 'build/kibana/node_modules/**/{test,tests}/**',
     deepModuleBins: 'build/kibana/node_modules/*/node_modules/**/.bin/{' + modules.join(',') + '}',
     deepModules: 'build/kibana/node_modules/*/node_modules/**/{' + modules.join(',') + '}/',

tasks/test.js

@@ -1,5 +1,16 @@
-var _ = require('lodash');
+const _ = require('lodash');
+const visualRegression = require('../utilities/visual_regression');
+
 module.exports = function (grunt) {
+  grunt.registerTask(
+    'test:visualRegression',
+    'Compare screenshots and generate diff images.',
+    function () {
+      const done = this.async();
+      visualRegression.run(done);
+    }
+  );
+
   grunt.registerTask('test:server', [ 'esvm:test', 'simplemocha:all', 'esvm_shutdown:test' ]);
   grunt.registerTask('test:browser', [ 'run:testServer', 'karma:unit' ]);
   grunt.registerTask('test:coverage', [ 'run:testCoverageServer', 'karma:coverage' ]);
@@ -21,6 +32,7 @@ module.exports = function (grunt) {
     'run:testUIServer',
     'downloadSelenium',
     'run:seleniumServer',
+    'clean:screenshots',
     'intern:dev',
     'esvm_shutdown:ui',
     'stop:seleniumServer',
@@ -35,6 +47,7 @@ module.exports = function (grunt) {
   ]);
 
   grunt.registerTask('test:ui:runner', [
+    'clean:screenshots',
     'intern:dev'
   ]);
 
@@ -55,7 +68,7 @@ module.exports = function (grunt) {
     'intern:api'
   ]);
 
-  grunt.registerTask('test', function (subTask) {
+  grunt.registerTask('test', subTask => {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 
     grunt.task.run(_.compact([

test/functional/apps/settings/_index_pattern_results_sort.js

@@ -51,7 +51,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
           return col.selector();
         })
         .then(function (rowText) {
-          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-ascending.png`);
+          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-ascending`);
           expect(rowText).to.be(col.first);
         });
       });
@@ -65,7 +65,7 @@ bdd.describe('index result field sort', function describeIndexTests() {
           return col.selector();
         })
         .then(function (rowText) {
-          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-descending.png`);
+          common.saveScreenshot(`Settings-indices-column-${col.heading}-sort-descending`);
           expect(rowText).to.be(col.last);
         });
       });

test/functional/apps/visualize/_metric_chart.js

@@ -14,7 +14,6 @@ bdd.describe('visualize app', function describeIndexTests() {
   var toTime = '2015-09-23 18:31:44.000';
 
   bdd.before(function () {
-    var vizName1 = 'Visualization MetricChart';
     common.debug('navigateToApp visualize');
     return common.navigateToApp('visualize')
     .then(function () {

utilities/compareScreenshots.js

@@ -1,43 +0,0 @@
-
-const fs = require('fs');
-const path = require('path');
-const imageDiff = require('image-diff');
-const mkdirp = require('mkdirp');
-
-function compareScreenshots() {
-  const SCREENSHOTS_DIR = 'test/screenshots';
-  const BASELINE_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'baseline');
-  const DIFF_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'diff');
-  const SESSION_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'session');
-
-  // We don't need to create the baseline dir because it's committed.
-  mkdirp.sync(DIFF_SCREENSHOTS_DIR);
-  mkdirp.sync(SESSION_SCREENSHOTS_DIR);
-
-  fs.readdir(SESSION_SCREENSHOTS_DIR, (readDirError, files) => {
-    const screenshots = files.filter(file => file.indexOf('.png') !== -1);
-
-    screenshots.forEach(screenshot => {
-      const sessionImagePath = path.resolve(SESSION_SCREENSHOTS_DIR, screenshot);
-      const baselineImagePath = path.resolve(BASELINE_SCREENSHOTS_DIR, screenshot);
-      const diffImagePath = path.resolve(DIFF_SCREENSHOTS_DIR, screenshot);
-
-      imageDiff.getFullResult({
-        actualImage: sessionImagePath,
-        expectedImage: baselineImagePath,
-        diffImage: diffImagePath,
-        shadow: true,
-      }, (comparisonError, result) => {
-        if (comparisonError) {
-          throw comparisonError;
-        }
-
-        const change = result.percentage;
-        const changePercentage = (change * 100).toFixed(2);
-        console.log(`${screenshot} has changed by ${changePercentage}%`);
-      });
-    });
-  });
-}
-
-compareScreenshots();

utilities/templates/visual_regression_gallery.dot

@@ -0,0 +1,296 @@
+<!DOCTYPE html>
+<html lang="en" xmlns="http://www.w3.org/1999/xhtml">
+<head>
+  <title>Kibana Visual Regression Gallery</title>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <style>
+
+  body {
+    padding: 40px;
+    background-color: #f6f6f6;
+    font-family: 'Helvetica', Arial, sans-serif;
+    min-width: 2000px;
+  }
+
+  .title {
+    margin-bottom: 6px;
+    font-size: 28px;
+  }
+
+  .meta {
+    font-size: 14px;
+    margin-bottom: 30px;
+  }
+
+  .comparison {
+    padding: 20px;
+    margin-bottom: 40px;
+    background-color: #ffffff;
+  }
+
+    .comparison__title {
+      cursor: pointer;
+      padding-bottom: 10px;
+      margin-bottom: 10px;
+      font-size: 32px;
+      color: #2d2d2d;
+      border-bottom: 2px solid #ecf0f1;
+    }
+
+    .comparison__title:hover {
+      background-color: #e9e9e9;
+    }
+
+      .comparison__percent {
+        opacity: 0.4;
+      }
+
+    .comparison__subTitle {
+      display: -webkit-box;
+      display: -ms-flexbox;
+      display: flex;
+      padding-bottom: 10px;
+      font-weight: 700;
+      font-size: 14px;
+      color: #9c9c9c;
+      border-bottom: 1px solid #ecf0f1;
+    }
+
+    .comparison__body {
+      display: -webkit-box;
+      display: -ms-flexbox;
+      display: flex;
+      padding-top: 20px;
+      max-height: 500px;
+      overflow: auto;
+    }
+
+  .comparison--collapsed .comparison__title {
+    margin-bottom: 0;
+    border-bottom: 0;
+  }
+
+  .comparison--collapsed .comparison__subTitle {
+    display: none;
+  }
+
+  .comparison--collapsed .comparison__body {
+    display: none;
+  }
+
+  .comparison--warning {
+  }
+
+  .comparisonScreenshot {
+  }
+
+  .comparisonScreenshot + .comparisonScreenshot {
+    margin-left: 20px;
+  }
+
+    .comparisonScreenshot__image {
+      cursor: pointer;
+      width: 900px;
+      margin-bottom: 10px;
+      box-shadow: 0 0 5px rgba(0, 0, 0, 0.3);
+    }
+
+  .comparisonScreenshotTitle {
+    width: 900px;
+    background-color: white;
+  }
+
+  .comparisonScreenshotTitle + .comparisonScreenshotTitle {
+    margin-left: 20px;
+  }
+
+  .comparisonScreenshotTitle--session,
+  .comparisonScreenshotTitle--baseline {
+    cursor: pointer;
+  }
+
+  .comparisonScreenshotTitle--session:hover,
+  .comparisonScreenshotTitle--baseline:hover {
+    background-color: #e9e9e9;
+  }
+
+  </style>
+
+</head>
+
+<body>
+
+  <div class="title">
+    Kibana Visual Regression Gallery
+  </div>
+
+  <div class="meta">
+    {{=it.branch}} - {{=it.date}}
+  </div>
+
+  {{~it.comparisons :comparison:index}}
+  <div class="comparison{{? comparison.change <= it.hiddenThreshold }} comparison--collapsed{{?}} {{? comparison.change >= it.warningThreshold }} comparison--warning{{?}}">
+    <div class="comparison__title">
+      <span class="comparison__percent">({{=comparison.percentage}}%)</span> {{=comparison.name}}
+    </div>
+
+    <div class="comparison__subTitle">
+      <div class="comparisonScreenshotTitle">Diff</div>
+      <div
+        class="comparisonScreenshotTitle comparisonScreenshotTitle--session"
+        style="display: none"
+      >
+        Session
+      </div>
+      <div
+        class="comparisonScreenshotTitle comparisonScreenshotTitle--baseline"
+      >
+        Baseline
+      </div>
+    </div>
+
+    <div class="comparison__body">
+      <!-- Diff first, so we can scan for regressions. -->
+      <div class="comparisonScreenshot">
+        <img
+          class="comparisonScreenshot__image"
+          src="data:image/png;base64,{{=comparison.imageData.diff}}"
+        />
+      </div>
+
+      <!-- Session, see what happened during the test. -->
+      <div
+        class="comparisonScreenshot comparisonScreenshot--session"
+        style="display: none"
+      >
+        <img
+          class="comparisonScreenshot__image"
+          src="data:image/png;base64,{{=comparison.imageData.session}}"
+        />
+      </div>
+
+      <!-- Baseline, see what it's supposed to be. -->
+      <div class="comparisonScreenshot comparisonScreenshot--baseline">
+        <img
+          class="comparisonScreenshot__image"
+          src="data:image/png;base64,{{=comparison.imageData.baseline}}"
+        />
+      </div>
+    </div>
+  </div>
+  {{~}}
+
+<script>
+
+function start() {
+
+  function addClass(el, className) {
+    el.className += ' ' + className;
+  }
+
+  function removeClass(el, className) {
+    var classes = el.className.split(' ');
+    var classIndex = classes.indexOf(className);
+    if (classIndex !== -1) {
+      classes.splice(classIndex, 1);
+    }
+    el.className = classes.join(' ');
+  }
+
+  function onClickComparisonTitle(comparison) {
+    return function() {
+      var collapsedClass = 'comparison--collapsed';
+      if (comparison.className.indexOf(collapsedClass) === -1) {
+        addClass(comparison, collapsedClass);
+      } else {
+        removeClass(comparison, collapsedClass);
+      }
+    };
+  }
+
+  function onClickBaselineTitle(
+    baselineImage,
+    baselineTitle,
+    sessionImage,
+    sessionTitle
+  ) {
+    return function() {
+      // Hide the baseline image.
+      baselineImage.setAttribute('style', 'display: none');
+      baselineTitle.setAttribute('style', 'display: none');
+
+      // Show the session image.
+      sessionImage.setAttribute('style', '');
+      sessionTitle.setAttribute('style', '');
+    }
+  }
+
+  function onClickSessionTitle(
+    baselineImage,
+    baselineTitle,
+    sessionImage,
+    sessionTitle
+  ) {
+    return function() {
+      // Show the baseline image.
+      baselineImage.setAttribute('style', '');
+      baselineTitle.setAttribute('style', '');
+
+      // Hide the session image.
+      sessionImage.setAttribute('style', 'display: none');
+      sessionTitle.setAttribute('style', 'display: none');
+    }
+  }
+
+  function onClickImage(image) {
+    return function() {
+      const url = image.getAttribute('src');
+      window.open(url, '_blank');
+    }
+  }
+
+  var comparisons = document.querySelectorAll('.comparison');
+
+  for (var i = 0; i < comparisons.length; i++) {
+    var comparison = comparisons[i];
+    var comparisonTitle = comparison.querySelector('.comparison__title');
+    comparisonTitle.onclick = onClickComparisonTitle(comparison);
+
+    var baselineScreenshot = comparison.querySelector('.comparisonScreenshot--baseline');
+    var baselineTitle = comparison.querySelector('.comparisonScreenshotTitle--baseline');
+    var sessionScreenshot = comparison.querySelector('.comparisonScreenshot--session');
+    var sessionTitle = comparison.querySelector('.comparisonScreenshotTitle--session');
+
+    baselineTitle.onclick = onClickBaselineTitle(
+      baselineScreenshot,
+      baselineTitle,
+      sessionScreenshot,
+      sessionTitle
+    );
+
+    sessionTitle.onclick = onClickSessionTitle(
+      baselineScreenshot,
+      baselineTitle,
+      sessionScreenshot,
+      sessionTitle
+    );
+
+    const baselineImage = baselineScreenshot.querySelector('img');
+    baselineImage.onclick = onClickImage(baselineImage);
+
+    const sessionImage = sessionScreenshot.querySelector('img');
+    sessionImage.onclick = onClickImage(sessionImage);
+  }
+}
+
+document.onreadystatechange = function () {
+  if (document.readyState === 'complete') {
+    start();
+  }
+};
+
+</script>
+
+</body>
+</html>

utilities/visual_regression.js

@@ -0,0 +1,130 @@
+
+import bluebird, {
+  fromNode,
+  promisify,
+} from 'bluebird';
+import dot from 'dot';
+import fs from 'fs';
+import path from 'path';
+import imageDiff from 'image-diff';
+import mkdirp from 'mkdirp';
+import moment from 'moment';
+import SimpleGit from 'simple-git';
+
+const readDirAsync = promisify(fs.readdir);
+const readFileAsync = promisify(fs.readFile);
+const writeFileAsync = promisify(fs.writeFile);
+
+// Preserve whitespace in our HTML output.
+dot.templateSettings.strip = false;
+
+const templates = dot.process({
+  path: path.resolve('./utilities/templates')
+});
+
+function buildGallery(comparisons) {
+  const simpleGit = new SimpleGit();
+  const asyncBranch = promisify(simpleGit.branch, simpleGit);
+
+  return asyncBranch().then(data => {
+    const branch = data.current;
+
+    const html = templates.visual_regression_gallery({
+      date: moment().format('MMMM Do YYYY, h:mm:ss a'),
+      branch,
+      hiddenThreshold: 0,
+      warningThreshold: 0.03,
+      comparisons,
+    });
+
+    return writeFileAsync(
+      path.resolve('./test/screenshots/visual_regression_gallery.html'),
+      html
+    );
+  });
+}
+
+async function compareScreenshots() {
+  const SCREENSHOTS_DIR = 'test/screenshots';
+  const BASELINE_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'baseline');
+  const DIFF_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'diff');
+  const SESSION_SCREENSHOTS_DIR = path.resolve(SCREENSHOTS_DIR, 'session');
+
+  // We don't need to create the baseline dir because it's committed.
+  mkdirp.sync(DIFF_SCREENSHOTS_DIR);
+  mkdirp.sync(SESSION_SCREENSHOTS_DIR);
+  const files = await readDirAsync(SESSION_SCREENSHOTS_DIR);
+  const screenshots = files.filter(file => file.indexOf('.png') !== -1);
+
+  // We'll use this data to build a screenshot gallery in HTML.
+  return await bluebird.map(screenshots, async screenshot => {
+    // We're going to load image data and cache it in this object.
+    const comparison = {
+      name: screenshot,
+      change: undefined,
+      percentage: undefined,
+      imageData: {
+        session: undefined,
+        baseline: undefined,
+        diff: undefined,
+      }
+    };
+
+    const sessionImagePath = path.resolve(
+      SESSION_SCREENSHOTS_DIR,
+      screenshot
+    );
+
+    const baselineImagePath = path.resolve(
+      BASELINE_SCREENSHOTS_DIR,
+      screenshot
+    );
+
+    const diffImagePath = path.resolve(
+      DIFF_SCREENSHOTS_DIR,
+      screenshot
+    );
+
+    // Diff the images asynchronously.
+    const diffResult = await fromNode((cb) => {
+      imageDiff.getFullResult({
+        actualImage: sessionImagePath,
+        expectedImage: baselineImagePath,
+        diffImage: diffImagePath,
+        shadow: true,
+      }, cb);
+    });
+
+    const change = diffResult.percentage;
+    const changePercentage = (change * 100).toFixed(2);
+    console.log(`(${changePercentage}%) ${screenshot}`);
+    comparison.percentage = changePercentage;
+    comparison.change = change;
+
+    // Once the images have been diffed, we can load and store the image data.
+    comparison.imageData.session =
+      await readFileAsync(sessionImagePath, 'base64');
+
+    comparison.imageData.baseline =
+      await readFileAsync(baselineImagePath, 'base64');
+
+    comparison.imageData.diff =
+      await readFileAsync(diffImagePath, 'base64');
+
+    return comparison;
+  });
+}
+
+module.exports = {
+  run: done => {
+    compareScreenshots().then(screenshotComparisons => {
+      // Once all of the data has been loaded, we can build the gallery.
+      buildGallery(screenshotComparisons).then(() => {
+        done();
+      });
+    }, error => {
+      console.error(error);
+      done(false);
+    });
+  }
+};