Add ability to whitelist visible elements in percy (#45733) (#49881)

* add ability to whitelist visible elements in percy

* allow white and blacklisting elements in visual tests

* remove unnecessary webElement methods

* refactor snapshot options to use show and hide

* refactor add/remove and visual test helpers

* [percy] rework css rules to allow hiding inside shown elements

* [percy] adjust logic to support showing inside hiding

* attach styles to hide percy when capturing the snapshot

* refactor in order to make sure all logic is executed if snapshot fails

* remove sleeps

* add back skipFirefox tag

src/legacy/core_plugins/kibana/public/discover/index.html

@@ -170,6 +170,7 @@
                 chart-data="histogramData"
                 timefilter-update-handler="timefilterUpdateHandler"
                 watch-depth="reference"
+                data-test-subj="discoverChart"
               ></discover-histogram>
             </section>
 

src/legacy/core_plugins/kibana/public/index.scss

@@ -36,10 +36,3 @@
 // Dashboard styles
 // MUST STAY AT THE BOTTOM BECAUSE OF DARK THEME IMPORTS
 @import './dashboard/index';
-
-// helper class that hides elements when rendered in percy
-@media only percy {
-  .hideInPercy {
-    visibility: hidden;
-  }
-}

test/visual_regression/services/visual_testing/take_percy_snapshot.js

@@ -20,42 +20,88 @@
 import { readFileSync } from 'fs';
 import { agentJsFilename } from '@percy/agent/dist/utils/sdk-utils';
 
-export function takePercySnapshot() {
+export function takePercySnapshot(show, hide) {
   if (!window.PercyAgent) {
     return false;
   }
 
-  const agent = new window.PercyAgent({
-    handleAgentCommunication: false
-  });
+  // add percy styles to hide/show specific elements
+  const styleElement = document.createElement('style');
+  styleElement.appendChild(document.createTextNode(`
+    .hideInPercy {
+      visibility: hidden;
 
-  const queryAll = selector => [
-    ...document.querySelectorAll(selector)
-  ];
+      .showInPercy {
+        visibility: visible;
+      }
+    }
 
-  // array of canvas/image replacements
-  const replacements = [];
+    .showInPercy {
+      visibility: visible;
+
+      .hideInPercy {
+        visibility: hidden;
+      }
+    }
+  `));
+  document.head.appendChild(styleElement);
+
+  const add = (selectors, className) => {
+    for (const selector of selectors) {
+      for (const element of document.querySelectorAll(selector)) {
+        element.classList.add(className);
+      }
+    }
+  };
+
+  const remove = (selectors, className) => {
+    for (const selector of selectors) {
+      for (const element of document.querySelectorAll(selector)) {
+        element.classList.remove(className);
+      }
+    }
+  };
+
+  // set Percy visibility on elements
+  add(hide, 'hideInPercy');
+  if (show.length > 0) {
+    // hide the body by default
+    add(['body'], 'hideInPercy');
+    add(show, 'showInPercy');
+  }
 
   // convert canvas elements into static images
-  for (const canvas of queryAll('canvas')) {
+  const replacements = [];
+  for (const canvas of document.querySelectorAll('canvas')) {
     const image = document.createElement('img');
+    image.classList.value = canvas.classList.value;
     image.src = canvas.toDataURL();
     image.style.cssText = window.getComputedStyle(canvas).cssText;
     canvas.parentElement.replaceChild(image, canvas);
     replacements.push({ canvas, image });
   }
 
-  // cache the dom snapshot containing the images
-  const snapshot = agent.snapshot(document, {
-    widths: [document.documentElement.clientWidth]
-  });
+  try {
+    const agent = new window.PercyAgent({
+      handleAgentCommunication: false
+    });
 
-  // restore replaced canvases
-  for (const { image, canvas } of replacements) {
-    image.parentElement.replaceChild(canvas, image);
+    // cache the dom snapshot containing the images
+    return agent.snapshot(document, {
+      widths: [document.documentElement.clientWidth]
+    });
+  } finally {
+    // restore replaced canvases
+    for (const { image, canvas } of replacements) {
+      image.parentElement.replaceChild(canvas, image);
+    }
+
+    // restore element visibility
+    document.head.removeChild(styleElement);
+    remove(['body'], 'hideInPercy');
+    remove(show, 'showInPercy');
+    remove(hide, 'hideInPercy');
   }
-
-  return snapshot;
 }
 
 export const takePercySnapshotWithAgent = `

test/visual_regression/services/visual_testing/visual_testing.ts

@@ -18,8 +18,10 @@
  */
 
 import { postSnapshot } from '@percy/agent/dist/utils/sdk-utils';
-
 import { Test } from 'mocha';
+import _ from 'lodash';
+
+import testSubjSelector from '@kbn/test-subj-selector';
 
 import { pkg } from '../../../../src/legacy/utils/package_json';
 import { FtrProviderContext } from '../../ftr_provider_context';
@@ -31,6 +33,21 @@ export const DEFAULT_OPTIONS = {
   widths: [1200],
 };
 
+export interface SnapshotOptions {
+  /**
+   * name to append to visual test name
+   */
+  name?: string;
+  /**
+   * test subject selectiors to __show__ in screenshot
+   */
+  show?: string[];
+  /**
+   * test subject selectiors to __hide__ in screenshot
+   */
+  hide?: string[];
+}
+
 export async function VisualTestingProvider({ getService }: FtrProviderContext) {
   const browser = getService('browser');
   const log = getService('log');
@@ -53,18 +70,21 @@ export async function VisualTestingProvider({ getService }: FtrProviderContext)
   }
 
   return new (class VisualTesting {
-    public async snapshot(name?: string) {
+    public async snapshot(options: SnapshotOptions = {}) {
       log.debug('Capturing percy snapshot');
 
       if (!currentTest) {
         throw new Error('unable to determine current test');
       }
 
-      const [domSnapshot, url] = await Promise.all([this.getSnapshot(), browser.getCurrentUrl()]);
-
+      const [domSnapshot, url] = await Promise.all([
+        this.getSnapshot(options.show, options.hide),
+        browser.getCurrentUrl(),
+      ]);
       const stats = getStats(currentTest);
       stats.snapshotCount += 1;
 
+      const { name } = options;
       const success = await postSnapshot({
         name: `${currentTest.fullTitle()} [${name ? name : stats.snapshotCount}]`,
         url,
@@ -78,11 +98,21 @@ export async function VisualTestingProvider({ getService }: FtrProviderContext)
       }
     }
 
-    private async getSnapshot() {
-      const snapshot = await browser.execute<[], string | false>(takePercySnapshot);
+    private async getSnapshot(show: string[] = [], hide: string[] = []) {
+      const showSelectors = show.map(testSubjSelector);
+      const hideSelectors = hide.map(testSubjSelector);
+      const snapshot = await browser.execute<[string[], string[]], string | false>(
+        takePercySnapshot,
+        showSelectors,
+        hideSelectors
+      );
       return snapshot !== false
         ? snapshot
-        : await browser.execute<[], string>(takePercySnapshotWithAgent);
+        : await browser.execute<[string[], string[]], string>(
+            takePercySnapshotWithAgent,
+            showSelectors,
+            hideSelectors
+          );
     }
   })();
 }

test/visual_regression/tests/discover/chart_visualization.js

@@ -52,28 +52,36 @@ export default function ({ getService, getPageObjects }) {
 
       it('should show bars in the correct time zone', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await visualTesting.snapshot();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
+        });
       });
 
       it('should show correct data for chart interval Hourly', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Hourly');
-        await visualTesting.snapshot();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
+        });
       });
 
       it('should show correct data for chart interval Daily', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Daily');
-        await retry.try(async () => {
-          await visualTesting.snapshot();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
         });
       });
 
       it('should show correct data for chart interval Weekly', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Weekly');
-        await retry.try(async () => {
-          await visualTesting.snapshot();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
         });
       });
 
@@ -84,25 +92,37 @@ export default function ({ getService, getPageObjects }) {
           expect(actualInterval).to.be('Daily');
         });
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await visualTesting.snapshot();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
+        });
       });
 
       it('should show correct data for chart interval Monthly', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Monthly');
-        await visualTesting.snapshot();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
+        });
       });
 
       it('should show correct data for chart interval Yearly', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Yearly');
-        await visualTesting.snapshot();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
+        });
       });
 
       it('should show correct data for chart interval Auto', async function () {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Auto');
-        await visualTesting.snapshot();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
+        });
       });
     });
 
@@ -113,11 +133,11 @@ export default function ({ getService, getPageObjects }) {
         await PageObjects.header.awaitKibanaChrome();
         await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await retry.try(async function () {
-          await visualTesting.snapshot();
+        await PageObjects.discover.waitUntilSearchingHasFinished();
+        await visualTesting.snapshot({
+          show: ['discoverChart'],
         });
       });
-
     });
   });
 }