Turn reporting snapshot tests back on with new baselines (#21441) (#21696)

* Turn reporting snapshot tests back on with new baselines

* Use snapshots generated from chromium.  Bump the threshold so phantom comparisons pass

* Increase timeouts because of chromium functional tests.

* remove accidentally checked in session snapshots

* add a better vis baseline, run once through at 0 to check expected threshold

* turn on verbose logs

* Use threshold, turn debug mode on in jenkins

* Wrap both convert pages in the retry

* Update baselines

Preserve layout one got messed up - it was actually a print layout.

* bump threshold even more

test/scripts/jenkins_xpack.sh

@@ -37,6 +37,6 @@ tar -xzf "$linuxBuild" -C "$installDir" --strip=1
 
 echo " -> Running functional and api tests"
 cd "$XPACK_DIR"
-xvfb-run node scripts/functional_tests --bail --kibana-install-dir "$installDir" --esFrom=source
+xvfb-run node scripts/functional_tests --debug --bail --kibana-install-dir "$installDir" --esFrom=source
 echo ""
 echo ""

x-pack/test/reporting/configs/phantom_functional.js

@@ -21,6 +21,7 @@ export default async function ({ readConfigFile }) {
       serverArgs: [
         ...functionalConfig.kbnTestServer.serverArgs,
         `--xpack.reporting.capture.browser.type=phantom`,
+        `--logging.verbose=true`,
       ],
     },
   };

x-pack/test/reporting/functional/lib/compare_pdfs.js

@@ -77,24 +77,34 @@ export async function checkIfPdfsMatch(actualPdfPath, baselinePdfPath, screensho
 
   let pageNum = 0;
   let diffTotal = 0;
+  // Ran across an instance where the page conversion failed with `Failed to convert page to image` for no known
+  // reason. Seeing if a loop will resolve these flaky errors.
+  let failCount = 0;
   while (true) {
     let expectedPagePng;
+    let actualPagePng;
     try {
+      log.debug(`Converting expected pdf page ${pageNum} to png`);
       expectedPagePng = await expectedPdfImage.convertPage(pageNum);
+      log.debug(`Converting actual pdf page ${pageNum} to png`);
+      actualPagePng = await actualPdfImage.convertPage(pageNum);
     } catch (e) {
+      log.error(`Error caught while converting pdf page ${pageNum} to png: ${e.message}`);
       if (JSON.stringify(e).indexOf('Requested FirstPage is greater than the number of pages in the file') >= 0) {
         break;
       } else {
-        log.error('PDF to image conversion failed. Make sure you have the required dependencies ' +
-                  'imagemagick, ghostscript and poppler installed.');
-        throw e;
+        if (failCount < 3) {
+          log.error(`${failCount}: Will try conversion again...`);
+          failCount++;
+          continue;
+        } else {
+          log.error(`Failed ${failCount} times, throwing error`);
+          throw e;
+        }
       }
     }
 
-    const actualPagePng = await actualPdfImage.convertPage(pageNum);
-
     const diffPngPath = path.resolve(failureDirectoryPath, `${baselinePdfFileName}-${pageNum}.png`);
-
     diffTotal += await comparePngs(actualPagePng, expectedPagePng, diffPngPath, log);
     pageNum++;
   }

x-pack/test/reporting/functional/reporting.js

@@ -80,10 +80,11 @@ export default function ({ getService, getPageObjects }) {
       });
 
       describe('Print Layout', () => {
-        it.skip('matches baseline report', async function () {
+        it('matches baseline report', async function () {
           // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
-          // function is taking about 15 seconds per comparison in jenkins.
-          this.timeout(180000);
+          // function is taking about 15 seconds per comparison in jenkins. Also Chromium takes a lot longer to generate a
+          // report than phantom.
+          this.timeout(360000);
 
           await PageObjects.dashboard.clickEdit();
           await PageObjects.reporting.setTimepickerInDataRange();
@@ -116,10 +117,11 @@ export default function ({ getService, getPageObjects }) {
           expect(diffCount).to.be(0);
         });
 
-        it.skip('matches same baseline report with margins turned on', async function () {
+        it('matches same baseline report with margins turned on', async function () {
           // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
-          // function is taking about 15 seconds per comparison in jenkins.
-          this.timeout(180000);
+          // function is taking about 15 seconds per comparison in jenkins. Also Chromium takes a lot longer to generate a
+          // report than phantom.
+          this.timeout(360000);
 
           await PageObjects.dashboard.clickEdit();
           await PageObjects.dashboard.useMargins(true);
@@ -145,11 +147,12 @@ export default function ({ getService, getPageObjects }) {
       });
 
       describe('Preserve Layout', () => {
-        it.skip('matches baseline report', async function () {
+        it('matches baseline report', async function () {
 
           // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
-          // function is taking about 15 seconds per comparison in jenkins.
-          this.timeout(180000);
+          // function is taking about 15 seconds per comparison in jenkins. Also Chromium takes a lot longer to generate a
+          // report than phantom.
+          this.timeout(360000);
 
           await PageObjects.reporting.openReportingPanel();
           await PageObjects.reporting.forceSharedItemsContainerSize({ width: 1405 });
@@ -170,8 +173,11 @@ export default function ({ getService, getPageObjects }) {
             config.get('screenshots.directory'),
             log
           );
-          // After expected OS differences, the diff count came to be around 250k
-          expect(diffCount).to.be.lessThan(250000);
+          // After expected OS differences, the diff count came to be around 350k. Due to
+          // https://github.com/elastic/kibana/issues/21485 this jumped up to something like 368 when
+          // comparing the same baseline for chromium and phantom.
+          expect(diffCount).to.be.lessThan(400000);
+
         });
 
 
@@ -214,11 +220,42 @@ export default function ({ getService, getPageObjects }) {
         });
 
         it('becomes available when saved', async () => {
+          await PageObjects.reporting.setTimepickerInDataRange();
+          await PageObjects.visualize.clickBucket('X-Axis');
+          await PageObjects.visualize.selectAggregation('Date Histogram');
+          await PageObjects.visualize.clickGo();
           await PageObjects.visualize.saveVisualization('my viz');
           await expectEnabledGenerateReportButton();
         });
 
-        it('generates a report', async () => await expectReportCanBeCreated());
+        it('matches baseline report', async function () {
+          // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
+          // function is taking about 15 seconds per comparison in jenkins.
+          this.timeout(180000);
+
+          await PageObjects.reporting.openReportingPanel();
+          await PageObjects.reporting.clickGenerateReportButton();
+          await PageObjects.reporting.clickDownloadReportButton(60000);
+
+          const url = await PageObjects.reporting.getUrlOfTab(1);
+          const reportData = await PageObjects.reporting.getRawPdfReportData(url);
+
+          await PageObjects.reporting.closeTab(1);
+          const reportFileName = 'visualize_print';
+          const sessionReportPath = await writeSessionReport(reportFileName, reportData);
+          const diffCount = await checkIfPdfsMatch(
+            sessionReportPath,
+            getBaselineReportPath(reportFileName),
+            config.get('screenshots.directory'),
+            log
+          );
+          // After expected OS and browser differences, the diff count came up to max 800564 that I saw.
+          // This is pretty bad. https://github.com/elastic/kibana/issues/21486 is filed to lower this
+          // which will be much easier when we only support one browser type (chromium instead of phantom).
+          // The reason this is so high currently is because of a phantom bug:
+          // https://github.com/elastic/kibana/issues/21485
+          expect(diffCount).to.be.lessThan(810000);
+        });
       });
     });
   });