Fix visual baseline job (#59348) (#59470)

* Establish Percy baselines

* move Jenkinsfile changed back to `.ci` directory

* rename xpack workers

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.ci/Jenkinsfile_visual_baseline

@@ -6,13 +6,15 @@ kibanaLibrary.load()
 kibanaPipeline(timeoutMinutes: 120) {
   catchError {
     parallel([
-      workers.base(name: 'oss-visualRegression', label: 'linux && immutable') {
-        kibanaPipeline.buildOss()
-        kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')
+      'oss-visualRegression': {
+        workers.ci(name: 'oss-visualRegression', label: 'linux && immutable', ramDisk: false) {
+          kibanaPipeline.functionalTestProcess('oss-visualRegression', './test/scripts/jenkins_visual_regression.sh')(1)
+        }
       },
-      workers.base(name: 'xpack-visualRegression', label: 'linux && immutable') {
-        kibanaPipeline.buildXpack()
-        kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')
+      'xpack-visualRegression': {
+        workers.ci(name: 'xpack-visualRegression', label: 'linux && immutable', ramDisk: false) {
+          kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')(1)
+        }
       },
     ])
   }

test/functional/services/elastic_chart.ts

@@ -42,11 +42,11 @@ export function ElasticChartProvider({ getService }: FtrProviderContext) {
       return Number(renderingCount);
     }
 
-    public async waitForRenderingCount(dataTestSubj: string, previousCount = 1) {
-      await retry.waitFor(`rendering count to be equal to [${previousCount + 1}]`, async () => {
+    public async waitForRenderingCount(dataTestSubj: string, minimumCount: number) {
+      await retry.waitFor(`rendering count to be equal to [${minimumCount}]`, async () => {
         const currentRenderingCount = await this.getVisualizationRenderingCount(dataTestSubj);
         log.debug(`-- currentRenderingCount=${currentRenderingCount}`);
-        return currentRenderingCount === previousCount + 1;
+        return currentRenderingCount >= minimumCount;
       });
     }
   }

test/scripts/jenkins_visual_regression.sh

@@ -1,10 +1,18 @@
 #!/usr/bin/env bash
 
-source test/scripts/jenkins_test_setup_xpack.sh
+source src/dev/ci_setup/setup_env.sh
 source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
 
-checks-reporter-with-killswitch "Kibana visual regression tests" \
-  yarn run percy exec -t 500 \
+echo " -> building and extracting OSS Kibana distributable for use in functional tests"
+node scripts/build --debug --oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+echo " -> running visual regression tests from kibana directory"
+checks-reporter-with-killswitch "X-Pack visual regression tests" \
+  yarn percy exec -t 500 \
   node scripts/functional_tests \
     --debug --bail \
     --kibana-install-dir "$installDir" \

test/scripts/jenkins_xpack_visual_regression.sh

@@ -1,11 +1,21 @@
 #!/usr/bin/env bash
 
-source test/scripts/jenkins_test_setup_xpack.sh
+source src/dev/ci_setup/setup_env.sh
 source "$KIBANA_DIR/src/dev/ci_setup/setup_percy.sh"
 
+echo " -> building and extracting default Kibana distributable"
+cd "$KIBANA_DIR"
+node scripts/build --debug --no-oss
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+echo " -> running visual regression tests from x-pack directory"
+cd "$XPACK_DIR"
 checks-reporter-with-killswitch "X-Pack visual regression tests" \
-  yarn run percy exec -t 500 \
+  yarn percy exec -t 500 \
   node scripts/functional_tests \
     --debug --bail \
-    --kibana-install-dir "$KIBANA_INSTALL_DIR" \
-    --config test/visual_regression/config.js;
+    --kibana-install-dir "$installDir" \
+    --config test/visual_regression/config.ts;

test/visual_regression/services/visual_testing/visual_testing.ts

@@ -71,6 +71,13 @@ export async function VisualTestingProvider({ getService }: FtrProviderContext)
 
   return new (class VisualTesting {
     public async snapshot(options: SnapshotOptions = {}) {
+      if (process.env.DISABLE_VISUAL_TESTING) {
+        log.warning(
+          'Capturing of percy snapshots disabled, would normally capture a snapshot here!'
+        );
+        return;
+      }
+
       log.debug('Capturing percy snapshot');
 
       if (!currentTest) {

test/visual_regression/tests/discover/chart_visualization.ts

@@ -19,8 +19,9 @@
 
 import expect from '@kbn/expect';
 
-export default function({ getService, getPageObjects }) {
-  const log = getService('log');
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const esArchiver = getService('esArchiver');
   const browser = getService('browser');
@@ -34,58 +35,56 @@ export default function({ getService, getPageObjects }) {
 
   describe('discover', function describeIndexTests() {
     before(async function() {
-      log.debug('load kibana index with default index pattern');
       await esArchiver.load('discover');
 
       // and load a set of makelogs data
       await esArchiver.loadIfNeeded('logstash_functional');
       await kibanaServer.uiSettings.replace(defaultSettings);
-      log.debug('discover');
       await PageObjects.common.navigateToApp('discover');
       await PageObjects.timePicker.setDefaultAbsoluteRange();
     });
 
+    after(function unloadMakelogs() {
+      return esArchiver.unload('logstash_functional');
+    });
+
+    async function refreshDiscover() {
+      await browser.refresh();
+      await PageObjects.header.awaitKibanaChrome();
+      await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+      await PageObjects.discover.waitUntilSearchingHasFinished();
+      await PageObjects.discover.waitForChartLoadingComplete(1);
+    }
+
+    async function takeSnapshot() {
+      await refreshDiscover();
+      await visualTesting.snapshot({
+        show: ['discoverChart'],
+      });
+    }
+
     describe('query', function() {
       this.tags(['skipFirefox']);
-      let renderCounter = 0;
 
       it('should show bars in the correct time zone', async function() {
         await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
         await PageObjects.discover.waitUntilSearchingHasFinished();
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('should show correct data for chart interval Hourly', async function() {
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Hourly');
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('should show correct data for chart interval Daily', async function() {
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Daily');
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('should show correct data for chart interval Weekly', async function() {
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Weekly');
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('browser back button should show previous interval Daily', async function() {
@@ -94,57 +93,31 @@ export default function({ getService, getPageObjects }) {
           const actualInterval = await PageObjects.discover.getChartInterval();
           expect(actualInterval).to.be('Daily');
         });
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('should show correct data for chart interval Monthly', async function() {
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Monthly');
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('should show correct data for chart interval Yearly', async function() {
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Yearly');
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
 
       it('should show correct data for chart interval Auto', async function() {
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
         await PageObjects.discover.setChartInterval('Auto');
-        await PageObjects.discover.waitForChartLoadingComplete(++renderCounter);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
     });
 
     describe('time zone switch', () => {
       it('should show bars in the correct time zone after switching', async function() {
         await kibanaServer.uiSettings.replace({ 'dateFormat:tz': 'America/Phoenix' });
-        await browser.refresh();
-        await PageObjects.header.awaitKibanaChrome();
+        await refreshDiscover();
         await PageObjects.timePicker.setDefaultAbsoluteRange();
-        await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
-        await PageObjects.discover.waitUntilSearchingHasFinished();
-        await PageObjects.discover.waitForChartLoadingComplete(1);
-        await visualTesting.snapshot({
-          show: ['discoverChart'],
-        });
+        await takeSnapshot();
       });
     });
   });

test/visual_regression/tests/discover/index.ts

@@ -18,12 +18,12 @@
  */
 
 import { DEFAULT_OPTIONS } from '../../services/visual_testing/visual_testing';
+import { FtrProviderContext } from '../../ftr_provider_context';
 
 // Width must be the same as visual_testing or canvas image widths will get skewed
 const [SCREEN_WIDTH] = DEFAULT_OPTIONS.widths || [];
 
-export default function({ getService, loadTestFile }) {
-  const esArchiver = getService('esArchiver');
+export default function({ getService, loadTestFile }: FtrProviderContext) {
   const browser = getService('browser');
 
   describe('discover app', function() {
@@ -33,10 +33,6 @@ export default function({ getService, loadTestFile }) {
       return browser.setWindowSize(SCREEN_WIDTH, 1000);
     });
 
-    after(function unloadMakelogs() {
-      return esArchiver.unload('logstash_functional');
-    });
-
     loadTestFile(require.resolve('./chart_visualization'));
   });
 }

x-pack/test/visual_regression/config.ts

@@ -4,9 +4,11 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { services as ossVisualRegressionServices } from '../../../test/visual_regression/services';
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
 
-export default async function({ readConfigFile }) {
+import { services } from './services';
+
+export default async function({ readConfigFile }: FtrConfigProviderContext) {
   const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
 
   return {
@@ -19,10 +21,7 @@ export default async function({ readConfigFile }) {
       require.resolve('./tests/infra'),
     ],
 
-    services: {
-      ...functionalConfig.get('services'),
-      visualTesting: ossVisualRegressionServices.visualTesting,
-    },
+    services,
 
     junit: {
       reportName: 'X-Pack Visual Regression Tests',

x-pack/test/visual_regression/ftr_provider_context.d.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
+
+import { pageObjects } from './page_objects';
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;

x-pack/test/visual_regression/page_objects.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { pageObjects } from '../functional/page_objects';
+
+export { pageObjects };

x-pack/test/visual_regression/services.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { services as ossVisualRegressionServices } from '../../../test/visual_regression/services';
+import { services as functionalServices } from '../functional/services';
+
+export const services = {
+  ...functionalServices,
+  visualTesting: ossVisualRegressionServices.visualTesting,
+};

x-pack/test/visual_regression/tests/login_page.ts

@@ -4,7 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export default function({ getService, getPageObjects }) {
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const visualTesting = getService('visualTesting');
   const testSubjects = getService('testSubjects');