[firefox] Run functional tests in dedicated CI groups (#38416) (#39089)

* 2 jobs for Firefox tests

* review fixes

* make sh files executable

* add check reporter

.ci/jobs.yml

@@ -1,6 +1,7 @@
 JOB:
   - kibana-intake
   - x-pack-intake
+  - kibana-firefoxSmoke
   - kibana-ciGroup1
   - kibana-ciGroup2
   - kibana-ciGroup3
@@ -16,6 +17,7 @@ JOB:
   # - kibana-visualRegression
 
   # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
+  - x-pack-firefoxSmoke
   - x-pack-ciGroup1
   - x-pack-ciGroup2
   - x-pack-ciGroup3

.ci/run.sh

@@ -20,6 +20,9 @@ kibana-ciGroup*)
 kibana-visualRegression*)
   ./test/scripts/jenkins_visual_regression.sh
   ;;
+kibana-firefoxSmoke*)
+  ./test/scripts/jenkins_firefox_smoke.sh
+  ;;
 x-pack-intake)
   ./test/scripts/jenkins_xpack.sh
   ;;
@@ -30,6 +33,9 @@ x-pack-ciGroup*)
 x-pack-visualRegression*)
   ./test/scripts/jenkins_xpack_visual_regression.sh
   ;;
+x-pack-firefoxSmoke*)
+  ./test/scripts/jenkins_xpack_firefox_smoke.sh
+  ;;
 *)
   echo "JOB '$JOB' is not implemented."
   exit 1

test/functional/apps/console/_console.ts

@@ -38,6 +38,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['common', 'console']);
 
   describe('console app', function describeIndexTests() {
+    this.tags('smoke');
     before(async () => {
       log.debug('navigateTo console');
       await PageObjects.common.navigateToApp('console');

test/functional/apps/context/_discover_navigation.js

@@ -31,6 +31,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'discover', 'timePicker']);
 
   describe('context link in discover', function contextSize() {
+    this.tags('smoke');
     before(async function () {
       await PageObjects.common.navigateToApp('discover');
       await PageObjects.timePicker.setAbsoluteRange(TEST_DISCOVER_START_TIME, TEST_DISCOVER_END_TIME);

test/functional/apps/dashboard/dashboard_filtering.js

@@ -34,7 +34,8 @@ export default function ({ getService, getPageObjects }) {
   const dashboardPanelActions = getService('dashboardPanelActions');
   const PageObjects = getPageObjects(['dashboard', 'header', 'visualize']);
 
-  describe('dashboard filtering', async () => {
+  describe('dashboard filtering', async function () {
+    this.tags('smoke');
     before(async () => {
       await PageObjects.dashboard.gotoDashboardLandingPage();
     });

test/functional/apps/dashboard/dashboard_save.js

@@ -23,6 +23,7 @@ export default function ({ getPageObjects }) {
   const PageObjects = getPageObjects(['dashboard', 'header']);
 
   describe('dashboard save', function describeIndexTests() {
+    this.tags('smoke');
     const dashboardName = 'Dashboard Save Test';
     const dashboardNameEnterKey = 'Dashboard Save Test with Enter Key';
 

test/functional/apps/dashboard/panel_controls.js

@@ -33,6 +33,8 @@ export default function ({ getService, getPageObjects }) {
   const dashboardName = 'Dashboard Panel Controls Test';
 
   describe('dashboard panel controls', function viewEditModeTests() {
+    this.tags('smoke');
+
     before(async function () {
       await PageObjects.dashboard.initTests();
       await browser.refresh();

test/functional/apps/dashboard/time_zones.js

@@ -24,7 +24,8 @@ export default function ({ getService, getPageObjects }) {
   const pieChart = getService('pieChart');
   const PageObjects = getPageObjects(['dashboard', 'timePicker', 'settings', 'common']);
 
-  describe('dashboard time zones', () => {
+  describe('dashboard time zones', function () {
+    this.tags('smoke');
     before(async () => {
       await PageObjects.settings.navigateTo();
       await PageObjects.settings.clickKibanaSavedObjects();

test/functional/apps/discover/_field_data.js

@@ -28,6 +28,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']);
 
   describe('discover tab', function describeIndexTests() {
+    this.tags('smoke');
     before(async function () {
       const fromTime = '2015-09-19 06:31:44.000';
       const toTime = '2015-09-23 18:31:44.000';

test/functional/apps/getting_started/index.js

@@ -21,7 +21,7 @@ export default function ({ getService, loadTestFile }) {
   const browser = getService('browser');
 
   describe('Getting Started ', function () {
-    this.tags('ciGroup6');
+    this.tags(['ciGroup6', 'smoke']);
 
     before(async function () {
       await browser.setWindowSize(1200, 800);

test/functional/apps/home/_home.js

@@ -26,6 +26,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'home']);
 
   describe('Kibana takes you home', function describeIndexTests() {
+    this.tags('smoke');
 
     it('clicking on kibana logo should take you to home page', async ()=> {
       await PageObjects.common.navigateToApp('settings');

test/functional/apps/home/_sample_data.js

@@ -27,6 +27,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'header', 'home', 'dashboard', 'timePicker']);
 
   describe('sample data', function describeIndexTests() {
+    this.tags('smoke');
 
     before(async () => {
       await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData');

test/functional/apps/management/_create_index_pattern_wizard.js

@@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['settings', 'common']);
 
   describe('"Create Index Pattern" wizard', function () {
+    this.tags('smoke');
 
     before(async function () {
       // delete .kibana index and then wait for Kibana to re-create it

test/functional/apps/visualize/_experimental_vis.js

@@ -23,7 +23,8 @@ export default ({ getService, getPageObjects }) => {
   const log = getService('log');
   const PageObjects = getPageObjects(['common', 'visualize']);
 
-  describe('visualize app', () => {
+  describe('visualize app', function () {
+    this.tags('smoke');
 
     describe('experimental visualizations', () => {
 

test/functional/apps/visualize/_gauge_chart.js

@@ -26,6 +26,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'visualize', 'timePicker']);
 
   describe('gauge chart', function indexPatternCreation() {
+    this.tags('smoke');
     const fromTime = '2015-09-19 06:31:44.000';
     const toTime = '2015-09-23 18:31:44.000';
 

test/functional/apps/visualize/_heatmap_chart.js

@@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'visualize', 'timePicker']);
 
   describe('heatmap chart', function indexPatternCreation() {
+    this.tags('smoke');
     const vizName1 = 'Visualization HeatmapChart';
     const fromTime = '2015-09-19 06:31:44.000';
     const toTime = '2015-09-23 18:31:44.000';

test/functional/apps/visualize/_inspector.js

@@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }) {
   const PageObjects = getPageObjects(['common', 'visualize', 'timePicker']);
 
   describe('inspector', function describeIndexTests() {
+    this.tags('smoke');
     before(async function () {
       const fromTime = '2015-09-19 06:31:44.000';
       const toTime = '2015-09-23 18:31:44.000';

test/functional/apps/visualize/_tsvb_chart.ts

@@ -30,6 +30,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
   const PageObjects = getPageObjects(['visualize', 'visualBuilder', 'timePicker']);
 
   describe('visual builder', function describeIndexTests() {
+    this.tags('smoke');
     beforeEach(async () => {
       await PageObjects.visualize.navigateToNewVisualization();
       await PageObjects.visualize.clickVisualBuilder();

test/scripts/jenkins_ci_group.sh

@@ -18,4 +18,4 @@ if [ "$CI_GROUP" == "1" ]; then
   cd -;
   yarn run grunt run:pluginFunctionalTestsRelease --from=source;
   yarn run grunt run:interpreterFunctionalTestsRelease;
-fi
+fi

test/scripts/jenkins_firefox_smoke.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
+
+node scripts/build --debug --oss;
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-oss-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+export TEST_BROWSER_HEADLESS=1
+
+checks-reporter-with-killswitch "Firefox smoke test" \
+  node scripts/functional_tests \
+    --bail --debug \
+    --kibana-install-dir "$installDir" \
+    --include-tag "smoke" \
+    --config test/functional/config.firefox.js;

test/scripts/jenkins_xpack_ci_group.sh

@@ -46,4 +46,3 @@ echo ""
 #   --config "test/functional/config.firefox.js"
 # echo ""
 # echo ""
-

test/scripts/jenkins_xpack_firefox_smoke.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -e
+trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT
+
+node scripts/build --debug --no-oss;
+linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+installDir="$PARENT_DIR/install/kibana"
+mkdir -p "$installDir"
+tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+export TEST_BROWSER_HEADLESS=1
+
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack firefox smoke test" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$installDir" \
+    --include-tag "smoke" \
+    --config test/functional/config.firefox.js;

x-pack/test/functional/apps/canvas/smoke_test.js

@@ -14,16 +14,15 @@ export default function canvasSmokeTest({ getService, getPageObjects }) {
   const retry = getService('retry');
   const PageObjects = getPageObjects(['common']);
 
-  describe('smoke test', async () => {
+  describe('smoke test', async function () {
+    this.tags('smoke');
     const workpadListSelector = 'canvasWorkpadLoaderTable canvasWorkpadLoaderWorkpad';
     const testWorkpadId = 'workpad-1705f884-6224-47de-ba49-ca224fe6ec31';
 
     before(async () => {
       // init data
-      await Promise.all([
-        esArchiver.loadIfNeeded('logstash_functional'),
-        esArchiver.load('canvas/default'),
-      ]);
+      await esArchiver.loadIfNeeded('logstash_functional');
+      await esArchiver.load('canvas/default');
 
       // load canvas
       // see also navigateToUrl(app, hash)

x-pack/test/functional/apps/grok_debugger/grok_debugger.js

@@ -11,7 +11,8 @@ export default function ({ getService, getPageObjects }) {
 
   const PageObjects = getPageObjects(['grokDebugger']);
 
-  describe('grok debugger app', () => {
+  describe('grok debugger app', function () {
+    this.tags('smoke');
     before(async () => {
       await esArchiver.load('empty_kibana');
       // Increase window height to ensure "Simulate" button is shown above the

x-pack/test/functional/apps/infra/home_page.ts

@@ -15,7 +15,8 @@ export default ({ getPageObjects, getService }: KibanaFunctionalTestDefaultProvi
   const esArchiver = getService('esArchiver');
   const pageObjects = getPageObjects(['common', 'infraHome']);
 
-  describe('Home page', () => {
+  describe('Home page', function() {
+    this.tags('smoke');
     before(async () => {
       await esArchiver.load('empty_kibana');
     });

x-pack/test/functional/apps/rollup_job/rollup_jobs.js

@@ -12,13 +12,12 @@ export default function ({ getService, getPageObjects }) {
   const log = getService('log');
   const PageObjects = getPageObjects(['security', 'rollup', 'common', 'header']);
 
-  describe('rollup job', async () => {
+  describe('rollup job', async function () {
+    this.tags('smoke');
     before(async () => {
       // init data
-      await Promise.all([
-        esArchiver.loadIfNeeded('logstash_functional'),
-        esArchiver.load('canvas/default'),
-      ]);
+      await esArchiver.loadIfNeeded('logstash_functional');
+      await esArchiver.load('canvas/default');
       await PageObjects.common.navigateToApp('rollupJob');
     });
 

x-pack/test/functional/apps/security/user_email.js

@@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }) {
   const esArchiver = getService('esArchiver');
 
   describe('useremail', function () {
+    this.tags('smoke');
     before(async () => {
       await esArchiver.load('security/discover');
       await PageObjects.settings.navigateTo();

x-pack/test/functional/apps/security/users.js

@@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }) {
   const log = getService('log');
 
   describe('users', function () {
+    this.tags('smoke');
     before(async () => {
 
       log.debug('users');

x-pack/test/functional/apps/status_page/status_page.ts

@@ -10,7 +10,8 @@ export default function statusPageFunctonalTests({ getService, getPageObjects }:
   const esArchiver = getService('esArchiver');
   const PageObjects = getPageObjects(['security', 'statusPage', 'home']);
 
-  describe('Status Page', () => {
+  describe('Status Page', function() {
+    this.tags('smoke');
     before(async () => await esArchiver.load('empty_kibana'));
     after(async () => await esArchiver.unload('empty_kibana'));
 

x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts

@@ -14,7 +14,8 @@ export default function upgradeAssistantFunctionalTests({
   const esArchiver = getService('esArchiver');
   const PageObjects = getPageObjects(['upgradeAssistant']);
 
-  describe('Upgrade Checkup', () => {
+  describe('Upgrade Checkup', function() {
+    this.tags('smoke');
     before(async () => await esArchiver.load('empty_kibana'));
     after(async () => {
       await PageObjects.upgradeAssistant.expectTelemetryHasFinish();

x-pack/test/functional/apps/watcher/index.js

@@ -6,7 +6,7 @@
 
 export default function ({ loadTestFile }) {
   describe('watcher app', function () {
-    this.tags('ciGroup1');
+    this.tags(['ciGroup1', 'smoke']);
 
     //loadTestFile(require.resolve('./management'));
     loadTestFile(require.resolve('./watcher_test'));