[Security Solution] Cypress tests executed on Firefox (#91524)

* allows Cypress tests to be executed on Firefox

* fixes failing tests

* fixes issues

* fixes exceptions table failing test

* updates readme

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

test/scripts/jenkins_security_solution_cypress_chrome.sh

@@ -5,7 +5,7 @@ source test/scripts/jenkins_test_setup_xpack.sh
 echo " -> Running security solution cypress tests"
 cd "$XPACK_DIR"
 
-checks-reporter-with-killswitch "Security Solution Cypress Tests" \
+checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome)" \
  node scripts/functional_tests \
    --debug --bail \
    --kibana-install-dir "$KIBANA_INSTALL_DIR" \

test/scripts/jenkins_security_solution_cypress_firefox.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source test/scripts/jenkins_test_setup_xpack.sh
+
+echo " -> Running security solution cypress tests"
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "Security Solution Cypress Tests (Firefox)" \
+ node scripts/functional_tests \
+   --debug --bail \
+   --kibana-install-dir "$KIBANA_INSTALL_DIR" \
+   --config test/security_solution_cypress/config.firefox.ts
+
+echo ""
+echo ""

vars/tasks.groovy

@@ -118,7 +118,8 @@ def functionalXpack(Map params = [:]) {
       'x-pack/plugins/triggers_actions_ui/public/application/context/actions_connectors_context.tsx',
     ]) {
       if (githubPr.isPr()) {
-        task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh'))
+        task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypressChrome', './test/scripts/jenkins_security_solution_cypress_chrome.sh'))
+        task(kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypressFirefox', './test/scripts/jenkins_security_solution_cypress_firefox.sh'))
       }
     }
   }

x-pack/plugins/security_solution/cypress/README.md

@@ -29,7 +29,7 @@ This configuration runs cypress tests against an arbitrary host.
 
 ### Test Execution: Examples
 
-#### FTR + Headless
+#### FTR + Headless (Chrome)
 
 Since this is how tests are run on CI, this will likely be the configuration you want to reproduce failures locally, etc.
 
@@ -45,6 +45,22 @@ cd x-pack/plugins/security_solution
 yarn cypress:run-as-ci
 ```
 
+#### FTR + Headless (Firefox)
+
+Since this is how tests are run on CI, this will likely be the configuration you want to reproduce failures locally, etc.
+
+```shell
+# bootstrap kibana from the project root
+yarn kbn bootstrap
+
+# build the plugins/assets that cypress will execute against
+node scripts/build_kibana_platform_plugins
+
+# launch the cypress test runner
+cd x-pack/plugins/security_solution
+yarn cypress:run-as-ci:firefox
+```
+
 #### FTR + Interactive
 
 This is the preferred mode for developing new tests.
@@ -61,7 +77,9 @@ cd x-pack/plugins/security_solution
 yarn cypress:open-as-ci
 ```
 
-#### Custom Target + Headless
+Note that you can select the browser you want to use on the top right side of the interactive runner.
+
+#### Custom Target + Headless (Chrome)
 
 This mode may be useful for testing a release, e.g. spin up a build candidate
 and point cypress at it to catch regressions.
@@ -79,6 +97,24 @@ cd x-pack/plugins/security_solution
 CYPRESS_BASE_URL=http(s)://<username>:<password>@<kbnUrl> CYPRESS_ELASTICSEARCH_URL=http(s)://<username>:<password>@<elsUrl> CYPRESS_ELASTICSEARCH_USERNAME=<username> CYPRESS_ELASTICSEARCH_PASSWORD=password yarn cypress:run
 ```
 
+#### Custom Target + Headless (Firefox)
+
+This mode may be useful for testing a release, e.g. spin up a build candidate
+and point cypress at it to catch regressions.
+
+```shell
+# bootstrap kibana from the project root
+yarn kbn bootstrap
+
+# load auditbeat data needed for test execution (which FTR normally does for us)
+cd x-pack/plugins/security_solution
+node ../../../scripts/es_archiver load auditbeat --dir ../../test/security_solution_cypress/es_archives --config ../../../test/functional/config.js --es-url http(s)://<username>:<password>@<elsUrl> --kibana-url http(s)://<userName>:<password>@<kbnUrl>
+
+# launch the cypress test runner with overridden environment variables
+cd x-pack/plugins/security_solution
+CYPRESS_BASE_URL=http(s)://<username>:<password>@<kbnUrl> CYPRESS_ELASTICSEARCH_URL=http(s)://<username>:<password>@<elsUrl> CYPRESS_ELASTICSEARCH_USERNAME=<username> CYPRESS_ELASTICSEARCH_PASSWORD=password yarn cypress:run:firefox
+```
+
 ## Folder Structure
 
 ### integration/

x-pack/plugins/security_solution/cypress/integration/timelines/data_providers.spec.ts

@@ -69,28 +69,40 @@ describe('timeline data providers', () => {
   it('sets the background to euiColorSuccess with a 10% alpha channel when the user starts dragging a host, but is not hovering over the data providers', () => {
     dragFirstHostToTimeline();
 
-    cy.get(TIMELINE_DATA_PROVIDERS)
-      .filter(':visible')
-      .should(
-        'have.css',
-        'background',
-        'rgba(1, 125, 115, 0.1) none repeat scroll 0% 0% / auto padding-box border-box'
-      );
+    if (Cypress.browser.name === 'firefox') {
+      cy.get(TIMELINE_DATA_PROVIDERS)
+        .filter(':visible')
+        .should('have.css', 'background-color', 'rgba(1, 125, 115, 0.1)');
+    } else {
+      cy.get(TIMELINE_DATA_PROVIDERS)
+        .filter(':visible')
+        .should(
+          'have.css',
+          'background',
+          'rgba(1, 125, 115, 0.1) none repeat scroll 0% 0% / auto padding-box border-box'
+        );
+    }
   });
 
   it('sets the background to euiColorSuccess with a 20% alpha channel and renders the dashed border color as euiColorSuccess when the user starts dragging a host AND is hovering over the data providers', () => {
     dragFirstHostToEmptyTimelineDataProviders();
 
-    cy.get(TIMELINE_DATA_PROVIDERS_EMPTY)
-      .filter(':visible')
-      .should(
-        'have.css',
-        'background',
-        'rgba(1, 125, 115, 0.2) none repeat scroll 0% 0% / auto padding-box border-box'
-      );
+    if (Cypress.browser.name === 'firefox') {
+      cy.get(TIMELINE_DATA_PROVIDERS_EMPTY)
+        .filter(':visible')
+        .should('have.css', 'background-color', 'rgba(1, 125, 115, 0.2)');
+    } else {
+      cy.get(TIMELINE_DATA_PROVIDERS_EMPTY)
+        .filter(':visible')
+        .should(
+          'have.css',
+          'background',
+          'rgba(1, 125, 115, 0.2) none repeat scroll 0% 0% / auto padding-box border-box'
+        );
 
-    cy.get(TIMELINE_DATA_PROVIDERS)
-      .filter(':visible')
-      .should('have.css', 'border', '3.1875px dashed rgb(1, 125, 115)');
+      cy.get(TIMELINE_DATA_PROVIDERS)
+        .filter(':visible')
+        .should('have.css', 'border', '3.1875px dashed rgb(1, 125, 115)');
+    }
   });
 });

x-pack/plugins/security_solution/cypress/integration/timelines/flyout_button.spec.ts

@@ -68,12 +68,18 @@ describe('timeline flyout button', () => {
   it('sets the data providers background to euiColorSuccess with a 10% alpha channel when the user starts dragging a host, but is not hovering over the data providers area', () => {
     dragFirstHostToTimeline();
 
-    cy.get(TIMELINE_DATA_PROVIDERS)
-      .filter(':visible')
-      .should(
-        'have.css',
-        'background',
-        'rgba(1, 125, 115, 0.1) none repeat scroll 0% 0% / auto padding-box border-box'
-      );
+    if (Cypress.browser.name === 'firefox') {
+      cy.get(TIMELINE_DATA_PROVIDERS)
+        .filter(':visible')
+        .should('have.css', 'background-color', 'rgba(1, 125, 115, 0.1)');
+    } else {
+      cy.get(TIMELINE_DATA_PROVIDERS)
+        .filter(':visible')
+        .should(
+          'have.css',
+          'background',
+          'rgba(1, 125, 115, 0.1) none repeat scroll 0% 0% / auto padding-box border-box'
+        );
+    }
   });
 });

x-pack/plugins/security_solution/cypress/integration/urls/state.spec.ts

@@ -48,6 +48,8 @@ const ABSOLUTE_DATE = {
   newStartTimeTyped: 'Aug 01, 2019 @ 14:33:29.186',
   startTime: '2019-08-01T20:03:29.186Z',
   startTimeTimeline: '2019-08-02T20:03:29.186Z',
+  firefoxEndTimeTyped: '2019-08-01T15:03:29',
+  firefoxStartTimeTyped: '2019-08-01T14:33:29',
 };
 
 describe('url state', () => {
@@ -73,13 +75,20 @@ describe('url state', () => {
     setEndDate(ABSOLUTE_DATE.newEndTimeTyped);
     updateDates();
 
+    let startDate: string;
+    let endDate: string;
+
+    if (Cypress.browser.name === 'firefox') {
+      startDate = new Date(ABSOLUTE_DATE.firefoxStartTimeTyped).toISOString().replace('000', '186');
+      endDate = new Date(ABSOLUTE_DATE.firefoxEndTimeTyped).toISOString().replace('000', '186');
+    } else {
+      startDate = new Date(ABSOLUTE_DATE.newStartTimeTyped).toISOString();
+      endDate = new Date(ABSOLUTE_DATE.newEndTimeTyped).toISOString();
+    }
+
     cy.url().should(
       'include',
-      `(global:(linkTo:!(timeline),timerange:(from:%27${new Date(
-        ABSOLUTE_DATE.newStartTimeTyped
-      ).toISOString()}%27,kind:absolute,to:%27${new Date(
-        ABSOLUTE_DATE.newEndTimeTyped
-      ).toISOString()}%27))`
+      `(global:(linkTo:!(timeline),timerange:(from:%27${startDate}%27,kind:absolute,to:%27${endDate}%27))`
     );
   });
 
@@ -130,13 +139,20 @@ describe('url state', () => {
     setTimelineEndDate(ABSOLUTE_DATE.newEndTimeTyped);
     updateTimelineDates();
 
+    let startDate: string;
+    let endDate: string;
+
+    if (Cypress.browser.name === 'firefox') {
+      startDate = new Date(ABSOLUTE_DATE.firefoxStartTimeTyped).toISOString().replace('000', '186');
+      endDate = new Date(ABSOLUTE_DATE.firefoxEndTimeTyped).toISOString().replace('000', '186');
+    } else {
+      startDate = new Date(ABSOLUTE_DATE.newStartTimeTyped).toISOString();
+      endDate = new Date(ABSOLUTE_DATE.newEndTimeTyped).toISOString();
+    }
+
     cy.url().should(
       'include',
-      `timeline:(linkTo:!(),timerange:(from:%27${new Date(
-        ABSOLUTE_DATE.newStartTimeTyped
-      ).toISOString()}%27,kind:absolute,to:%27${new Date(
-        ABSOLUTE_DATE.newEndTimeTyped
-      ).toISOString()}%27))`
+      `timeline:(linkTo:!(),timerange:(from:%27${startDate}%27,kind:absolute,to:%27${endDate}%27))`
     );
   });
 

x-pack/plugins/security_solution/cypress/tasks/exceptions_table.ts

@@ -26,7 +26,11 @@ export const waitForExceptionsTableToBeLoaded = () => {
 };
 
 export const searchForExceptionList = (searchText: string) => {
-  cy.get(EXCEPTIONS_TABLE_SEARCH).type(searchText, { force: true }).trigger('search');
+  if (Cypress.browser.name === 'firefox') {
+    cy.get(EXCEPTIONS_TABLE_SEARCH).type(`${searchText}{enter}`, { force: true });
+  } else {
+    cy.get(EXCEPTIONS_TABLE_SEARCH).type(searchText, { force: true }).trigger('search');
+  }
 };
 
 export const deleteExceptionListWithoutRuleReference = () => {

x-pack/plugins/security_solution/package.json

@@ -12,6 +12,8 @@
     "cypress:open-as-ci": "node ../../../scripts/functional_tests --config ../../test/security_solution_cypress/visual_config.ts",
     "cypress:run": "../../../node_modules/.bin/cypress run --browser chrome --headless --spec ./cypress/integration/**/*.spec.ts --config-file ./cypress/cypress.json --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json; status=$?; ../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json; ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results; mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/ && exit $status;",
     "cypress:run-as-ci": "node --max-old-space-size=2048 ../../../scripts/functional_tests --config ../../test/security_solution_cypress/cli_config.ts",
+    "cypress:run:firefox": "../../../node_modules/.bin/cypress run --browser firefox --headless --spec ./cypress/integration/**/*.spec.ts --config-file ./cypress/cypress.json --reporter ../../../node_modules/cypress-multi-reporters --reporter-options configFile=./cypress/reporter_config.json; status=$?; ../../../node_modules/.bin/mochawesome-merge ../../../target/kibana-security-solution/cypress/results/mochawesome*.json > ../../../target/kibana-security-solution/cypress/results/output.json; ../../../node_modules/.bin/marge ../../../target/kibana-security-solution/cypress/results/output.json --reportDir ../../../target/kibana-security-solution/cypress/results; mkdir -p ../../../target/junit && cp ../../../target/kibana-security-solution/cypress/results/*.xml ../../../target/junit/ && exit $status;",
+    "cypress:run-as-ci:firefox": "node --max-old-space-size=2048 ../../../scripts/functional_tests --config ../../test/security_solution_cypress/config.firefox.ts",
     "test:generate": "node scripts/endpoint/resolver_generator"
   }
 }

x-pack/test/security_solution_cypress/config.firefox.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { resolve } from 'path';
+
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+
+import { CA_CERT_PATH } from '@kbn/dev-utils';
+
+import { SecuritySolutionCypressCliFirefoxTestRunner } from './runner';
+
+export default async function ({ readConfigFile }: FtrConfigProviderContext) {
+  const kibanaCommonTestsConfig = await readConfigFile(
+    require.resolve('../../../test/common/config.js')
+  );
+  const xpackFunctionalTestsConfig = await readConfigFile(
+    require.resolve('../functional/config.js')
+  );
+
+  return {
+    ...kibanaCommonTestsConfig.getAll(),
+
+    esArchiver: {
+      directory: resolve(__dirname, 'es_archives'),
+    },
+
+    browser: {
+      type: 'firefox',
+      acceptInsecureCerts: true,
+    },
+
+    esTestCluster: {
+      ...xpackFunctionalTestsConfig.get('esTestCluster'),
+      serverArgs: [
+        ...xpackFunctionalTestsConfig.get('esTestCluster.serverArgs'),
+        // define custom es server here
+        // API Keys is enabled at the top level
+        'xpack.security.enabled=true',
+      ],
+    },
+
+    kbnTestServer: {
+      ...xpackFunctionalTestsConfig.get('kbnTestServer'),
+      serverArgs: [
+        ...xpackFunctionalTestsConfig.get('kbnTestServer.serverArgs'),
+        '--csp.strict=false',
+        // define custom kibana server args here
+        `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+      ],
+    },
+
+    testRunner: SecuritySolutionCypressCliFirefoxTestRunner,
+  };
+}

x-pack/test/security_solution_cypress/runner.ts

@@ -49,6 +49,45 @@ export async function SecuritySolutionCypressCliTestRunner({ getService }: FtrPr
   });
 }
 
+export async function SecuritySolutionCypressCliFirefoxTestRunner({
+  getService,
+}: FtrProviderContext) {
+  const log = getService('log');
+  const config = getService('config');
+  const esArchiver = getService('esArchiver');
+
+  await esArchiver.load('auditbeat');
+
+  await withProcRunner(log, async (procs) => {
+    await procs.run('cypress', {
+      cmd: 'yarn',
+      args: ['cypress:run:firefox'],
+      cwd: resolve(__dirname, '../../plugins/security_solution'),
+      env: {
+        FORCE_COLOR: '1',
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        CYPRESS_baseUrl: Url.format(config.get('servers.kibana')),
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        CYPRESS_protocol: config.get('servers.kibana.protocol'),
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        CYPRESS_hostname: config.get('servers.kibana.hostname'),
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        CYPRESS_configport: config.get('servers.kibana.port'),
+        CYPRESS_ELASTICSEARCH_URL: Url.format(config.get('servers.elasticsearch')),
+        CYPRESS_ELASTICSEARCH_USERNAME: config.get('servers.elasticsearch.username'),
+        CYPRESS_ELASTICSEARCH_PASSWORD: config.get('servers.elasticsearch.password'),
+        CYPRESS_KIBANA_URL: Url.format({
+          protocol: config.get('servers.kibana.protocol'),
+          hostname: config.get('servers.kibana.hostname'),
+          port: config.get('servers.kibana.port'),
+        }),
+        ...process.env,
+      },
+      wait: true,
+    });
+  });
+}
+
 export async function SecuritySolutionCypressVisualTestRunner({ getService }: FtrProviderContext) {
   const log = getService('log');
   const config = getService('config');