[a11y] add initial accessibility functional tests (#43584)

* [a11y] add initial accessibility functional tests

* add accessibility jobs

* fix config path

* remove percy setup from scripts

* disable color-contrast rule

* apply changes from @myasonik

* define aria-controls/owns props even when suggestions aren't visible

* [ftr/a11y] only throw error when there are errors

* adding tests for management page

* add a11y test for management page

* adding ignore rules' to a11y

* accessibility test for kibana home

* 7 passing tests, 0 failures

* jest snapshot update

* support a11y test in pipeline job

* update a11y test script for pipelines

* use oss compatible ci setup

* fix exclude syntax

* add default exclusion syntax

.ci/jobs.yml

@@ -14,6 +14,7 @@ JOB:
   - kibana-ciGroup10
   - kibana-ciGroup11
   - kibana-ciGroup12
+  - kibana-accessibility
   - kibana-visualRegression
 
   # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
@@ -28,6 +29,7 @@ JOB:
   - x-pack-ciGroup8
   - x-pack-ciGroup9
   - x-pack-ciGroup10
+  - x-pack-accessibility
   - x-pack-visualRegression
 
 # `~` is yaml for `null`

.ci/run.sh

@@ -21,6 +21,9 @@ kibana-ciGroup*)
 kibana-visualRegression*)
   ./test/scripts/jenkins_visual_regression.sh
   ;;
+kibana-accessibility*)
+  ./test/scripts/jenkins_accessibility.sh
+  ;;
 kibana-firefoxSmoke*)
   ./test/scripts/jenkins_firefox_smoke.sh
   ;;
@@ -34,6 +37,9 @@ x-pack-ciGroup*)
 x-pack-visualRegression*)
   ./test/scripts/jenkins_xpack_visual_regression.sh
   ;;
+x-pack-accessibility*)
+  ./test/scripts/jenkins_xpack_accessibility.sh
+  ;;
 x-pack-firefoxSmoke*)
   ./test/scripts/jenkins_xpack_firefox_smoke.sh
   ;;

.eslintrc.js

@@ -400,6 +400,7 @@ module.exports = {
         'x-pack/test/functional/apps/**/*.js',
         'x-pack/legacy/plugins/apm/**/*.js',
         'test/*/config.ts',
+        'test/*/{tests,test_suites,apis,apps}/**/*',
         'test/visual_regression/tests/**/*',
         'x-pack/test/*/{tests,test_suites,apis,apps}/**/*',
         'x-pack/test/*/*config.*ts',

Jenkinsfile

@@ -25,6 +25,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
               'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
               'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' }),
+              'oss-accessibility': kibanaPipeline.getPostBuildWorker('accessibility', { runbld './test/scripts/jenkins_accessibility.sh' }),
               'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }),
             ]),
             'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
@@ -39,6 +40,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
               'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
               'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' }),
+              'xpack-accessibility': kibanaPipeline.getPostBuildWorker('xpack-accessibility', { runbld './test/scripts/jenkins_xpack_accessibility.sh' }),
               'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }),
             ]),
           ])
@@ -47,4 +49,4 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
       }
     }
   }
-}
+}

package.json

@@ -350,6 +350,7 @@
     "@typescript-eslint/parser": "^2.5.0",
     "angular-mocks": "^1.7.8",
     "archiver": "^3.1.1",
+    "axe-core": "^3.3.2",
     "babel-eslint": "^10.0.3",
     "babel-jest": "^24.9.0",
     "babel-plugin-dynamic-import-node": "^2.3.0",

src/legacy/core_plugins/kibana/public/management/sections/settings/components/field/__snapshots__/field.test.js.snap

@@ -463,7 +463,7 @@ exports[`Field for boolean setting should render as read only if saving is disab
         display="row"
         error={null}
         fullWidth={false}
-        hasChildLabel={true}
+        hasChildLabel={false}
         hasEmptyLabelSpace={false}
         helpText={null}
         isInvalid={false}
@@ -557,7 +557,7 @@ exports[`Field for boolean setting should render as read only with help text if
         display="row"
         error={null}
         fullWidth={false}
-        hasChildLabel={true}
+        hasChildLabel={false}
         hasEmptyLabelSpace={false}
         helpText={
           <EuiText
@@ -650,7 +650,7 @@ exports[`Field for boolean setting should render custom setting icon if it is cu
         display="row"
         error={null}
         fullWidth={false}
-        hasChildLabel={true}
+        hasChildLabel={false}
         hasEmptyLabelSpace={false}
         helpText={null}
         isInvalid={false}
@@ -722,7 +722,7 @@ exports[`Field for boolean setting should render default value if there is no us
         display="row"
         error={null}
         fullWidth={false}
-        hasChildLabel={true}
+        hasChildLabel={false}
         hasEmptyLabelSpace={false}
         helpText={null}
         isInvalid={false}
@@ -816,7 +816,7 @@ exports[`Field for boolean setting should render user value if there is user val
         display="row"
         error={null}
         fullWidth={false}
-        hasChildLabel={true}
+        hasChildLabel={false}
         hasEmptyLabelSpace={false}
         helpText={
           <span>

src/legacy/core_plugins/kibana/public/management/sections/settings/components/field/field.js

@@ -195,7 +195,7 @@ export class Field extends PureComponent {
     this.setState({
       unsavedValue: newUnsavedValue,
       isInvalid,
-      error
+      error,
     });
   };
 
@@ -764,6 +764,7 @@ export class Field extends PureComponent {
               helpText={this.renderHelpText(setting)}
               describedByIds={[`${setting.name}-aria`]}
               className="mgtAdvancedSettings__fieldRow"
+              hasChildLabel={setting.type !== 'boolean'}
             >
               {this.renderField(setting)}
             </EuiFormRow>

src/legacy/ui/public/chrome/directives/kbn_chrome.html

@@ -1,4 +1,4 @@
-<div class="app-wrapper-panel">
+<div class="app-wrapper-panel" data-test-subj="appA11yRoot">
   <div id="globalBannerList"></div>
 
   <div

test/accessibility/apps/discover.ts

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+const FROM_TIME = '2015-09-19 06:31:44.000';
+const TO_TIME = '2015-09-23 18:31:44.000';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'timePicker']);
+  const a11y = getService('a11y');
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+
+  describe('Discover', () => {
+    before(async () => {
+      await esArchiver.load('discover');
+      await esArchiver.loadIfNeeded('logstash_functional');
+      await kibanaServer.uiSettings.update({
+        defaultIndex: 'logstash-*',
+      });
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.timePicker.setAbsoluteRange(FROM_TIME, TO_TIME);
+    });
+
+    it('main view', async () => {
+      await a11y.testAppSnapshot();
+    });
+  });
+}

test/accessibility/apps/home.ts

@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common']);
+  const a11y = getService('a11y');
+
+  describe('Kibana Home', () => {
+    before(async () => {
+      await PageObjects.common.navigateToApp('home');
+    });
+
+    it('Kibana Home view', async () => {
+      await a11y.testAppSnapshot();
+    });
+  });
+}

test/accessibility/apps/management.ts

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const PageObjects = getPageObjects(['common', 'settings']);
+  const a11y = getService('a11y');
+
+  describe('Management', () => {
+    before(async () => {
+      await PageObjects.common.navigateToApp('settings');
+    });
+
+    it('main view', async () => {
+      await a11y.testAppSnapshot();
+    });
+
+    it('index pattern page', async () => {
+      await PageObjects.settings.clickKibanaIndexPatterns();
+      await a11y.testAppSnapshot();
+    });
+
+    it('Single indexpattern view', async () => {
+      await PageObjects.settings.clickIndexPatternLogstash();
+      await a11y.testAppSnapshot();
+    });
+
+    it('Saved objects view', async () => {
+      await PageObjects.settings.clickKibanaSavedObjects();
+      await a11y.testAppSnapshot();
+    });
+
+    it('Advanced settings', async () => {
+      await PageObjects.settings.clickKibanaSettings();
+      await a11y.testAppSnapshot();
+    });
+  });
+}

test/accessibility/config.ts

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+import { services } from './services';
+import { pageObjects } from './page_objects';
+
+export default async function({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [
+      require.resolve('./apps/discover'),
+      require.resolve('./apps/management'),
+      require.resolve('./apps/home'),
+    ],
+    pageObjects,
+    services,
+
+    junit: {
+      reportName: 'Accessibility Tests',
+    },
+  };
+}

test/accessibility/ftr_provider_context.d.ts

@@ -0,0 +1,25 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
+
+import { pageObjects } from './page_objects';
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;

test/accessibility/page_objects.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { pageObjects } from '../functional/page_objects';

test/accessibility/services/a11y/a11y.ts

@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import testSubjectToCss from '@kbn/test-subj-selector';
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+import { AxeReport, printResult } from './axe_report';
+// @ts-ignore JS that is run in browser as is
+import { analyzeWithAxe, analyzeWithAxeWithClient } from './analyze_with_axe';
+
+interface AxeContext {
+  include?: string[];
+  exclude?: string[][];
+}
+
+interface TestOptions {
+  excludeTestSubj?: string | string[];
+}
+
+export const normalizeResult = (report: any) => {
+  if (report.error) {
+    throw report.error;
+  }
+
+  return report.result as false | AxeReport;
+};
+
+export function A11yProvider({ getService }: FtrProviderContext) {
+  const browser = getService('browser');
+  const Wd = getService('__webdriver__');
+  const log = getService('log');
+
+  /**
+   * Accessibility testing service using the Axe (https://www.deque.com/axe/)
+   * toolset to validate a11y rules similar to ESLint. In order to test against
+   * the rules we must load up the UI and feed a full HTML snapshot into Axe.
+   */
+  return new (class Accessibility {
+    public async testAppSnapshot(options: TestOptions = {}) {
+      const context = this.getAxeContext(true, options.excludeTestSubj);
+      const report = await this.captureAxeReport(context);
+      await this.testAxeReport(report);
+    }
+
+    public async testGlobalSnapshot(options: TestOptions = {}) {
+      const context = this.getAxeContext(false, options.excludeTestSubj);
+      const report = await this.captureAxeReport(context);
+      await this.testAxeReport(report);
+    }
+
+    private getAxeContext(global: boolean, excludeTestSubj?: string | string[]): AxeContext {
+      return {
+        include: global ? undefined : [testSubjectToCss('appA11yRoot')],
+        exclude: ([] as string[])
+          .concat(excludeTestSubj || [])
+          .map(ts => [testSubjectToCss(ts)])
+          .concat([['.ace_scrollbar']]),
+      };
+    }
+
+    private testAxeReport(report: AxeReport) {
+      const errorMsgs = [];
+
+      for (const result of report.incomplete) {
+        // these items require human review and can't be definitively validated
+        log.warning(printResult(chalk.yellow('UNABLE TO VALIDATE'), result));
+      }
+
+      for (const result of report.violations) {
+        errorMsgs.push(printResult(chalk.red('VIOLATION'), result));
+      }
+
+      if (errorMsgs.length) {
+        throw new Error(`a11y report:\n${errorMsgs.join('\n')}`);
+      }
+    }
+
+    private async captureAxeReport(context: AxeContext): Promise<AxeReport> {
+      const axeOptions = {
+        reporter: 'v2',
+        runOnly: ['wcag2a', 'wcag2aa'],
+        rules: {
+          'color-contrast': {
+            enabled: false,
+          },
+        },
+      };
+
+      await (Wd.driver.manage() as any).setTimeouts({
+        ...(await (Wd.driver.manage() as any).getTimeouts()),
+        script: 600000,
+      });
+
+      const report = normalizeResult(
+        await browser.executeAsync(analyzeWithAxe, context, axeOptions)
+      );
+
+      if (report !== false) {
+        return report;
+      }
+
+      const withClientReport = normalizeResult(
+        await browser.executeAsync(analyzeWithAxeWithClient, context, axeOptions)
+      );
+
+      if (withClientReport === false) {
+        throw new Error('Attempted to analyze with axe but failed to load axe client');
+      }
+
+      return withClientReport;
+    }
+  })();
+}

test/accessibility/services/a11y/analyze_with_axe.js

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readFileSync } from 'fs';
+
+export function analyzeWithAxe(context, options, callback) {
+  Promise.resolve()
+    .then(() => {
+      if (window.axe) {
+        return window.axe.run(context, options);
+      }
+
+      // return a false report to trigger analyzeWithAxeWithClient
+      return false;
+    })
+    .then(result => callback({ result }), error => callback({ error }));
+}
+
+export const analyzeWithAxeWithClient = `
+  ${readFileSync(require.resolve('axe-core/axe.js'), 'utf8')}
+
+  return (${analyzeWithAxe.toString()}).apply(null, arguments);
+`;

test/accessibility/services/a11y/axe_report.ts

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+type AxeImpact = 'minor' | 'moderate' | 'serious' | 'critical';
+
+type AxeRelatedNodes = Array<{
+  data: any;
+  id: string;
+  impact: AxeImpact;
+  message: string;
+  relatedNodes: [];
+}>;
+
+export interface AxeResult {
+  /* Rule description */
+  description: string;
+  /* rule title/error message */
+  help: string;
+  /* documentation url */
+  helpUrl: string;
+  /* rule id */
+  id: string;
+  /* severity level */
+  impact?: AxeImpact;
+  /* tags used to group rules */
+  tags: string[];
+  /* nodes grouped in this result */
+  nodes: Array<{
+    all: AxeRelatedNodes;
+    any: AxeRelatedNodes;
+    none: AxeRelatedNodes;
+
+    html: string;
+    impact: AxeImpact;
+    target: string[];
+  }>;
+}
+
+export type AxeResultGroup = AxeResult[];
+
+export interface AxeReport {
+  inapplicable: AxeResultGroup;
+  passes: AxeResultGroup;
+  incomplete: AxeResultGroup;
+  violations: AxeResultGroup;
+}
+
+export const printResult = (title: string, result: AxeResult) => `
+${title}
+  [${result.id}]: ${result.description}
+    Help: ${result.helpUrl}
+    Elements:
+      - ${result.nodes.map(node => node.target).join('\n      - ')}`;

test/accessibility/services/a11y/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { A11yProvider } from './a11y';

test/accessibility/services/index.ts

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { services as kibanaFunctionalServices } from '../../functional/services';
+import { A11yProvider } from './a11y';
+
+export const services = {
+  ...kibanaFunctionalServices,
+  a11y: A11yProvider,
+};

test/functional/services/browser.ts

@@ -461,10 +461,7 @@ export async function BrowserProvider({ getService }: FtrProviderContext) {
       );
     }
 
-    public async executeAsync<A extends any[], R>(
+    public async executeAsync<R>(fn: string | ((...args: any[]) => R), ...args: any[]): Promise<R> {
-      fn: string | ((...args: A) => R),
-      ...args: A
-    ): Promise<R> {
       return await driver.executeAsyncScript(
         fn,
         ...cloneDeep<any>(args, arg => {

test/scripts/jenkins_accessibility.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ -n "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
+export TEST_BROWSER_HEADLESS=1
+
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  yarn run grunt functionalTests:ensureAllTestsInCiGroup;
+  node scripts/build --debug --oss;
+else
+  installDir="$(realpath $PARENT_DIR/kibana/build/oss/kibana-*-SNAPSHOT-linux-x86_64)"
+  destDir=${installDir}-${CI_WORKER_NUMBER}
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
+
+checks-reporter-with-killswitch "Kibana accessibility tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$installDir" \
+    --config test/accessibility/config.ts;

test/scripts/jenkins_xpack_accessibility.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ -n "$IS_PIPELINE_JOB" ]] ; then
+  source src/dev/ci_setup/setup_env.sh
+fi
+
+if [[ -z "$IS_PIPELINE_JOB" ]] ; then
+  echo " -> building and extracting default Kibana distributable for use in functional tests"
+  node scripts/build --debug --no-oss
+
+  linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
+  installDir="$PARENT_DIR/install/kibana"
+
+  mkdir -p "$installDir"
+  tar -xzf "$linuxBuild" -C "$installDir" --strip=1
+
+  export KIBANA_INSTALL_DIR="$installDir"
+else
+  installDir="$PARENT_DIR/install/kibana"
+  destDir="${installDir}-${CI_WORKER_NUMBER}"
+  cp -R "$installDir" "$destDir"
+
+  export KIBANA_INSTALL_DIR="$destDir"
+fi
+
+export TEST_BROWSER_HEADLESS=1
+cd "$XPACK_DIR"
+
+checks-reporter-with-killswitch "X-Pack accessibility tests" \
+  node scripts/functional_tests \
+    --debug --bail \
+    --kibana-install-dir "$installDir" \
+    --config test/accessibility/config.ts;

x-pack/test/accessibility/apps/login_page.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FtrProviderContext } from '../ftr_provider_context';
+
+export default function({ getService, getPageObjects }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const a11y = getService('a11y');
+  const testSubjects = getService('testSubjects');
+  const retry = getService('retry');
+  const PageObjects = getPageObjects(['common', 'security']);
+
+  describe('Security', () => {
+    describe('Login Page', () => {
+      before(async () => {
+        await esArchiver.load('empty_kibana');
+        await PageObjects.security.logout();
+      });
+
+      after(async () => {
+        await esArchiver.unload('empty_kibana');
+      });
+
+      afterEach(async () => {
+        await PageObjects.security.logout();
+      });
+
+      it('meets a11y requirements', async () => {
+        await PageObjects.common.navigateToApp('login');
+
+        await retry.waitFor(
+          'login page visible',
+          async () => await testSubjects.exists('loginSubmit')
+        );
+
+        await a11y.testAppSnapshot();
+      });
+    });
+  });
+}

x-pack/test/accessibility/config.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FtrConfigProviderContext } from '@kbn/test/types/ftr';
+import { services } from './services';
+import { pageObjects } from './page_objects';
+
+export default async function({ readConfigFile }: FtrConfigProviderContext) {
+  const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
+
+  return {
+    ...functionalConfig.getAll(),
+
+    testFiles: [require.resolve('./apps/login_page')],
+
+    pageObjects,
+    services,
+
+    junit: {
+      reportName: 'X-Pack Accessibility Tests',
+    },
+  };
+}

x-pack/test/accessibility/ftr_provider_context.d.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { GenericFtrProviderContext } from '@kbn/test/types/ftr';
+
+import { pageObjects } from './page_objects';
+import { services } from './services';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;

x-pack/test/accessibility/page_objects.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { pageObjects } from '../functional/page_objects';

x-pack/test/accessibility/services.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { services as kibanaA11yServices } from '../../../test/accessibility/services';
+import { services as functionalServices } from '../functional/services';
+
+export const services = {
+  ...kibanaA11yServices,
+  ...functionalServices,
+};

yarn.lock

@@ -5785,6 +5785,11 @@ aws4@^1.6.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
   integrity sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=
 
+axe-core@^3.3.2:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-3.3.2.tgz#7baf3c55a5cf1621534a2c38735f5a1bf2f7e1a8"
+  integrity sha512-lRdxsRt7yNhqpcXQk1ao1BL73OZDzmFCWOG0mC4tGR/r14ohH2payjHwCMQjHGbBKm924eDlmG7utAGHiX/A6g==
+
 axios@^0.18.0, axios@^0.18.1:
   version "0.18.1"
   resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.1.tgz#ff3f0de2e7b5d180e757ad98000f1081b87bcea3"