Save html for failed tests (#16850) (#17160)

* When failure, log URL and save page source

* log Taking screenshot  at info level

* Force a failure, then revert this

* [ftr/failureDebugging] create failure debugging service

* Cleanup and create the failure_debug/html dir, strip special chars from filenames

* Remove a debugging log

* refactored replacing invalid filename chars

* Revert forced failing tests

.gitignore

@@ -12,6 +12,7 @@ target
 .idea
 *.iml
 *.log
+/test/*/failure_debug
 /test/*/screenshots/diff
 /test/*/screenshots/failure
 /test/*/screenshots/session

src/functional_test_runner/lib/config/schema.js

@@ -109,4 +109,9 @@ export const schema = Joi.object().keys({
   screenshots: Joi.object().keys({
     directory: Joi.string().default(defaultRelativeToConfigPath('screenshots'))
   }).default(),
+
+  // settings for the failureDebugging module
+  failureDebugging: Joi.object().keys({
+    htmlDirectory: Joi.string().default(defaultRelativeToConfigPath('failure_debug/html'))
+  }).default(),
 }).default();

test/functional/apps/console/_console.js

@@ -22,7 +22,7 @@ export default function ({ getService, getPageObjects }) {
       return PageObjects.common.navigateToApp('console');
     });
 
-    it('should show the default request', function () {
+    it('should show the default *%^$# @ ! ~ request', function () {
       // collapse the help pane because we only get the VISIBLE TEXT, not the part that is scrolled
       return PageObjects.console.collapseHelp()
         .then(function () {

test/functional/config.js

@@ -25,6 +25,7 @@ import {
   ScreenshotsProvider,
   DashboardVisualizationProvider,
   DashboardExpectProvider,
+  FailureDebuggingProvider,
 } from './services';
 
 export default async function ({ readConfigFile }) {
@@ -73,6 +74,7 @@ export default async function ({ readConfigFile }) {
       screenshots: ScreenshotsProvider,
       dashboardVisualizations: DashboardVisualizationProvider,
       dashboardExpect: DashboardExpectProvider,
+      failureDebugging: FailureDebuggingProvider,
     },
     servers: commonConfig.get('servers'),
     apps: {

test/functional/services/failure_debugging.js

@@ -0,0 +1,46 @@
+import { resolve } from 'path';
+import { writeFile } from 'fs';
+import mkdirp from 'mkdirp';
+import del from 'del';
+import { promisify } from 'bluebird';
+
+const writeFileAsync = promisify(writeFile);
+const mkdirAsync = promisify(mkdirp);
+
+export async function FailureDebuggingProvider({ getService }) {
+  const screenshots = getService('screenshots');
+  const config = getService('config');
+  const lifecycle = getService('lifecycle');
+  const log = getService('log');
+  const remote = getService('remote');
+
+  await del(config.get('failureDebugging.htmlDirectory'));
+
+  async function logCurrentUrl() {
+    const currentUrl = await remote.getCurrentUrl();
+    log.info(`Current URL is: ${currentUrl}`);
+  }
+
+  async function savePageHtml(name) {
+    await mkdirAsync(config.get('failureDebugging.htmlDirectory'));
+    const htmlOutputFileName = resolve(config.get('failureDebugging.htmlDirectory'), `${name}.html`);
+    const pageSource = await remote.getPageSource();
+    log.info(`Saving page source to: ${htmlOutputFileName}`);
+    await writeFileAsync(htmlOutputFileName, pageSource);
+  }
+
+  async function onFailure(error, test) {
+    // Replace characters in test names which can't be used in filenames, like *
+    const name = test.fullTitle().replace(/([^ a-zA-Z0-9/-]+)/g, '_');
+
+    await Promise.all([
+      screenshots.takeForFailure(name),
+      logCurrentUrl(),
+      savePageHtml(name)
+    ]);
+  }
+
+  lifecycle
+    .on('testFailure', onFailure)
+    .on('testHookFailure', onFailure);
+}

test/functional/services/index.js

@@ -5,4 +5,5 @@ export { TestSubjectsProvider } from './test_subjects';
 export { RemoteProvider } from './remote';
 export { DocTableProvider } from './doc_table';
 export { ScreenshotsProvider } from './screenshots';
+export { FailureDebuggingProvider } from './failure_debugging';
 export * from './dashboard';

test/functional/services/screenshots.js

@@ -12,7 +12,6 @@ export async function ScreenshotsProvider({ getService }) {
   const log = getService('log');
   const config = getService('config');
   const remote = getService('remote');
-  const lifecycle = getService('lifecycle');
 
   const SESSION_DIRECTORY = resolve(config.get('screenshots.directory'), 'session');
   const FAILURE_DIRECTORY = resolve(config.get('screenshots.directory'), 'failure');
@@ -50,12 +49,12 @@ export async function ScreenshotsProvider({ getService }) {
     }
 
     async takeForFailure(name) {
-      return await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`));
+      await this._take(resolve(FAILURE_DIRECTORY, `${name}.png`));
     }
 
     async _take(path) {
       try {
-        log.debug(`Taking screenshot "${path}"`);
+        log.info(`Taking screenshot "${path}"`);
         const [screenshot] = await Promise.all([
           remote.takeScreenshot(),
           fcb(cb => mkdirp(dirname(path), cb)),
@@ -68,11 +67,5 @@ export async function ScreenshotsProvider({ getService }) {
     }
   }
 
-  const screenshots = new Screenshots();
-
-  lifecycle
-    .on('testFailure', (err, test) => screenshots.takeForFailure(test.fullTitle()))
-    .on('testHookFailure', (err, test) => screenshots.takeForFailure(test.fullTitle()));
-
-  return screenshots;
+  return new Screenshots();
 }