[ftr] remove digdug, use chromedriver directly (#11558)

* [ftr] remove digdug, use chromedriver directly

why?
 - digdug is meant to be used by intern, and expects a certain lifecycle that the FTR has tried and continuously fails to mimic
 - rather than continue trying to force digdug into the stack, just spawn chromedriver ourselves. We know how to handle it
 - cleans up verbose remote logging while we're in there, since selenium-standalone-server went with digdug, which was previously doing the verbose logging
 - deprecate config.servers.webdriver
 - add config.chromedriver.url
 - if url at config.chromedriver.url points to a server that responds to http requests use it as the chromedriver instance, enables running chrome in a VM or container
 - if pings to config.chromedriver.url fail a local chromedriver is started

* address review requests

package.json

@@ -45,7 +45,6 @@
     "test:browser": "grunt test:browser",
     "test:ui": "grunt test:ui",
     "test:ui:server": "grunt test:ui:server",
-    "test:ui:runner": "grunt test:ui:runner",
     "test:server": "grunt test:server",
     "test:coverage": "grunt test:coverage",
     "test:visualRegression": "grunt test:visualRegression",
@@ -213,9 +212,8 @@
     "chance": "1.0.6",
     "cheerio": "0.22.0",
     "chokidar": "1.6.0",
-    "chromedriver": "2.28.0",
+    "chromedriver": "^2.29.0",
     "classnames": "2.2.5",
-    "digdug": "1.6.3",
     "enzyme": "2.7.0",
     "enzyme-to-json": "1.4.5",
     "eslint": "3.11.1",
@@ -278,6 +276,7 @@
     "source-map-support": "0.2.10",
     "supertest": "1.2.0",
     "supertest-as-promised": "2.0.2",
+    "tree-kill": "^1.1.0",
     "webpack-dev-server": "1.14.1"
   },
   "engines": {

src/cli/cluster/worker.js

@@ -89,7 +89,7 @@ module.exports = class Worker extends EventEmitter {
       // we don't need to react to process.exit anymore
       this.processBinder.destroy();
 
-      // wait until the cluster reports this fork has exitted, then resolve
+      // wait until the cluster reports this fork has exited, then resolve
       await new Promise(resolve => this.once('fork:exit', resolve));
     }
   }
@@ -153,7 +153,7 @@ module.exports = class Worker extends EventEmitter {
     this.forkBinder.on('online', () => this.onOnline());
     this.forkBinder.on('disconnect', () => this.onDisconnect());
 
-    // when the cluster says a fork has exitted, check if it is ours
+    // when the cluster says a fork has exited, check if it is ours
     this.clusterBinder.on('exit', (fork, code) => this.onExit(fork, code));
 
     // when the process exits, make sure we kill our workers

src/functional_test_runner/lib/config/read_config_file.js

@@ -1,6 +1,7 @@
 import { defaultsDeep } from 'lodash';
 
 import { Config } from './config';
+import { transformDeprecations } from './transform_deprecations';
 
 export async function readConfigFile(log, configFile, settingOverrides = {}) {
   log.debug('Loading config file from %j', configFile);
@@ -24,5 +25,7 @@ export async function readConfigFile(log, configFile, settingOverrides = {}) {
     })
   );
 
-  return new Config(settings);
+  return new Config(transformDeprecations(settings, msg => {
+    log.error(msg);
+  }));
 }

src/functional_test_runner/lib/config/schema.js

@@ -58,11 +58,14 @@ export const schema = Joi.object().keys({
   ),
 
   servers: Joi.object().keys({
-    webdriver: urlPartsSchema(),
     kibana: urlPartsSchema(),
     elasticsearch: urlPartsSchema(),
   }).default(),
 
+  chromedriver: Joi.object().keys({
+    url: Joi.string().uri({ scheme: /https?/ }).default('http://localhost:9515')
+  }).default(),
+
   // definition of apps that work with `common.navigateToApp()`
   apps: Joi.object().pattern(
     ID_PATTERN,

src/functional_test_runner/lib/config/transform_deprecations.js

@@ -0,0 +1,5 @@
+import { createTransform, Deprecations } from '../../../deprecation';
+
+export const transformDeprecations = createTransform([
+  Deprecations.unused('servers.webdriver')
+]);

src/utils/tooling_log/__tests__/log_levels.js

@@ -1,13 +1,13 @@
 import expect from 'expect.js';
 import Chance from 'chance';
-import { createLogLevelFlags } from '../log_levels';
+import { parseLogLevel } from '../log_levels';
 
 const chance = new Chance();
 
-describe('createLogLevelFlags()', () => {
+describe('parseLogLevel(logLevel).flags', () => {
   describe('logLevel=silent', () => {
     it('produces correct map', () => {
-      expect(createLogLevelFlags('silent')).to.eql({
+      expect(parseLogLevel('silent').flags).to.eql({
         silent: true,
         error: false,
         warning: false,
@@ -20,7 +20,7 @@ describe('createLogLevelFlags()', () => {
 
   describe('logLevel=error', () => {
     it('produces correct map', () => {
-      expect(createLogLevelFlags('error')).to.eql({
+      expect(parseLogLevel('error').flags).to.eql({
         silent: true,
         error: true,
         warning: false,
@@ -33,7 +33,7 @@ describe('createLogLevelFlags()', () => {
 
   describe('logLevel=warning', () => {
     it('produces correct map', () => {
-      expect(createLogLevelFlags('warning')).to.eql({
+      expect(parseLogLevel('warning').flags).to.eql({
         silent: true,
         error: true,
         warning: true,
@@ -46,7 +46,7 @@ describe('createLogLevelFlags()', () => {
 
   describe('logLevel=info', () => {
     it('produces correct map', () => {
-      expect(createLogLevelFlags('info')).to.eql({
+      expect(parseLogLevel('info').flags).to.eql({
         silent: true,
         error: true,
         warning: true,
@@ -59,7 +59,7 @@ describe('createLogLevelFlags()', () => {
 
   describe('logLevel=debug', () => {
     it('produces correct map', () => {
-      expect(createLogLevelFlags('debug')).to.eql({
+      expect(parseLogLevel('debug').flags).to.eql({
         silent: true,
         error: true,
         warning: true,
@@ -72,7 +72,7 @@ describe('createLogLevelFlags()', () => {
 
   describe('logLevel=verbose', () => {
     it('produces correct map', () => {
-      expect(createLogLevelFlags('verbose')).to.eql({
+      expect(parseLogLevel('verbose').flags).to.eql({
         silent: true,
         error: true,
         warning: true,
@@ -89,7 +89,7 @@ describe('createLogLevelFlags()', () => {
       // by specifying a long length
       const level = chance.word({ length: 10 });
 
-      expect(() => createLogLevelFlags(level))
+      expect(() => parseLogLevel(level))
         .to.throwError(level);
     });
   });

src/utils/tooling_log/log_levels.js

@@ -8,20 +8,21 @@ const LEVELS = [
   'verbose',
 ];
 
-export function createLogLevelFlags(levelLimit) {
-  const levelLimitI = LEVELS.indexOf(levelLimit);
+export function parseLogLevel(name) {
+  const i = LEVELS.indexOf(name);
 
-  if (levelLimitI === -1) {
+  if (i === -1) {
     const msg = (
-      `Invalid log level "${levelLimit}" ` +
+      `Invalid log level "${name}" ` +
       `(expected one of ${LEVELS.join(',')})`
     );
     throw new Error(msg);
   }
 
   const flags = {};
-  LEVELS.forEach((level, i) => {
-    flags[level] = i <= levelLimitI;
+  LEVELS.forEach((level, levelI) => {
+    flags[level] = levelI <= i;
   });
-  return flags;
+
+  return { name, flags };
 }

src/utils/tooling_log/tooling_log.js

@@ -1,37 +1,40 @@
 import { format } from 'util';
 import { PassThrough } from 'stream';
 
-import { createLogLevelFlags } from './log_levels';
+import { parseLogLevel } from './log_levels';
 import { magenta, yellow, red, blue, brightBlack } from 'ansicolors';
 
-export function createToolingLog(logLevel = 'silent') {
-  const logLevelFlags = createLogLevelFlags(logLevel);
+export function createToolingLog(initialLogLevelName = 'silent') {
+  // current log level (see logLevel.name and logLevel.flags) changed
+  // with ToolingLog#setLevel(newLogLevelName);
+  let logLevel = parseLogLevel(initialLogLevelName);
 
+  // current indentation level, changed with ToolingLog#indent(delta)
   let indentString = '';
 
   class ToolingLog extends PassThrough {
     verbose(...args) {
-      if (!logLevelFlags.verbose) return;
+      if (!logLevel.flags.verbose) return;
       this.write(' %s ', magenta('sill'), format(...args));
     }
 
     debug(...args) {
-      if (!logLevelFlags.debug) return;
+      if (!logLevel.flags.debug) return;
       this.write(' %s ', brightBlack('debg'), format(...args));
     }
 
     info(...args) {
-      if (!logLevelFlags.info) return;
+      if (!logLevel.flags.info) return;
       this.write(' %s ', blue('info'), format(...args));
     }
 
     warning(...args) {
-      if (!logLevelFlags.warning) return;
+      if (!logLevel.flags.warning) return;
       this.write(' %s ', yellow('warn'), format(...args));
     }
 
     error(err) {
-      if (!logLevelFlags.error) return;
+      if (!logLevel.flags.error) return;
 
       if (typeof err !== 'string' && !(err instanceof Error)) {
         err = new Error(`"${err}" thrown`);
@@ -46,6 +49,14 @@ export function createToolingLog(logLevel = 'silent') {
       return indentString.length;
     }
 
+    getLevel() {
+      return logLevel.name;
+    }
+
+    setLevel(newLogLevelName) {
+      logLevel = parseLogLevel(newLogLevelName);
+    }
+
     write(...args) {
       format(...args).split('\n').forEach((line, i) => {
         const subLineIndent = i === 0 ? '' : '       ';

tasks/config/run.js

@@ -1,6 +1,6 @@
 import { format } from 'url';
 import { resolve } from 'path';
-import chromedriver from 'chromedriver';
+
 module.exports = function (grunt) {
   const platform = require('os').platform();
   const root = p => resolve(__dirname, '../../', p);
@@ -155,34 +155,6 @@ module.exports = function (grunt) {
       ]
     },
 
-    chromeDriver: {
-      options: {
-        wait: false,
-        ready: /Starting ChromeDriver/,
-        quiet: false,
-        failOnError: false
-      },
-      cmd: chromedriver.path,
-      args: [
-        `--port=${uiConfig.servers.webdriver.port}`,
-        '--url-base=wd/hub',
-      ]
-    },
-
-    devChromeDriver: {
-      options: {
-        wait: false,
-        ready: /Starting ChromeDriver/,
-        quiet: false,
-        failOnError: false
-      },
-      cmd: chromedriver.path,
-      args: [
-        `--port=${uiConfig.servers.webdriver.port}`,
-        '--url-base=wd/hub',
-      ]
-    },
-
     optimizeBuild: {
       options: {
         wait: false,

tasks/test.js

@@ -80,12 +80,6 @@ module.exports = function (grunt) {
     'run:testUIDevServer:keepalive'
   ]);
 
-  grunt.registerTask('test:ui:runner', [
-    'checkPlugins',
-    'clean:screenshots',
-    'functionalTestRunner'
-  ]);
-
   grunt.registerTask('test:api', [
     'esvm:ui',
     'run:apiTestServer',

test/functional/services/remote/chromedriver_api/chromedriver_api.js

@@ -0,0 +1,75 @@
+import { EventEmitter } from 'events';
+
+import { createLocalChromedriverApi } from './chromedriver_local_api';
+import { createRemoteChromedriverApi } from './chromedriver_remote_api';
+import { ping } from './ping';
+
+const noop = () => {};
+
+/**
+ *  Api for interacting with a local or remote instance of Chromedriver
+ *
+ *  @type {Object}
+ */
+export class ChromedriverApi extends EventEmitter {
+  static async factory(log, url) {
+    return (await ping(url))
+      ? createRemoteChromedriverApi(log, url)
+      : createLocalChromedriverApi(log, url);
+  }
+
+  constructor(options = {}) {
+    super();
+
+    const {
+      url,
+      start = noop,
+      stop = noop,
+    } = options;
+
+    if (!url) {
+      throw new TypeError('url is a required parameter');
+    }
+
+    this._url = url;
+    this._state = undefined;
+    this._callCustomStart = () => start(this);
+    this._callCustomStop = () => stop(this);
+    this._beforeStopFns = [];
+  }
+
+  getUrl() {
+    return this._url;
+  }
+
+  beforeStop(fn) {
+    this._beforeStopFns.push(fn);
+  }
+
+  isStopped() {
+    return this._state === 'stopped';
+  }
+
+  async start() {
+    if (this._state !== undefined) {
+      throw new Error('Chromedriver can only be started once');
+    }
+
+    this._state = 'started';
+    await this._callCustomStart();
+  }
+
+  async stop() {
+    if (this._state !== 'started') {
+      throw new Error('Chromedriver can only be stopped after being started');
+    }
+
+    this._state = 'stopped';
+
+    for (const fn of this._beforeStopFns.splice(0)) {
+      await fn();
+    }
+
+    await this._callCustomStop();
+  }
+}

test/functional/services/remote/chromedriver_api/chromedriver_local_api.js

@@ -0,0 +1,55 @@
+import { spawn } from 'child_process';
+import { parse as parseUrl } from 'url';
+
+import treeKill from 'tree-kill';
+import { fromNode as fcb } from 'bluebird';
+import { path as CHROMEDRIVER_EXEC } from 'chromedriver';
+
+import { ping } from './ping';
+import { ChromedriverApi } from './chromedriver_api';
+const START_TIMEOUT = 2000;
+const PING_INTERVAL = 150;
+
+export function createLocalChromedriverApi(log, url) {
+  let proc = null;
+
+  return new ChromedriverApi({
+    url,
+
+    async start(api) {
+      const { port } = parseUrl(url);
+      log.info('Starting local chromedriver at port %d', port);
+
+      proc = spawn(CHROMEDRIVER_EXEC, [`--port=${port}`], {
+        stdio: ['ignore', 'pipe', 'pipe'],
+      });
+
+      proc.stdout.on('data', chunk => {
+        log.debug('[chromedriver:stdout]', chunk.toString('utf8').trim());
+      });
+      proc.stderr.on('data', chunk => {
+        log.debug('[chromedriver:stderr]', chunk.toString('utf8').trim());
+      });
+
+      proc.on('exit', (code) => {
+        if (!api.isStopped() || code > 0) {
+          api.emit('error', new Error(`Chromedriver exited with code ${code}`));
+        }
+      });
+
+      let pingSuccess = false;
+      let remainingAttempts = Math.floor(START_TIMEOUT / PING_INTERVAL);
+      while (!pingSuccess && (remainingAttempts--) > 0) {
+        pingSuccess = await ping(url);
+      }
+
+      if (!pingSuccess) {
+        throw new Error(`Chromedriver did not start within the ${START_TIMEOUT}ms timeout`);
+      }
+    },
+
+    async stop() {
+      await fcb(cb => treeKill(proc.pid, undefined, cb));
+    }
+  });
+}

test/functional/services/remote/chromedriver_api/chromedriver_remote_api.js

@@ -0,0 +1,12 @@
+import { ChromedriverApi } from './chromedriver_api';
+
+export function createRemoteChromedriverApi(log, url) {
+  return new ChromedriverApi({
+    url,
+
+    start() {
+      log.info(`Reusing instance at %j`, url);
+    }
+
+  });
+}

test/functional/services/remote/chromedriver_api/index.js

@@ -0,0 +1 @@
+export { ChromedriverApi } from './chromedriver_api';

test/functional/services/remote/chromedriver_api/ping.js

@@ -0,0 +1,11 @@
+import request from 'request';
+import { fromNode as fcb } from 'bluebird';
+
+export async function ping(url) {
+  try {
+    await fcb(cb => request({ url, timeout: 1000 }, cb));
+    return true;
+  } catch (err) {
+    return false;
+  }
+}

test/functional/services/remote/leadfoot_command.js

@@ -1,12 +1,12 @@
 import { delay } from 'bluebird';
 import Command from 'leadfoot/Command';
+import Server from 'leadfoot/Server';
 
-import { createTunnel } from './leadfoot_tunnel';
-import { createSession } from './leadfoot_session';
+import { initVerboseRemoteLogging } from './verbose_remote_logging';
 
 const MINUTE = 1000 * 60;
 
-export async function initLeadfootCommand({ log, tunnelConfig, lifecycle }) {
+export async function initLeadfootCommand({ log, chromedriverApi }) {
   return await Promise.race([
     (async () => {
       await delay(2 * MINUTE);
@@ -14,27 +14,32 @@ export async function initLeadfootCommand({ log, tunnelConfig, lifecycle }) {
     })(),
 
     (async () => {
-      const tunnel = await createTunnel({ log, tunnelConfig, lifecycle });
-      const session = await createSession({ log, tunnel });
+      // a `leadfoot/Server` object knows how to communicate with the webdriver
+      // backend (chromedriver in this case). it helps with session management
+      // and all communication to the remote browser go through it, so we shim
+      // some of it's methods to enable very verbose logging.
+      const server = initVerboseRemoteLogging(log, new Server(chromedriverApi.getUrl()));
 
-      const command = new Command(session);
+      // by default, calling server.createSession() automatically fixes the webdriver
+      // "capabilities" hash so that leadfoot knows the hoops it has to jump through
+      // to have feature compliance. This is sort of like building "$.support" in jQuery.
+      // Unfortunately this process takes a couple seconds, so if we let leadfoot
+      // do it and we have an error, are killed, or for any other reason have to
+      // teardown we won't have a session object until the auto-fixing is complete.
+      //
+      // To avoid this we disable auto-fixing with this flag and call
+      // `server._fillCapabilities()` ourselves to do the fixing once we have a reference
+      // to the session and have registered it for teardown before stopping the
+      // chromedriverApi.
+      server.fixSessionCapabilities = false;
+      const session = await server.createSession({ browserName: 'chrome' });
+      chromedriverApi.beforeStop(async () => session.quit());
+      await server._fillCapabilities(session);
 
-      lifecycle.on('cleanup', async () => {
-        log.verbose('remote: closing leadfoot remote');
-        await command.quit();
-
-        log.verbose('remote: closing digdug tunnel');
-        await tunnel.stop();
-      });
-
-      log.verbose('remote: created leadfoot command');
-      tunnel.on('stdout', chunk => log.verbose('Tunnel [stdout]:', chunk.toString('utf8').trim()));
-      tunnel.on('stderr', chunk => log.verbose('Tunnel [stderr]:', chunk.toString('utf8').trim()));
-
-      // command looks like a promise beacuse it has a then function
-      // so we wrap it in an object to prevent our promise from trying to unwrap/resolve
-      // the remote
-      return { command };
+      // command looks like a promise beacuse it has a `.then()` function
+      // so we wrap it in an object to prevent async/await from trying to
+      // unwrap/resolve it
+      return { command: new Command(session) };
     })()
   ]);
 }

test/functional/services/remote/leadfoot_session.js

@@ -1,16 +0,0 @@
-import Server from 'leadfoot/Server';
-
-// intern.Runner#loadTestModules
-//   intern.Runner#_createSuites
-export async function createSession({ log, tunnel }) {
-  const server = new Server(tunnel.clientUrl);
-  log.verbose('remote: created leadfoot server');
-
-  const session = await server.createSession({
-    browserName: 'chrome'
-  });
-
-  log.verbose('remote: created leadfoot session');
-
-  return session;
-}

test/functional/services/remote/leadfoot_tunnel.js

@@ -1,29 +0,0 @@
-import SeleniumTunnel from 'digdug/SeleniumTunnel';
-import uuid from 'node-uuid';
-
-// intern.Runner#loadTunnel
-export async function createTunnel({ log, tunnelConfig }) {
-  const tunnel = new SeleniumTunnel({
-    // https://git.io/vDnfv
-    ...tunnelConfig,
-    drivers: ['chrome'],
-    tunnelId: uuid.v4()
-  });
-
-  // override https://git.io/vDnfe so shutdown is fast
-  tunnel._stop = async () => {
-    const proc = tunnel._process;
-
-    if (!proc) {
-      log.error('Update to stop tunnel, child process not found');
-      return;
-    }
-
-    const exitted = new Promise(resolve => proc.on('exit', resolve));
-    tunnel._process.kill('SIGTERM');
-    await exitted;
-  };
-
-  await tunnel.start();
-  return tunnel;
-}

test/functional/services/remote/remote.js

@@ -1,17 +1,18 @@
 import { initLeadfootCommand } from './leadfoot_command';
 import { createRemoteInterceptors } from './interceptors';
+import { ChromedriverApi } from './chromedriver_api';
 
 export async function RemoteProvider({ getService }) {
   const lifecycle = getService('lifecycle');
   const config = getService('config');
   const log = getService('log');
 
-  const { command } = await initLeadfootCommand({
-    log,
-    lifecycle,
-    tunnelConfig: config.get('servers.webdriver'),
-  });
+  const chromedriverApi = await ChromedriverApi.factory(log, config.get('chromedriver.url'));
+  lifecycle.on('cleanup', async () => await chromedriverApi.stop());
 
+  await chromedriverApi.start();
+
+  const { command } = await initLeadfootCommand({ log, chromedriverApi });
   const interceptors = createRemoteInterceptors(command);
 
   log.info('Remote initialized');

test/functional/services/remote/verbose_remote_logging.js

@@ -0,0 +1,37 @@
+import { green, magenta } from 'ansicolors';
+
+export function initVerboseRemoteLogging(log, server) {
+  const wrap = (original, httpMethod) => (path, requestData, pathParts) => {
+    const url = '/' + path.split('/').slice(2).join('/').replace(/\$(\d)/, function (_, index) {
+      return encodeURIComponent(pathParts[index]);
+    });
+
+    if (requestData == null) {
+      log.verbose('[remote] >  %s %s', httpMethod, url);
+    } else {
+      log.verbose('[remote] >  %s %s %j', httpMethod, url, requestData);
+    }
+
+    return original.call(server, path, requestData, pathParts)
+    .then(result => {
+      log.verbose(`[remote]  < %s %s ${green('OK')}`, httpMethod, url);
+      return result;
+    })
+    .catch(error => {
+      let message;
+      try {
+        message = JSON.parse(error.response.data).value.message;
+      } catch (err) {
+        message = err.message;
+      }
+
+      log.verbose(`[remote]  < %s %s ${magenta('ERR')} %j`, httpMethod, url, message.split(/\r?\n/)[0]);
+      throw error;
+    });
+  };
+
+  server._get = wrap(server._get, 'GET');
+  server._post = wrap(server._post, 'POST');
+  server._delete = wrap(server._delete, 'DELETE');
+  return server;
+}

test/server_config.js

@@ -3,11 +3,6 @@ const kibanaURL = '/app/kibana';
 
 module.exports = {
   servers: {
-    webdriver: {
-      protocol: process.env.TEST_WEBDRIVER_PROTOCOL || 'http',
-      hostname: process.env.TEST_WEBDRIVER_HOSTNAME || 'localhost',
-      port: parseInt(process.env.TEST_WEBDRIVER_PORT, 10) || 4444
-    },
     kibana: {
       protocol: process.env.TEST_KIBANA_PROTOCOL || 'http',
       hostname: process.env.TEST_KIBANA_HOSTNAME || 'localhost',