[ftr] make room for more projects (#11848)

Squashed commit of the following:

commit 659ea986fdeb9a5ff2ca1fa5360cccb01c671ede
Author: spalger <spalger@users.noreply.github.com>
Date:   Wed May 17 09:19:22 2017 -0700

    [test/*/fixtures] rename es_archives to es_archiver

commit d3667457c78e88e2d6974f3c38dd0fe61b846b91
Author: spalger <spalger@users.noreply.github.com>
Date:   Wed May 17 08:22:03 2017 -0700

    [ftr/config] fix default directory value functions

commit 9a6a2cc0b295e2281e83da04fcea40e9d6f00781
Author: spalger <spalger@users.noreply.github.com>
Date:   Wed May 17 07:30:52 2017 -0700

    fix import paths

commit fcb65a877d54b5b1d36b8c81f1264b36845e826e
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 21:39:57 2017 -0700

    [grunt/ftr] use named exports for configs

commit 7d7f38c7615cdbf8eb0119efc0f2a5188bca8792
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:50:04 2017 -0700

    [test] remove unnecessary directory definitions

commit 0c28984669768482f0a2ee7fc2800d5bcaf49025
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:46:29 2017 -0700

    [ftr/config] make default directories relative to config path

commit cd2f33612624cacffec138797f3fc0f4ecb46cca
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:32:12 2017 -0700

    [test/common] put server config into common

commit 7851ed811a236576c63bd20850b3ef2099be2a4e
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:18:20 2017 -0700

    [grunt] "deprecate" test:api:runner task

commit b2ac4c26593a1947c94f0168191fe8123ff74122
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 18:15:41 2017 -0700

    [ftr] accept the project name as an unnamed arg

commit 47e292894fc70c0a04883403c50c5d2ae0738d76
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 17:56:34 2017 -0700

    [ftr/grunt] convert ftr task to multi-task with config

commit 83375855f88e5e7b3fa8b6a1c5d24a9f54766ce5
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 17:54:54 2017 -0700

    [test/functional] move fixtures into test/functional project

commit 05994e9c92cf134c58f831c285b3b522a801acbc
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 17:02:51 2017 -0700

    [src/test_utils] merge with test/utils directory

commit c77ee5ed36b8b7eadf876cb6d9482a49dfc92b66
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 16:53:00 2017 -0700

    [test/api_integration] migrate api tests to functional test runner

commit ca328c34648dd7e07f70e1844e07cfc392e41103
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 16:50:12 2017 -0700

    [esArchiver] refresh modified indices after load

commit cde74a540850fd97578f441d6dccaefd1444e656
Author: spalger <spalger@users.noreply.github.com>
Date:   Tue May 16 16:46:48 2017 -0700

    [test/functional] move shared services into test/common

commit 0ea2646aea5817f6d1595e6ae0d356c426f138f0
Author: spalger <spalger@users.noreply.github.com>
Date:   Mon May 15 22:51:23 2017 -0700

    [scripts/mocha] run _mocha script when debugging

commit 1cc80600d90e318d4738920aa557d124075a4570
Author: spalger <spalger@users.noreply.github.com>
Date:   Mon May 15 22:48:12 2017 -0700

    [ftr/config] allow child config files to have no testFiles config

commit 2bb6c957443b18cebc419baa6f9db301c8f4dc4f
Author: spalger <spalger@users.noreply.github.com>
Date:   Mon May 15 21:52:26 2017 -0700

    [ftr] move screenshots into test/functional
(cherry picked from commit d3ba428796)

.gitignore

@@ -12,10 +12,10 @@ target
 .idea
 *.iml
 *.log
-/test/screenshots/diff
-/test/screenshots/failure
-/test/screenshots/session
-/test/screenshots/visual_regression_gallery.html
+/test/*/screenshots/diff
+/test/*/screenshots/failure
+/test/*/screenshots/session
+/test/*/screenshots/visual_regression_gallery.html
 /html_docs
 /esvm
 .htpasswd

Gruntfile.js

@@ -12,7 +12,6 @@ module.exports = function (grunt) {
     plugins: __dirname + '/src/core_plugins',
     server: __dirname + '/src/server',
     target: __dirname + '/target', // location of the compressed build targets
-    testUtilsDir: __dirname + '/src/test_utils',
     configFile: __dirname + '/src/config/kibana.yml',
 
     karmaBrowser: (function () {

package.json

@@ -137,7 +137,7 @@
     "inert": "4.0.2",
     "jade": "1.11.0",
     "jade-loader": "0.7.1",
-    "joi": "6.6.1",
+    "joi": "10.4.1",
     "jquery": "2.2.4",
     "js-yaml": "3.4.1",
     "json-loader": "0.5.3",
@@ -272,7 +272,7 @@
     "source-map": "0.5.6",
     "source-map-support": "0.2.10",
     "strip-ansi": "^3.0.1",
-    "supertest": "1.2.0",
+    "supertest": "3.0.0",
     "supertest-as-promised": "2.0.2",
     "tree-kill": "1.1.0",
     "webpack-dev-server": "1.14.1"

scripts/mocha.js

@@ -1 +1,2 @@
-require('mocha/bin/mocha');
+require('../src/optimize/babel/register');
+require('../test/scripts/run_mocha');

src/core_plugins/elasticsearch/lib/__tests__/ensure_es_version.js

@@ -5,7 +5,7 @@ import expect from 'expect.js';
 import url from 'url';
 import SetupError from '../setup_error';
 
-import serverConfig from '../../../../../test/server_config';
+import { esTestServerUrlParts } from '../../../../../test/es_test_server_url_parts';
 import { ensureEsVersion } from '../ensure_es_version';
 
 describe('plugins/elasticsearch', () => {
@@ -27,7 +27,7 @@ describe('plugins/elasticsearch', () => {
             status: {
               red: sinon.stub()
             },
-            url: url.format(serverConfig.servers.elasticsearch)
+            url: url.format(esTestServerUrlParts)
           }
         }
       };

src/core_plugins/elasticsearch/lib/__tests__/health_check.js

@@ -8,10 +8,10 @@ const NoConnections = require('elasticsearch').errors.NoConnections;
 import mappings from './fixtures/mappings';
 import healthCheck from '../health_check';
 import kibanaVersion from '../kibana_version';
-import serverConfig from '../../../../../test/server_config';
+import { esTestServerUrlParts } from '../../../../../test/es_test_server_url_parts';
 
-const esPort = serverConfig.servers.elasticsearch.port;
-const esUrl = url.format(serverConfig.servers.elasticsearch);
+const esPort = esTestServerUrlParts.port;
+const esUrl = url.format(esTestServerUrlParts);
 
 describe('plugins/elasticsearch', () => {
   describe('lib/health_check', function () {

src/core_plugins/elasticsearch/lib/__tests__/routes.js

@@ -1,6 +1,6 @@
 import { format } from 'util';
 
-import * as kbnTestServer from '../../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../../test_utils/kbn_server';
 import { fromRoot } from '../../../../utils';
 
 describe('plugins/elasticsearch', function () {

src/core_plugins/kibana/server/lib/__tests__/manage_uuid.js

@@ -1,6 +1,6 @@
 import expect from 'expect.js';
 import sinon from 'sinon';
-import * as kbnTestServer from '../../../../../../test/utils/kbn_server.js';
+import * as kbnTestServer from '../../../../../test_utils/kbn_server.js';
 import { fromRoot } from '../../../../../utils';
 import manageUuid from '../manage_uuid';
 

src/core_plugins/tests_bundle/index.js

@@ -71,7 +71,7 @@ export default (kibana) => {
         ng_mock$: fromRoot('src/core_plugins/dev_mode/public/ng_mock'),
         'angular-mocks$': require.resolve('./webpackShims/angular-mocks'),
         fixtures: fromRoot('src/fixtures'),
-        test_utils: fromRoot('src/test_utils'),
+        test_utils: fromRoot('src/test_utils/public'),
       }
     }
   });

src/es_archiver/actions/load.js

@@ -31,8 +31,14 @@ export async function loadAction({ name, skipExisting, client, dataDir, log }) {
     ]);
   }
 
+  const indicesToRefresh = [];
   stats.forEachIndex((index, { docs }) => {
     log.info('[%s] Indexed %d docs into %j', name, docs.indexed, index);
+    indicesToRefresh.push(index);
+  });
+
+  await client.indices.refresh({
+    index: indicesToRefresh
   });
 
   return stats.toJSON();

src/functional_test_runner/__tests__/lib/kibana.js

@@ -1,6 +1,6 @@
 import { resolve } from 'path';
 
-import { createServer } from '../../../../test/utils/kbn_server';
+import { createServer } from '../../../test_utils/kbn_server';
 
 export async function startupKibana({ port, esUrl }) {
   const server = createServer({

src/functional_test_runner/cli.js

@@ -6,7 +6,6 @@ import { createToolingLog } from '../utils';
 import { createFunctionalTestRunner } from './functional_test_runner';
 
 const cmd = new Command('node scripts/functional_test_runner');
-
 const resolveConfigPath = v => resolve(process.cwd(), v);
 const defaultConfigPath = resolveConfigPath('test/functional/config.js');
 
@@ -20,13 +19,6 @@ cmd
   .option('--debug', 'Run in debug mode', false)
   .parse(process.argv);
 
-if (!cmd.config) {
-  console.log('');
-  console.log('  --config is a required parameter');
-  console.log('');
-  process.exit(1);
-}
-
 let logLevel = 'info';
 if (cmd.silent) logLevel = 'silent';
 if (cmd.quiet) logLevel = 'error';

src/functional_test_runner/lib/config/__tests__/fixtures/config.1.js

@@ -0,0 +1,7 @@
+export default function () {
+  return {
+    testFiles: [
+      'config.1'
+    ]
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.2.js

@@ -0,0 +1,10 @@
+export default async function ({ readConfigFile }) {
+  const config1 = await readConfigFile(require.resolve('./config.1.js'));
+
+  return {
+    testFiles: [
+      ...config1.get('testFiles'),
+      'config.2'
+    ]
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.3.js

@@ -0,0 +1,11 @@
+export default async function ({ readConfigFile }) {
+  const config4 = await readConfigFile(require.resolve('./config.4'));
+  return {
+    testFiles: [
+      'baz'
+    ],
+    screenshots: {
+      ...config4.get('screenshots')
+    }
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.4.js

@@ -0,0 +1,7 @@
+export default function () {
+  return {
+    screenshots: {
+      directory: 'bar'
+    }
+  };
+}

src/functional_test_runner/lib/config/__tests__/fixtures/config.invalid.js

@@ -0,0 +1,5 @@
+export default async function () {
+  return {
+    foo: 'bar'
+  };
+}

src/functional_test_runner/lib/config/__tests__/read_config_file.js

@@ -0,0 +1,58 @@
+import expect from 'expect.js';
+
+import { createToolingLog } from '../../../../utils';
+import { readConfigFile } from '../read_config_file';
+import { Config } from '../config';
+
+const log = createToolingLog().resume();
+
+describe('readConfigFile()', () => {
+  it('reads config from a file, returns an instance of Config class', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.1'));
+    expect(config).to.be.a(Config);
+    expect(config.get('testFiles')).to.eql([
+      'config.1'
+    ]);
+  });
+
+  it('merges setting overrides into log', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.1'), {
+      screenshots: {
+        directory: 'foo.bar'
+      }
+    });
+
+    expect(config.get('screenshots.directory')).to.be('foo.bar');
+  });
+
+  it('supports loading config files from within config files', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.2'));
+    expect(config.get('testFiles')).to.eql([
+      'config.1',
+      'config.2',
+    ]);
+  });
+
+  it('throws if settings are invalid', async () => {
+    try {
+      await readConfigFile(log, require.resolve('./fixtures/config.invalid'));
+      throw new Error('expected readConfigFile() to fail');
+    } catch (err) {
+      expect(err.message).to.match(/"foo"/);
+    }
+  });
+
+  it('throws if config does not define testFiles', async () => {
+    try {
+      await readConfigFile(log, require.resolve('./fixtures/config.4'));
+      throw new Error('expected readConfigFile() to fail');
+    } catch (err) {
+      expect(err.message).to.match(/"testFiles"/);
+    }
+  });
+
+  it('does not throw if child config file does not have any testFiles', async () => {
+    const config = await readConfigFile(log, require.resolve('./fixtures/config.3'));
+    expect(config.get('screenshots.directory')).to.be('bar');
+  });
+});

src/functional_test_runner/lib/config/config.js

@@ -6,8 +6,25 @@ import { schema } from './schema';
 const $values = Symbol('values');
 
 export class Config {
-  constructor(settings) {
-    const { error, value } = schema.validate(settings);
+  constructor(options = {}) {
+    const {
+      settings = {},
+      primary = false,
+      path,
+    } = options;
+
+    if (!path) {
+      throw new TypeError('path is a required option');
+    }
+
+    const { error, value } = schema.validate(settings, {
+      abortEarly: false,
+      context: {
+        primary: !!primary,
+        path,
+      }
+    });
+
     if (error) throw error;
     this[$values] = value;
   }

src/functional_test_runner/lib/config/read_config_file.js

@@ -3,29 +3,37 @@ import { defaultsDeep } from 'lodash';
 import { Config } from './config';
 import { transformDeprecations } from './transform_deprecations';
 
-export async function readConfigFile(log, configFile, settingOverrides = {}) {
-  log.debug('Loading config file from %j', configFile);
+async function getSettingsFromFile(log, path, settingOverrides) {
+  log.debug('Loading config file from %j', path);
 
-  const configModule = require(configFile);
+  const configModule = require(path);
   const configProvider = configModule.__esModule
     ? configModule.default
     : configModule;
 
-  const settings = defaultsDeep(
+  const settingsWithDefaults = defaultsDeep(
     {},
     settingOverrides,
     await configProvider({
       log,
-
-      // give a version of the readConfigFile function to
-      // the config file that already has has the logger bound
-      readConfigFile: async (...args) => (
-        await readConfigFile(log, ...args)
-      )
+      async readConfigFile(...args) {
+        return new Config({
+          settings: await getSettingsFromFile(log, ...args),
+          primary: false,
+          path,
+        });
+      }
     })
   );
 
-  return new Config(transformDeprecations(settings, msg => {
-    log.error(msg);
-  }));
+  const logDeprecation = (...args) => log.error(...args);
+  return transformDeprecations(settingsWithDefaults, logDeprecation);
+}
+
+export async function readConfigFile(log, path, settingOverrides) {
+  return new Config({
+    settings: await getSettingsFromFile(log, path, settingOverrides),
+    primary: true,
+    path,
+  });
 }

src/functional_test_runner/lib/config/schema.js

@@ -1,3 +1,5 @@
+import { resolve, dirname } from 'path';
+
 import Joi from 'joi';
 
 import { ConsoleReporterProvider } from '../reporters';
@@ -18,8 +20,20 @@ const urlPartsSchema = () => Joi.object().keys({
   hash: Joi.string().regex(/^\//, 'start with a /')
 }).default();
 
+const defaultRelativeToConfigPath = path => {
+  const makeDefault = (locals, options) => (
+    resolve(dirname(options.context.path), path)
+  );
+  makeDefault.description = `<config.js directory>/${path}`;
+  return makeDefault;
+};
+
 export const schema = Joi.object().keys({
-  testFiles: Joi.array().items(Joi.string()).required(),
+  testFiles: Joi.array().items(Joi.string()).when('$primary', {
+    is: true,
+    then: Joi.required(),
+    otherwise: Joi.default([]),
+  }),
 
   services: Joi.object().pattern(
     ID_PATTERN,
@@ -74,11 +88,11 @@ export const schema = Joi.object().keys({
 
   // settings for the esArchiver module
   esArchiver: Joi.object().keys({
-    directory: Joi.string().required()
-  }),
+    directory: Joi.string().default(defaultRelativeToConfigPath('fixtures/es_archiver'))
+  }).default(),
 
   // settings for the screenshots module
   screenshots: Joi.object().keys({
-    directory: Joi.string().required()
-  }),
+    directory: Joi.string().default(defaultRelativeToConfigPath('screenshots'))
+  }).default(),
 }).default();

src/server/__tests__/base_path.js

@@ -1,6 +1,6 @@
 import expect from 'expect.js';
 
-import * as kbnTestServer from '../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../test_utils/kbn_server';
 const basePath = '/kibana';
 
 describe('Server basePath config', function () {

src/server/http/__tests__/index.js

@@ -1,5 +1,5 @@
 import expect from 'expect.js';
-import * as kbnTestServer from '../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../test_utils/kbn_server';
 import { fromRoot } from '../../../utils';
 
 describe('routes', function () {

src/server/http/__tests__/version_check.js

@@ -1,7 +1,7 @@
 import expect from 'expect.js';
 import { fromNode } from 'bluebird';
 import { resolve } from 'path';
-import * as kbnTestServer from '../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../test_utils/kbn_server';
 
 const src = resolve.bind(null, __dirname, '../../../../src');
 

src/server/http/__tests__/xsrf.js

@@ -1,7 +1,7 @@
 import expect from 'expect.js';
 import { fromNode as fn } from 'bluebird';
 import { resolve } from 'path';
-import * as kbnTestServer from '../../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../../test_utils/kbn_server';
 
 const nonDestructiveMethods = ['GET', 'HEAD'];
 const destructiveMethods = ['POST', 'PUT', 'DELETE'];

src/test_utils/kbn_server.js

@@ -1,9 +1,9 @@
 import url from 'url';
 import { defaultsDeep, set } from 'lodash';
 import { header as basicAuthHeader } from './base_auth';
-import { kibanaUser, kibanaServer } from '../shield';
+import { kibanaUser, kibanaServer } from '../../test/shield';
+import { esTestServerUrlParts } from '../../test/es_test_server_url_parts';
 import KbnServer from '../../src/server/kbn_server';
-import serverConfig from '../server_config';
 
 const SERVER_DEFAULTS = {
   server: {
@@ -20,7 +20,7 @@ const SERVER_DEFAULTS = {
     enabled: false
   },
   elasticsearch: {
-    url: url.format(serverConfig.servers.elasticsearch),
+    url: url.format(esTestServerUrlParts),
     username: kibanaServer.username,
     password: kibanaServer.password
   }

src/test_utils/public/no_digest_promises.js

@@ -8,7 +8,7 @@ Bluebird.longStackTraces();
  * replace the Promise service with Bluebird so that tests
  * can use promises without having to call $rootScope.apply()
  *
- * let noDigestPromises = require('test_utils/no_digest_promises');
+ * import noDigestPromises from 'test_utils/no_digest_promises';
  *
  * describe('some module that does complex shit with promises', function () {
  *   beforeEach(noDigestPromises.activate);

src/ui/__tests__/ui_exports.js

@@ -2,7 +2,7 @@ import expect from 'expect.js';
 import { resolve } from 'path';
 
 import UiExports from '../ui_exports';
-import * as kbnTestServer from '../../../test/utils/kbn_server';
+import * as kbnTestServer from '../../test_utils/kbn_server';
 
 describe('UiExports', function () {
   describe('#find()', function () {

src/ui/public/utils/__tests__/scanner.js

@@ -6,7 +6,7 @@ import ngMock from 'ng_mock';
 import sinon from 'sinon';
 import url from 'url';
 
-import serverConfig from '../../../../../test/server_config';
+import { esTestServerUrlParts } from '../../../../../test/es_test_server_url_parts';
 
 describe('Scanner', function () {
   let es;
@@ -14,7 +14,7 @@ describe('Scanner', function () {
   beforeEach(ngMock.module('kibana'));
   beforeEach(ngMock.inject(function (esFactory) {
     es = esFactory({
-      host: url.format(serverConfig.servers.elasticsearch),
+      host: url.format(esTestServerUrlParts),
       defer: function () {
         return Bluebird.defer();
       }

tasks/config/esvm.js

@@ -1,4 +1,5 @@
-import serverConfig from '../../test/server_config';
+import { esTestServerUrlParts } from '../../test/es_test_server_url_parts';
+
 module.exports = function (grunt) {
   const resolve = require('path').resolve;
   const directory = resolve(__dirname, '../../esvm');
@@ -11,6 +12,10 @@ module.exports = function (grunt) {
       config: {
         http: {
           port: 9200
+        },
+        script: {
+          inline: true,
+          stored: true
         }
       }
     },
@@ -88,7 +93,7 @@ module.exports = function (grunt) {
         purge: true,
         config: {
           http: {
-            port: serverConfig.servers.elasticsearch.port
+            port: esTestServerUrlParts.port
           },
           cluster: {
             name: 'esvm-test'
@@ -97,7 +102,7 @@ module.exports = function (grunt) {
             zen: {
               ping: {
                 unicast: {
-                  hosts: [ `localhost:${serverConfig.servers.elasticsearch.port}` ]
+                  hosts: [ `localhost:${esTestServerUrlParts.port}` ]
                 }
               }
             }
@@ -112,7 +117,7 @@ module.exports = function (grunt) {
         purge: true,
         config: {
           http: {
-            port: serverConfig.servers.elasticsearch.port
+            port: esTestServerUrlParts.port
           },
           cluster: {
             name: 'esvm-ui'
@@ -121,7 +126,7 @@ module.exports = function (grunt) {
             zen: {
               ping: {
                 unicast: {
-                  hosts: [ `localhost:${serverConfig.servers.elasticsearch.port}` ]
+                  hosts: [ `localhost:${esTestServerUrlParts.port}` ]
                 }
               }
             }

tasks/config/functional_test_runner.js

@@ -0,0 +1,13 @@
+export const functional = {
+  options: {
+    logLevel: 'debug',
+    configFile: require.resolve('../../test/functional/config.js')
+  }
+};
+
+export const apiIntegration = {
+  options: {
+    logLevel: 'debug',
+    configFile: require.resolve('../../test/api_integration/config.js')
+  }
+};

tasks/config/run.js

@@ -1,12 +1,13 @@
 import { format } from 'url';
-import { resolve } from 'path';
+import { esTestServerUrlParts } from '../../test/es_test_server_url_parts';
+import { kibanaTestServerUrlParts } from '../../test/kibana_test_server_url_parts';
 
 module.exports = function (grunt) {
   const platform = require('os').platform();
-  const root = p => resolve(__dirname, '../../', p);
   const binScript =  /^win/.test(platform) ? '.\\bin\\kibana.bat' : './bin/kibana';
   const buildScript =  /^win/.test(platform) ? '.\\build\\kibana\\bin\\kibana.bat' : './build/kibana/bin/kibana';
-  const uiConfig = require(root('test/server_config'));
+  const pkgVersion = grunt.config.get('pkg.version');
+  const releaseBinScript = `./build/kibana-${pkgVersion}-linux-x86_64/bin/kibana`;
 
   const stdDevArgs = [
     '--env.name=development',
@@ -54,8 +55,8 @@ module.exports = function (grunt) {
       args: [
         ...stdDevArgs,
         '--optimize.enabled=false',
-        '--elasticsearch.url=' + format(uiConfig.servers.elasticsearch),
-        '--server.port=' + uiConfig.servers.kibana.port,
+        '--elasticsearch.url=' + format(esTestServerUrlParts),
+        '--server.port=' + kibanaTestServerUrlParts.port,
         '--server.xsrf.disableProtection=true',
         ...kbnServerFlags,
       ]
@@ -71,8 +72,24 @@ module.exports = function (grunt) {
       cmd: binScript,
       args: [
         ...stdDevArgs,
-        '--server.port=' + uiConfig.servers.kibana.port,
-        '--elasticsearch.url=' + format(uiConfig.servers.elasticsearch),
+        '--server.port=' + kibanaTestServerUrlParts.port,
+        '--elasticsearch.url=' + format(esTestServerUrlParts),
+        ...kbnServerFlags,
+      ]
+    },
+
+    testUIReleaseServer: {
+      options: {
+        wait: false,
+        ready: /Server running/,
+        quiet: false,
+        failOnError: false
+      },
+      cmd: releaseBinScript,
+      args: [
+        ...stdDevArgs,
+        '--server.port=' + kibanaTestServerUrlParts.port,
+        '--elasticsearch.url=' + format(esTestServerUrlParts),
         ...kbnServerFlags,
       ]
     },
@@ -87,8 +104,8 @@ module.exports = function (grunt) {
       cmd: binScript,
       args: [
         ...stdDevArgs,
-        '--server.port=' + uiConfig.servers.kibana.port,
-        '--elasticsearch.url=' + format(uiConfig.servers.elasticsearch),
+        '--server.port=' + kibanaTestServerUrlParts.port,
+        '--elasticsearch.url=' + format(esTestServerUrlParts),
         '--dev',
         '--no-base-path',
         '--no-ssl',

tasks/config/simplemocha.js

@@ -15,11 +15,5 @@ module.exports = {
       '!src/**/public/**',
       '!**/_*.js'
     ]
-  },
-  api: {
-    src: [
-      'test/mocha_setup.js',
-      'test/unit/**/*.js'
-    ]
   }
 };

tasks/functional_test_runner.js

@@ -1,14 +1,15 @@
-import { resolve } from 'path';
-
 import moment from 'moment';
-
 import { createFunctionalTestRunner } from '../src/functional_test_runner';
 import { createToolingLog, createMapStream } from '../src/utils';
 
 export default function (grunt) {
-  grunt.registerTask('functionalTestRunner', function () {
-    const log = createToolingLog('debug');
+  grunt.registerMultiTask('functional_test_runner', 'run tests with the functional test runner', function () {
+    const {
+      logLevel,
+      configFile,
+    } = this.options();
 
+    const log = createToolingLog(logLevel);
     log
       .pipe(createMapStream(line => {
         return `${moment().format('hh:mm:ss.SSS')} ${line}`;
@@ -17,7 +18,7 @@ export default function (grunt) {
 
     const functionalTestRunner = createFunctionalTestRunner({
       log,
-      configFile: resolve(__dirname, '../test/functional/config.js'),
+      configFile,
     });
 
     const callback = this.async();

tasks/test.js

@@ -58,7 +58,7 @@ module.exports = function (grunt) {
     'checkPlugins',
     'esvm:ui',
     'run:testUIServer',
-    'functionalTestRunner',
+    'functional_test_runner:functional',
     'esvm_shutdown:ui',
     'stop:testUIServer'
   ]);
@@ -67,7 +67,7 @@ module.exports = function (grunt) {
     'checkPlugins',
     'esvm:ui',
     'run:testUIReleaseServer',
-    'functionalTestRunner',
+    'functional_test_runner:functional',
     'esvm_shutdown:ui',
     'stop:testUIReleaseServer'
   ]);
@@ -81,7 +81,7 @@ module.exports = function (grunt) {
   grunt.registerTask('test:api', [
     'esvm:ui',
     'run:apiTestServer',
-    'simplemocha:api',
+    'functional_test_runner:apiIntegration',
     'esvm_shutdown:ui',
     'stop:apiTestServer'
   ]);
@@ -91,9 +91,9 @@ module.exports = function (grunt) {
     'run:apiTestServer:keepalive'
   ]);
 
-  grunt.registerTask('test:api:runner', [
-    'simplemocha:api'
-  ]);
+  grunt.registerTask('test:api:runner', () => {
+    grunt.fail.fatal('test:api:runner has moved, use: `node scripts/function_test_runner --config test/api_integration/config.js`');
+  });
 
   grunt.registerTask('test', subTask => {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);

test/api_integration/apis/index.js

@@ -0,0 +1,6 @@
+export default function ({ loadTestFile }) {
+  describe('apis', () => {
+    loadTestFile(require.resolve('./scripts'));
+    loadTestFile(require.resolve('./search'));
+  });
+}

test/api_integration/apis/scripts/index.js

@@ -0,0 +1,5 @@
+export default function ({ loadTestFile }) {
+  describe('scripts', () => {
+    loadTestFile(require.resolve('./languages'));
+  });
+}

test/api_integration/apis/scripts/languages.js

@@ -0,0 +1,25 @@
+import expect from 'expect.js';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+
+  describe('Script Languages API', function getLanguages() {
+    it('should return 200 with an array of languages', () => (
+      supertest.get('/api/kibana/scripts/languages')
+        .expect(200)
+        .then((response) => {
+          expect(response.body).to.be.an('array');
+        })
+    ));
+
+    it('should only return langs enabled for inline scripting', () => (
+      supertest.get('/api/kibana/scripts/languages')
+        .expect(200)
+        .then((response) => {
+          expect(response.body).to.contain('expression');
+          expect(response.body).to.contain('painless');
+          expect(response.body).to.not.contain('groovy');
+        })
+    ));
+  });
+}

test/api_integration/apis/search/count.js

@@ -0,0 +1,42 @@
+import expect from 'expect.js';
+
+export default function ({ getService }) {
+  const esArchiver = getService('esArchiver');
+  const supertest = getService('supertest');
+
+  describe('Count API', function postIngest() {
+    before(() => esArchiver.load('search/count'));
+    after(() => esArchiver.unload('search/count'));
+
+    it('should return 200 with a document count for existing indices', () => (
+      supertest
+        .post('/api/kibana/foo-*/_count')
+        .expect(200)
+        .then((response) => {
+          expect(response.body.count).to.be(2);
+        })
+    ));
+
+    it('should support GET requests as well', () => (
+      supertest
+        .get('/api/kibana/foo-*/_count')
+        .expect(200)
+        .then((response) => {
+          expect(response.body.count).to.be(2);
+        })
+    ));
+
+    it('should return 404 if a pattern matches no indices', () => (
+      supertest
+        .post('/api/kibana/doesnotexist-*/_count')
+        .expect(404)
+    ));
+
+    it('should return 404 if a concrete index does not exist', () => (
+      supertest
+        .post('/api/kibana/concrete/_count')
+        .expect(404)
+    ));
+
+  });
+}

test/api_integration/apis/search/index.js

@@ -0,0 +1,5 @@
+export default function ({ loadTestFile }) {
+  describe('search', () => {
+    loadTestFile(require.resolve('./count'));
+  });
+}

test/api_integration/config.js

@@ -0,0 +1,19 @@
+import {
+  SupertestProvider,
+} from './services';
+
+export default async function ({ readConfigFile }) {
+  const commonConfig = await readConfigFile(require.resolve('../common/config'));
+
+  return {
+    testFiles: [
+      require.resolve('./apis'),
+    ],
+    services: {
+      es: commonConfig.get('services.es'),
+      esArchiver: commonConfig.get('services.esArchiver'),
+      supertest: SupertestProvider,
+    },
+    servers: commonConfig.get('servers'),
+  };
+}

test/api_integration/fixtures/es_archiver/search/count/mappings.json

@@ -0,0 +1,304 @@
+{
+  "type": "index",
+  "value": {
+    "index": ".kibana",
+    "settings": {
+      "index": {
+        "mapping": {
+          "single_type": "false"
+        },
+        "number_of_shards": "1",
+        "mapper": {
+          "dynamic": "false"
+        },
+        "number_of_replicas": "1"
+      }
+    },
+    "mappings": {
+      "url": {
+        "dynamic": "strict",
+        "properties": {
+          "accessCount": {
+            "type": "long"
+          },
+          "accessDate": {
+            "type": "date"
+          },
+          "createDate": {
+            "type": "date"
+          },
+          "url": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 2048
+              }
+            }
+          }
+        }
+      },
+      "timelion-sheet": {
+        "dynamic": "strict",
+        "properties": {
+          "description": {
+            "type": "text"
+          },
+          "hits": {
+            "type": "integer"
+          },
+          "kibanaSavedObjectMeta": {
+            "properties": {
+              "searchSourceJSON": {
+                "type": "text"
+              }
+            }
+          },
+          "timelion_chart_height": {
+            "type": "integer"
+          },
+          "timelion_columns": {
+            "type": "integer"
+          },
+          "timelion_interval": {
+            "type": "keyword"
+          },
+          "timelion_other_interval": {
+            "type": "keyword"
+          },
+          "timelion_rows": {
+            "type": "integer"
+          },
+          "timelion_sheet": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "version": {
+            "type": "integer"
+          }
+        }
+      },
+      "_default_": {
+        "dynamic": "strict"
+      },
+      "visualization": {
+        "dynamic": "strict",
+        "properties": {
+          "description": {
+            "type": "text"
+          },
+          "kibanaSavedObjectMeta": {
+            "properties": {
+              "searchSourceJSON": {
+                "type": "text"
+              }
+            }
+          },
+          "savedSearchId": {
+            "type": "keyword"
+          },
+          "title": {
+            "type": "text"
+          },
+          "uiStateJSON": {
+            "type": "text"
+          },
+          "version": {
+            "type": "integer"
+          },
+          "visState": {
+            "type": "text"
+          }
+        }
+      },
+      "config": {
+        "dynamic": "true",
+        "properties": {
+          "buildNum": {
+            "type": "keyword"
+          }
+        }
+      },
+      "search": {
+        "dynamic": "strict",
+        "properties": {
+          "columns": {
+            "type": "keyword"
+          },
+          "description": {
+            "type": "text"
+          },
+          "hits": {
+            "type": "integer"
+          },
+          "kibanaSavedObjectMeta": {
+            "properties": {
+              "searchSourceJSON": {
+                "type": "text"
+              }
+            }
+          },
+          "sort": {
+            "type": "keyword"
+          },
+          "title": {
+            "type": "text"
+          },
+          "version": {
+            "type": "integer"
+          }
+        }
+      },
+      "index-pattern": {
+        "dynamic": "strict",
+        "properties": {
+          "fieldFormatMap": {
+            "type": "text"
+          },
+          "fields": {
+            "type": "text"
+          },
+          "intervalName": {
+            "type": "keyword"
+          },
+          "notExpandable": {
+            "type": "boolean"
+          },
+          "sourceFilters": {
+            "type": "text"
+          },
+          "timeFieldName": {
+            "type": "keyword"
+          },
+          "title": {
+            "type": "text"
+          }
+        }
+      },
+      "dashboard": {
+        "dynamic": "strict",
+        "properties": {
+          "description": {
+            "type": "text"
+          },
+          "hits": {
+            "type": "integer"
+          },
+          "kibanaSavedObjectMeta": {
+            "properties": {
+              "searchSourceJSON": {
+                "type": "text"
+              }
+            }
+          },
+          "optionsJSON": {
+            "type": "text"
+          },
+          "panelsJSON": {
+            "type": "text"
+          },
+          "refreshInterval": {
+            "properties": {
+              "display": {
+                "type": "keyword"
+              },
+              "pause": {
+                "type": "boolean"
+              },
+              "section": {
+                "type": "integer"
+              },
+              "value": {
+                "type": "integer"
+              }
+            }
+          },
+          "timeFrom": {
+            "type": "keyword"
+          },
+          "timeRestore": {
+            "type": "boolean"
+          },
+          "timeTo": {
+            "type": "keyword"
+          },
+          "title": {
+            "type": "text"
+          },
+          "uiStateJSON": {
+            "type": "text"
+          },
+          "version": {
+            "type": "integer"
+          }
+        }
+      },
+      "server": {
+        "dynamic": "strict",
+        "properties": {
+          "uuid": {
+            "type": "keyword"
+          }
+        }
+      }
+    }
+  }
+}
+
+{
+  "type": "index",
+  "value": {
+    "index": "foo-2",
+    "settings": {
+      "index": {
+        "number_of_shards": "5",
+        "number_of_replicas": "1"
+      }
+    },
+    "mappings": {
+      "bar": {
+        "properties": {
+          "foo": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+{
+  "type": "index",
+  "value": {
+    "index": "foo-1",
+    "settings": {
+      "index": {
+        "number_of_shards": "5",
+        "number_of_replicas": "1"
+      }
+    },
+    "mappings": {
+      "bar": {
+        "properties": {
+          "foo": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

test/api_integration/services/index.js

@@ -0,0 +1 @@
+export { SupertestProvider } from './supertest';

test/api_integration/services/supertest.js

@@ -0,0 +1,9 @@
+import { format as formatUrl } from 'url';
+
+import supertestAsPromised from 'supertest-as-promised';
+
+export function SupertestProvider({ getService }) {
+  const config = getService('config');
+  const kibanaServerUrl = formatUrl(config.get('servers.kibana'));
+  return supertestAsPromised(kibanaServerUrl);
+}

test/common/config.js

@@ -0,0 +1,22 @@
+import {
+  KibanaServerProvider,
+  EsProvider,
+  EsArchiverProvider,
+} from './services';
+
+import { esTestServerUrlParts } from '../es_test_server_url_parts';
+import { kibanaTestServerUrlParts } from '../kibana_test_server_url_parts';
+
+export default function () {
+  return {
+    servers: {
+      kibana: kibanaTestServerUrlParts,
+      elasticsearch: esTestServerUrlParts,
+    },
+    services: {
+      kibanaServer: KibanaServerProvider,
+      es: EsProvider,
+      esArchiver: EsArchiverProvider,
+    }
+  };
+}

test/common/services/index.js

@@ -0,0 +1,3 @@
+export { KibanaServerProvider } from './kibana_server';
+export { EsProvider } from './es';
+export { EsArchiverProvider } from './es_archiver';

test/es_test_server_url_parts.js

@@ -0,0 +1,10 @@
+const shield = require('./shield');
+
+export const esTestServerUrlParts = {
+  protocol: process.env.TEST_ES_PROTOCOL || 'http',
+  hostname: process.env.TEST_ES_HOSTNAME || 'localhost',
+  port: parseInt(process.env.TEST_ES_PORT, 10) || 9220,
+  auth: shield.admin.username + ':' + shield.admin.password,
+  username: shield.admin.username,
+  password: shield.admin.password,
+};

test/functional/config.js

@@ -1,5 +1,3 @@
-import { resolve } from 'path';
-
 import {
   CommonPageProvider,
   ConsolePageProvider,
@@ -20,26 +18,23 @@ import {
   FindProvider,
   RetryProvider,
   TestSubjectsProvider,
-  KibanaServerProvider,
-  EsProvider,
-  EsArchiverProvider,
   DocTableProvider,
   ScreenshotsProvider,
 } from './services';
 
-import { servers, apps } from '../server_config';
+export default async function ({ readConfigFile }) {
+  const commonConfig = await readConfigFile(require.resolve('../common/config'));
 
-export default function () {
   return {
     testFiles: [
-      resolve(__dirname, './apps/console/index.js'),
-      resolve(__dirname, './apps/context/index.js'),
-      resolve(__dirname, './apps/dashboard/index.js'),
-      resolve(__dirname, './apps/discover/index.js'),
-      resolve(__dirname, './apps/management/index.js'),
-      resolve(__dirname, './apps/status_page/index.js'),
-      resolve(__dirname, './apps/visualize/index.js'),
-      resolve(__dirname, './apps/xpack/index.js'),
+      require.resolve('./apps/console'),
+      require.resolve('./apps/context'),
+      require.resolve('./apps/dashboard'),
+      require.resolve('./apps/discover'),
+      require.resolve('./apps/management'),
+      require.resolve('./apps/status_page'),
+      require.resolve('./apps/visualize'),
+      require.resolve('./apps/xpack'),
     ],
     pageObjects: {
       common: CommonPageProvider,
@@ -56,23 +51,45 @@ export default function () {
       gettingStarted: GettingStartedPageProvider,
     },
     services: {
-      kibanaServer: KibanaServerProvider,
+      es: commonConfig.get('services.es'),
+      esArchiver: commonConfig.get('services.esArchiver'),
+      kibanaServer: commonConfig.get('services.kibanaServer'),
       remote: RemoteProvider,
       find: FindProvider,
       retry: RetryProvider,
       testSubjects: TestSubjectsProvider,
-      es: EsProvider,
-      esArchiver: EsArchiverProvider,
       docTable: DocTableProvider,
       screenshots: ScreenshotsProvider,
     },
-    servers,
-    apps,
-    esArchiver: {
-      directory: resolve(__dirname, '../../src/fixtures/es_archives'),
+    servers: commonConfig.get('servers'),
+    apps: {
+      status_page: {
+        pathname: '/status',
+      },
+      discover: {
+        pathname: '/app/kibana',
+        hash: '/discover',
+      },
+      context: {
+        pathname: '/app/kibana',
+        hash: '/context',
+      },
+      visualize: {
+        pathname: '/app/kibana',
+        hash: '/visualize',
+      },
+      dashboard: {
+        pathname: '/app/kibana',
+        hash: '/dashboards',
+      },
+      settings: {
+        pathname: '/app/kibana',
+        hash: '/management',
+      },
+      console: {
+        pathname: '/app/kibana',
+        hash: '/dev_tools/console',
+      },
     },
-    screenshots: {
-      directory: resolve(__dirname, '../screenshots'),
-    }
   };
 }

test/functional/page_objects/common_page.js

@@ -1,6 +1,6 @@
 import { delay } from 'bluebird';
 
-import getUrl from '../../utils/get_url';
+import getUrl from '../../../src/test_utils/get_url';
 
 export function CommonPageProvider({ getService, getPageObjects }) {
   const log = getService('log');
@@ -24,7 +24,7 @@ export function CommonPageProvider({ getService, getPageObjects }) {
     }
 
     /**
-     * @param {string} appName As defined in the apps objects in test/server_config.js
+     * @param {string} appName As defined in the apps config
      * @param {string} subUrl The route after the hash (#)
      */
     navigateToUrl(appName, subUrl) {

test/functional/page_objects/context_page.js

@@ -1,6 +1,6 @@
 import rison from 'rison-node';
 
-import getUrl from '../../utils/get_url';
+import getUrl from '../../../src/test_utils/get_url';
 
 const DEFAULT_INITIAL_STATE = {
   columns: ['@message'],