Merge pull request #8284 from elastic/jasper/backport/8266/5.0

[backport 5.0] PR #8266 to 5.0 - Update elasticsearch plugin to require ES to have the same version as Kibana.

README.md

@@ -14,13 +14,26 @@ Kibana is an open source ([Apache Licensed](https://github.com/elastic/kibana/bl
 * Run `bin/kibana` on unix, or `bin\kibana.bat` on Windows.
 * Visit [http://localhost:5601](http://localhost:5601)
 
-
 ## Upgrade from previous version
 
 * Move any custom configurations in your old kibana.yml to your new one
 * Reinstall plugins
 * Start or restart Kibana
 
+## Version compatibility with Elasticsearch
+
+Ideally, you should be running Elasticsearch and Kibana with matching version numbers (💚 in the table below). If your Elasticsearch has an older version number or a newer _major_ number than Kibana, then Kibana will fail to run (🚫). If Elasticsearch has a newer minor or patch number than Kibana, then the Kibana Server will log a warning (⚠️).
+
+| Kibana version | ES version | Outcome | Description |
+| -------------- | ---------- | ------- | ----------- |
+| 6.1.2          | 6.1.2      | 💚      | Versions are the same. |
+| 6.1.2          | 6.1.5      | ⚠️      | ES patch number is newer. |
+| 6.1.2          | 6.5.0      | ⚠️      | ES minor number is newer. |
+| 6.1.2          | 7.0.0      | 🚫      | ES major number is newer. |
+| 6.1.2          | 6.1.0      | 🚫      | ES patch number is older. |
+| 6.1.2          | 6.0.0      | 🚫      | ES minor number is older. |
+| 6.1.2          | 5.0.0      | 🚫      | ES major number is older. |
+
 ## Quick Start
 
 You're up and running! Fantastic! Kibana is now running on port 5601, so point your browser at http://YOURDOMAIN.com:5601.

src/core_plugins/elasticsearch/index.js

@@ -33,7 +33,6 @@ module.exports = function ({ Plugin }) {
           key: string()
         }).default(),
         apiVersion: Joi.string().default('master'),
-        engineVersion: Joi.string().valid('^5.0.0').default('^5.0.0')
       }).default();
     },
 

src/core_plugins/elasticsearch/lib/__tests__/check_es_version.js

@@ -8,17 +8,20 @@ import SetupError from '../setup_error';
 import serverConfig from '../../../../../test/server_config';
 import checkEsVersion from '../check_es_version';
 
-describe('plugins/elasticsearch', function () {
-  describe('lib/check_es_version', function () {
+describe('plugins/elasticsearch', () => {
+  describe('lib/check_es_version', () => {
+    const KIBANA_VERSION = '5.1.0';
+
     let server;
     let plugin;
 
     beforeEach(function () {
-      const get = sinon.stub().withArgs('elasticsearch.engineVersion').returns('^1.4.3');
-      const config = function () { return { get: get }; };
       server = {
         log: _.noop,
-        config: config,
+        // This is required or else we get a SetupError.
+        config: () => ({
+          get: sinon.stub(),
+        }),
         plugins: {
           elasticsearch: {
             client: {
@@ -44,7 +47,9 @@ describe('plugins/elasticsearch', function () {
 
         const node = {
           version: version,
-          http_address: 'http_address',
+          http: {
+            publish_address: 'http_address',
+          },
           ip: 'ip'
         };
 
@@ -54,40 +59,41 @@ describe('plugins/elasticsearch', function () {
 
       const client = server.plugins.elasticsearch.client;
       client.nodes.info = sinon.stub().returns(Promise.resolve({ nodes: nodes }));
-
     }
 
-    it('passes with single a node that matches', function () {
-      setNodes('1.4.3');
-      return checkEsVersion(server);
+    it('returns true with single a node that matches', async () => {
+      setNodes('5.1.0');
+      const result = await checkEsVersion(server, KIBANA_VERSION);
+      expect(result).to.be(true);
     });
 
-    it('passes with multiple nodes that satisfy', function () {
-      setNodes('1.4.3', '1.4.4', '1.4.3-Beta1');
-      return checkEsVersion(server);
+    it('returns true with multiple nodes that satisfy', async () => {
+      setNodes('5.1.0', '5.2.0', '5.1.1-Beta1');
+      const result = await checkEsVersion(server, KIBANA_VERSION);
+      expect(result).to.be(true);
     });
 
-    it('fails with a single node that is out of date', function () {
-      setNodes('1.4.4', '1.4.2', '1.4.5');
-
-      checkEsVersion(server)
-      .catch(function (e) {
+    it('throws an error with a single node that is out of date', async () => {
+      // 5.0.0 ES is too old to work with a 5.1.0 version of Kibana.
+      setNodes('5.1.0', '5.2.0', '5.0.0');
+      try {
+        await checkEsVersion(server, KIBANA_VERSION);
+      } catch (e) {
         expect(e).to.be.a(SetupError);
-      });
+      }
     });
 
-    it('fails if that single node is a client node', function () {
+    it('fails if that single node is a client node', async () => {
       setNodes(
-        '1.4.4',
-        { version: '1.4.2', attributes: { client: 'true' } },
-        '1.4.5'
+        '5.1.0',
+        '5.2.0',
+        { version: '5.0.0', attributes: { client: 'true' } },
       );
-
-      checkEsVersion(server)
-      .catch(function (e) {
+      try {
+        await checkEsVersion(server, KIBANA_VERSION);
+      } catch (e) {
         expect(e).to.be.a(SetupError);
-      });
+      }
     });
-
   });
 });

src/core_plugins/elasticsearch/lib/__tests__/health_check.js

@@ -6,22 +6,24 @@ import url from 'url';
 const NoConnections = require('elasticsearch').errors.NoConnections;
 
 import healthCheck from '../health_check';
+import kibanaVersion from '../kibana_version';
 import serverConfig from '../../../../../test/server_config';
 
 const esPort = serverConfig.servers.elasticsearch.port;
 const esUrl = url.format(serverConfig.servers.elasticsearch);
 
-describe('plugins/elasticsearch', function () {
-  describe('lib/health_check', function () {
-
+describe('plugins/elasticsearch', () => {
+  describe('lib/health_check', () => {
     let health;
     let plugin;
-    let server;
-    let get;
-    let set;
     let client;
 
-    beforeEach(function () {
+    beforeEach(() => {
+      const COMPATIBLE_VERSION_NUMBER = '5.0.0';
+
+      // Stub the Kibana version instead of drawing from package.json.
+      sinon.stub(kibanaVersion, 'get').returns(COMPATIBLE_VERSION_NUMBER);
+
       // setup the plugin stub
       plugin = {
         name: 'elasticsearch',
@@ -31,9 +33,7 @@ describe('plugins/elasticsearch', function () {
           yellow: sinon.stub()
         }
       };
-      // setup the config().get()/.set() stubs
-      get = sinon.stub();
-      set = sinon.stub();
+
       // set up the elasticsearch client stub
       client = {
         cluster: { health: sinon.stub() },
@@ -45,17 +45,26 @@ describe('plugins/elasticsearch', function () {
         get: sinon.stub().returns(Promise.resolve({ found: false })),
         search: sinon.stub().returns(Promise.resolve({ hits: { hits: [] } })),
       };
+
       client.nodes.info.returns(Promise.resolve({
         nodes: {
           'node-01': {
-            version: '1.5.0',
+            version: COMPATIBLE_VERSION_NUMBER,
             http_address: `inet[/127.0.0.1:${esPort}]`,
             ip: '127.0.0.1'
           }
         }
       }));
+
+      // setup the config().get()/.set() stubs
+      const get = sinon.stub();
+      get.withArgs('elasticsearch.url').returns(esUrl);
+      get.withArgs('kibana.index').returns('.my-kibana');
+
+      const set = sinon.stub();
+
       // Setup the server mock
-      server = {
+      const server = {
         log: sinon.stub(),
         info: { port: 5601 },
         config: function () { return { get, set }; },
@@ -65,9 +74,11 @@ describe('plugins/elasticsearch', function () {
       health = healthCheck(plugin, server);
     });
 
+    afterEach(() => {
+      kibanaVersion.get.restore();
+    });
+
     it('should set the cluster green if everything is ready', function () {
-      get.withArgs('elasticsearch.engineVersion').returns('^1.4.4');
-      get.withArgs('kibana.index').returns('.my-kibana');
       client.ping.returns(Promise.resolve());
       client.cluster.health.returns(Promise.resolve({ timed_out: false, status: 'green' }));
       return health.run()
@@ -83,10 +94,6 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('should set the cluster red if the ping fails, then to green', function () {
-
-      get.withArgs('elasticsearch.url').returns(esUrl);
-      get.withArgs('elasticsearch.engineVersion').returns('^1.4.4');
-      get.withArgs('kibana.index').returns('.my-kibana');
       client.ping.onCall(0).returns(Promise.reject(new NoConnections()));
       client.ping.onCall(1).returns(Promise.resolve());
       client.cluster.health.returns(Promise.resolve({ timed_out: false, status: 'green' }));
@@ -104,13 +111,9 @@ describe('plugins/elasticsearch', function () {
           sinon.assert.calledOnce(plugin.status.green);
           expect(plugin.status.green.args[0][0]).to.be('Kibana index ready');
         });
-
     });
 
     it('should set the cluster red if the health check status is red, then to green', function () {
-      get.withArgs('elasticsearch.url').returns(esUrl);
-      get.withArgs('elasticsearch.engineVersion').returns('^1.4.4');
-      get.withArgs('kibana.index').returns('.my-kibana');
       client.ping.returns(Promise.resolve());
       client.cluster.health.onCall(0).returns(Promise.resolve({ timed_out: false, status: 'red' }));
       client.cluster.health.onCall(1).returns(Promise.resolve({ timed_out: false, status: 'green' }));
@@ -131,9 +134,6 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('should set the cluster yellow if the health check timed_out and create index', function () {
-      get.withArgs('elasticsearch.url').returns(esUrl);
-      get.withArgs('elasticsearch.engineVersion').returns('^1.4.4');
-      get.withArgs('kibana.index').returns('.my-kibana');
       client.ping.returns(Promise.resolve());
       client.cluster.health.onCall(0).returns(Promise.resolve({ timed_out: true, status: 'red' }));
       client.cluster.health.onCall(1).returns(Promise.resolve({ timed_out: false, status: 'green' }));

src/core_plugins/elasticsearch/lib/__tests__/is_es_compatible_with_kibana.js

@@ -0,0 +1,40 @@
+import expect from 'expect.js';
+import sinon from 'sinon';
+
+import isEsCompatibleWithKibana from '../is_es_compatible_with_kibana';
+
+describe('plugins/elasticsearch', () => {
+  describe('lib/is_es_compatible_with_kibana', () => {
+    describe('returns false', () => {
+      it('when ES major is greater than Kibana major', () => {
+        expect(isEsCompatibleWithKibana('1.0.0', '0.0.0')).to.be(false);
+      });
+
+      it('when ES major is less than Kibana major', () => {
+        expect(isEsCompatibleWithKibana('0.0.0', '1.0.0')).to.be(false);
+      });
+
+      it('when majors are equal, but ES minor is less than Kibana minor', () => {
+        expect(isEsCompatibleWithKibana('1.0.0', '1.1.0')).to.be(false);
+      });
+
+      it('when majors and minors are equal, but ES patch is less than Kibana patch', () => {
+        expect(isEsCompatibleWithKibana('1.1.0', '1.1.1')).to.be(false);
+      });
+    });
+
+    describe('returns true', () => {
+      it('when version numbers are the same', () => {
+        expect(isEsCompatibleWithKibana('1.1.1', '1.1.1')).to.be(true);
+      });
+
+      it('when majors are equal, and ES minor is greater than Kibana minor', () => {
+        expect(isEsCompatibleWithKibana('1.1.0', '1.0.0')).to.be(true);
+      });
+
+      it('when majors and minors are equal, and ES patch is greater than Kibana patch', () => {
+        expect(isEsCompatibleWithKibana('1.1.1', '1.1.0')).to.be(true);
+      });
+    });
+  });
+});

src/core_plugins/elasticsearch/lib/__tests__/routes.js

@@ -6,29 +6,30 @@ import fromRoot from '../../../../utils/from_root';
 
 describe('plugins/elasticsearch', function () {
   describe('routes', function () {
-
     let kbnServer;
 
-    before(function () {
-      this.timeout(60000); // sometimes waiting for server takes longer than 10
+    before(async function () {
+      // Sometimes waiting for server takes longer than 10s.
+      // NOTE: This can't be a fat-arrow function because `this` needs to refer to the execution
+      // context, not to the parent context.
+      this.timeout(60000);
 
       kbnServer = kbnTestServer.createServer({
         plugins: {
           scanDirs: [
             fromRoot('src/core_plugins')
           ]
-        }
+        },
       });
-      return kbnServer.ready()
-      .then(() => kbnServer.server.plugins.elasticsearch.waitUntilReady());
-    });
 
+      await kbnServer.ready();
+      await kbnServer.server.plugins.elasticsearch.waitUntilReady();
+    });
 
     after(function () {
       return kbnServer.close();
     });
 
-
     function testRoute(options) {
       if (typeof options.payload === 'object') {
         options.payload = JSON.stringify(options.payload);
@@ -49,7 +50,6 @@ describe('plugins/elasticsearch', function () {
       });
     }
 
-
     testRoute({
       method: 'GET',
       url: '/elasticsearch/_nodes'

src/core_plugins/elasticsearch/lib/__tests__/version_satisfies.js

@@ -1,45 +0,0 @@
-import versionSatisfies from '../version_satisfies';
-import expect from 'expect.js';
-
-const versionChecks = [
-  // order is: ['actual version', 'match expression', satisfied (true/false)]
-  ['0.90.0', '>=0.90.0', true],
-  ['1.2.0', '>=1.2.1 <2.0.0', false],
-  ['1.2.1', '>=1.2.1 <2.0.0', true],
-  ['1.4.4', '>=1.2.1 <2.0.0', true],
-  ['1.7.4', '>=1.3.1 <2.0.0', true],
-  ['2.0.0', '>=1.3.1 <2.0.0', false],
-  ['1.4.3', '^1.4.3', true],
-  ['1.4.3-Beta1', '^1.4.3', true],
-  ['1.4.4', '^1.4.3', true],
-  ['1.1.12', '^1.0.0', true],
-  ['1.1.12', '~1.0.0', false],
-  ['1.6.1-SNAPSHOT', '1.6.1', true],
-  ['1.6.1-SNAPSHOT', '1.6.2', false],
-  ['1.7.1-SNAPSHOT', '^1.3.1', true],
-  ['1.3.4', '^1.4.0', false],
-  ['2.0.1', '^2.0.0', true],
-  ['2.1.1', '^2.1.0', true],
-  ['2.2.0', '^2.1.0', true],
-  ['3.0.0-SNAPSHOT', '^2.1.0', false],
-  ['3.0.0', '^2.1.0', false],
-  ['2.10.20-SNAPSHOT', '^2.10.20', true],
-  ['2.10.999', '^2.10.20', true],
-];
-
-describe('plugins/elasticsearch', function () {
-  describe('lib/version_satisfies', function () {
-    versionChecks.forEach(function (spec) {
-      const actual = spec[0];
-      const match = spec[1];
-      const satisfied = spec[2];
-      const desc = actual + ' satisfies ' + match;
-
-      describe(desc, function () {
-        it('should be ' + satisfied, function () {
-          expect(versionSatisfies(actual, match)).to.be(satisfied);
-        });
-      });
-    });
-  });
-});

src/core_plugins/elasticsearch/lib/check_es_version.js

@@ -1,31 +1,78 @@
+/**
+ * ES and Kibana versions are locked, so Kibana should require that ES has the same version as
+ * that defined in Kibana's package.json.
+ */
+
 import _ from 'lodash';
 import esBool from './es_bool';
-import versionSatisfies from './version_satisfies';
+import semver from 'semver';
+import isEsCompatibleWithKibana from './is_es_compatible_with_kibana';
 import SetupError from './setup_error';
 
-module.exports = function (server) {
+module.exports = function checkEsVersion(server, kibanaVersion) {
   server.log(['plugin', 'debug'], 'Checking Elasticsearch version');
 
   const client = server.plugins.elasticsearch.client;
-  const engineVersion = server.config().get('elasticsearch.engineVersion');
 
   return client.nodes.info()
   .then(function (info) {
-    const badNodes = _.filter(info.nodes, function (node) {
-      // remove nodes that satify required engine version
-      return !versionSatisfies(node.version, engineVersion);
+    // Aggregate incompatible ES nodes.
+    const incompatibleNodes = [];
+
+    // Aggregate ES nodes which should prompt a Kibana upgrade.
+    const warningNodes = [];
+
+    _.forEach(info.nodes, esNode => {
+      if (!isEsCompatibleWithKibana(esNode.version, kibanaVersion)) {
+        // Exit early to avoid collecting ES nodes with newer major versions in the `warningNodes`.
+        return incompatibleNodes.push(esNode);
+      }
+
+      // It's acceptable if ES is ahead of Kibana, but we want to prompt users to upgrade Kibana
+      // to match it.
+      if (semver.gt(esNode.version, kibanaVersion)) {
+        warningNodes.push(esNode);
+      }
     });
 
-    if (!badNodes.length) return true;
+    function getHumanizedNodeNames(nodes) {
+      return nodes.map(node => {
+        return 'v' + node.version + ' @ ' + node.http.publish_address + ' (' + node.ip + ')';
+      });
+    }
 
-    const badNodeNames = badNodes.map(function (node) {
-      return 'Elasticsearch v' + node.version + ' @ ' + node.http_address + ' (' + node.ip + ')';
-    });
+    if (warningNodes.length) {
+      const simplifiedNodes = warningNodes.map(node => ({
+        version: node.version,
+        http: {
+          publish_address: node.http.publish_address,
+        },
+        ip: node.ip,
+      }));
 
-    const message = `This version of Kibana requires Elasticsearch ` +
-    `${engineVersion} on all nodes. I found ` +
-    `the following incompatible nodes in your cluster: ${badNodeNames.join(',')}`;
+      server.log(['warning'], {
+        tmpl: (
+          'You\'re running Kibana <%= kibanaVersion %> with some newer versions of ' +
+          'Elasticsearch. Update Kibana to the latest version to prevent compatibility issues: ' +
+          '<%= getHumanizedNodeNames(nodes).join(", ") %>'
+        ),
+        kibanaVersion,
+        getHumanizedNodeNames,
+        nodes: simplifiedNodes,
+      });
+    }
 
-    throw new SetupError(server, message);
+    if (incompatibleNodes.length) {
+      const incompatibleNodeNames = getHumanizedNodeNames(incompatibleNodes);
+
+      const errorMessage =
+        `This version of Kibana requires Elasticsearch v` +
+        `${kibanaVersion} on all nodes. I found ` +
+        `the following incompatible nodes in your cluster: ${incompatibleNodeNames.join(',')}`;
+
+      throw new SetupError(server, errorMessage);
+    }
+
+    return true;
   });
 };

src/core_plugins/elasticsearch/lib/health_check.js

@@ -5,6 +5,8 @@ import exposeClient from './expose_client';
 import migrateConfig from './migrate_config';
 import createKibanaIndex from './create_kibana_index';
 import checkEsVersion from './check_es_version';
+import kibanaVersion from './kibana_version';
+
 const NoConnections = elasticsearch.errors.NoConnections;
 import util from 'util';
 const format = util.format;
@@ -85,7 +87,7 @@ module.exports = function (plugin, server) {
 
   function check() {
     return waitForPong()
-    .then(_.partial(checkEsVersion, server))
+    .then(() => checkEsVersion(server, kibanaVersion.get()))
     .then(waitForShards)
     .then(setGreenStatus)
     .then(_.partial(migrateConfig, server))

src/core_plugins/elasticsearch/lib/is_es_compatible_with_kibana.js

@@ -0,0 +1,38 @@
+/**
+ * Let's weed out the ES versions that won't work with a given Kibana version.
+ * 1. Major version differences will never work together.
+ * 2. Older versions of ES won't work with newer versions of Kibana.
+ */
+
+import semver from 'semver';
+
+export default function isEsCompatibleWithKibana(esVersion, kibanaVersion) {
+  const esVersionNumbers = {
+    major: semver.major(esVersion),
+    minor: semver.minor(esVersion),
+    patch: semver.patch(esVersion),
+  };
+
+  const kibanaVersionNumbers = {
+    major: semver.major(kibanaVersion),
+    minor: semver.minor(kibanaVersion),
+    patch: semver.patch(kibanaVersion),
+  };
+
+  // Reject mismatching major version numbers.
+  if (esVersionNumbers.major !== kibanaVersionNumbers.major) {
+    return false;
+  }
+
+  // Reject older minor versions of ES.
+  if (esVersionNumbers.minor < kibanaVersionNumbers.minor) {
+    return false;
+  }
+
+  // Reject older patch versions of ES.
+  if (esVersionNumbers.patch < kibanaVersionNumbers.patch) {
+    return false;
+  }
+
+  return true;
+}

src/core_plugins/elasticsearch/lib/kibana_version.js

@@ -0,0 +1,10 @@
+import {
+  version as kibanaVersion,
+} from '../../../../package.json';
+
+export default {
+  // Make the version stubbable to improve testability.
+  get() {
+    return kibanaVersion;
+  },
+};

src/core_plugins/elasticsearch/lib/version_satisfies.js

@@ -1,16 +0,0 @@
-import semver from 'semver';
-
-module.exports = function (actual, expected) {
-  try {
-    const ver = cleanVersion(actual);
-    return semver.satisfies(ver, expected);
-  } catch (err) {
-    return false;
-  }
-
-  function cleanVersion(version) {
-    const match = version.match(/\d+\.\d+\.\d+/);
-    if (!match) return version;
-    return match[0];
-  }
-};