Merge pull request #8548 from elastic/jasper/backport/8539/5.0

[backport] PR #8539 to 5.0 - Compatibility fixes for ES master

src/core_plugins/elasticsearch/lib/__tests__/map_uri.js

@@ -1,5 +1,6 @@
 import expect from 'expect.js';
 import mapUri from '../map_uri';
+import { get, defaults } from 'lodash';
 import sinon from 'sinon';
 
 describe('plugins/elasticsearch', function () {
@@ -7,6 +8,16 @@ describe('plugins/elasticsearch', function () {
 
     let request;
 
+    function stubServer(settings) {
+      const values = defaults(settings || {}, {
+        'elasticsearch.url': 'http://localhost:9200',
+        'elasticsearch.requestHeadersWhitelist': ['authorization'],
+        'elasticsearch.customHeaders': {}
+      });
+      const config = { get: (key, def) => get(values, key, def) };
+      return { config: () => config };
+    }
+
     beforeEach(function () {
       request = {
         path: '/elasticsearch/some/path',
@@ -23,10 +34,9 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('sends custom headers if set', function () {
-      const get = sinon.stub();
-      get.withArgs('elasticsearch.requestHeadersWhitelist').returns([]);
-      get.withArgs('elasticsearch.customHeaders').returns({ foo: 'bar' });
-      const server = { config: () => ({ get }) };
+      const server = stubServer({
+        'elasticsearch.customHeaders': { foo: 'bar' }
+      });
 
       mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
         expect(err).to.be(null);
@@ -35,10 +45,10 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('sends configured custom headers even if the same named header exists in request', function () {
-      const get = sinon.stub();
-      get.withArgs('elasticsearch.requestHeadersWhitelist').returns(['x-my-custom-header']);
-      get.withArgs('elasticsearch.customHeaders').returns({'x-my-custom-header': 'asconfigured'});
-      const server = { config: () => ({ get }) };
+      const server = stubServer({
+        'elasticsearch.requestHeadersWhitelist': ['x-my-custom-header'],
+        'elasticsearch.customHeaders': {'x-my-custom-header': 'asconfigured'}
+      });
 
       mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
         expect(err).to.be(null);
@@ -47,10 +57,9 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('only proxies the whitelisted request headers', function () {
-      const get = sinon.stub();
-      get.withArgs('elasticsearch.requestHeadersWhitelist').returns(['x-my-custom-HEADER', 'Authorization']);
-      get.withArgs('elasticsearch.customHeaders').returns({});
-      const server = { config: () => ({ get }) };
+      const server = stubServer({
+        'elasticsearch.requestHeadersWhitelist': ['x-my-custom-HEADER', 'Authorization'],
+      });
 
       mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
         expect(err).to.be(null);
@@ -61,10 +70,9 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('proxies no headers if whitelist is set to []', function () {
-      const get = sinon.stub();
-      get.withArgs('elasticsearch.requestHeadersWhitelist').returns([]);
-      get.withArgs('elasticsearch.customHeaders').returns({});
-      const server = { config: () => ({ get }) };
+      const server = stubServer({
+        'elasticsearch.requestHeadersWhitelist': [],
+      });
 
       mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
         expect(err).to.be(null);
@@ -73,10 +81,11 @@ describe('plugins/elasticsearch', function () {
     });
 
     it('proxies no headers if whitelist is set to no value', function () {
-      const get = sinon.stub();
-      get.withArgs('elasticsearch.requestHeadersWhitelist').returns([ null ]); // This is how Joi returns it
-      get.withArgs('elasticsearch.customHeaders').returns({});
-      const server = { config: () => ({ get }) };
+      const server = stubServer({
+        // joi converts `elasticsearch.requestHeadersWhitelist: null` into
+        // an array with a null inside because of the `array().single()` rule.
+        'elasticsearch.requestHeadersWhitelist': [ null ],
+      });
 
       mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
         expect(err).to.be(null);
@@ -84,5 +93,47 @@ describe('plugins/elasticsearch', function () {
       });
     });
 
+    it('strips the /elasticsearch prefix from the path', () => {
+      request.path = '/elasticsearch/es/path';
+      const server = stubServer();
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(upstreamUri).to.be('http://localhost:9200/es/path');
+      });
+    });
+
+    it('extends the es.url path', function () {
+      request.path = '/elasticsearch/index/type';
+      const server = stubServer({ 'elasticsearch.url': 'https://localhost:9200/base-path' });
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(upstreamUri).to.be('https://localhost:9200/base-path/index/type');
+      });
+    });
+
+    it('extends the es.url query string', function () {
+      request.path = '/elasticsearch/*';
+      request.query = { foo: 'bar' };
+      const server = stubServer({ 'elasticsearch.url': 'https://localhost:9200/?base=query' });
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(upstreamUri).to.be('https://localhost:9200/*?foo=bar&base=query');
+      });
+    });
+
+    it('filters the _ querystring param', function () {
+      request.path = '/elasticsearch/*';
+      request.query = { _: Date.now() };
+      const server = stubServer();
+
+      mapUri(server)(request, function (err, upstreamUri, upstreamHeaders) {
+        expect(err).to.be(null);
+        expect(upstreamUri).to.be('http://localhost:9200/*');
+      });
+    });
+
   });
 });

src/core_plugins/elasticsearch/lib/__tests__/routes.js

@@ -38,12 +38,18 @@ describe('plugins/elasticsearch', function () {
       describe(format('%s %s', options.method, options.url), function () {
         it('should should return ' + statusCode, function (done) {
           kbnTestServer.makeRequest(kbnServer, options, function (res) {
-            try {
-              expect(res.statusCode).to.be(statusCode);
+            if (res.statusCode === statusCode) {
               done();
-            } catch (e) {
-              done(e);
+              return;
             }
+
+            done(new Error(`
+              Invalid response code from elasticseach:
+                ${res.statusCode} should be ${statusCode}
+
+              Response:
+                ${res.payload}
+            `));
           });
         });
       });
@@ -93,13 +99,13 @@ describe('plugins/elasticsearch', function () {
 
     testRoute({
       method: 'POST',
-      url: '/elasticsearch/_mget?timeout=0&ignore_unavailable=true&preference=1429574531063',
+      url: '/elasticsearch/_mget',
       payload: {docs: [{_index: '.kibana', _type: 'index-pattern', _id: '[logstash-]YYYY.MM.DD'}]}
     });
 
     testRoute({
       method: 'POST',
-      url: '/elasticsearch/_msearch?timeout=0&ignore_unavailable=true&preference=1429577952339',
+      url: '/elasticsearch/_msearch',
       payload: '{"index":"logstash-2015.04.21","ignore_unavailable":true}\n{"size":500,"sort":{"@timestamp":"desc"},"query":{"bool":{"must":[{"query_string":{"analyze_wildcard":true,"query":"*"}},{"bool":{"must":[{"range":{"@timestamp":{"gte":1429577068175,"lte":1429577968175}}}],"must_not":[]}}],"must_not":[]}},"highlight":{"pre_tags":["@kibana-highlighted-field@"],"post_tags":["@/kibana-highlighted-field@"],"fields":{"*":{}}},"aggs":{"2":{"date_histogram":{"field":"@timestamp","interval":"30s","min_doc_count":0,"extended_bounds":{"min":1429577068175,"max":1429577968175}}}},"stored_fields":["*"],"_source": true,"script_fields":{},"docvalue_fields":["timestamp_offset","@timestamp","utc_time"]}\n' // eslint-disable-line max-len
     });
 

src/core_plugins/elasticsearch/lib/map_uri.js

@@ -1,22 +1,48 @@
-import querystring from 'querystring';
-import { resolve } from 'url';
+import { defaults, omit, trimLeft, trimRight } from 'lodash';
+import { parse as parseUrl, format as formatUrl, resolve } from 'url';
 import filterHeaders from './filter_headers';
 import setHeaders from './set_headers';
 
 export default function mapUri(server, prefix) {
-
   const config = server.config();
+
+  function joinPaths(pathA, pathB) {
+    return trimRight(pathA, '/') + '/' + trimLeft(pathB, '/');
+  }
+
   return function (request, done) {
-    const path = request.path.replace('/elasticsearch', '');
-    let url = config.get('elasticsearch.url');
-    if (path) {
-      if (/\/$/.test(url)) url = url.substring(0, url.length - 1);
-      url += path;
+    const {
+      protocol: esUrlProtocol,
+      slashes: esUrlHasSlashes,
+      auth: esUrlAuth,
+      hostname: esUrlHostname,
+      port: esUrlPort,
+      pathname: esUrlBasePath,
+      query: esUrlQuery
+    } = parseUrl(config.get('elasticsearch.url'), true);
+
+    // copy most url components directly from the elasticsearch.url
+    const mappedUrlComponents = {
+      protocol: esUrlProtocol,
+      slashes: esUrlHasSlashes,
+      auth: esUrlAuth,
+      hostname: esUrlHostname,
+      port: esUrlPort
+    };
+
+    // pathname
+    const reqSubPath = request.path.replace('/elasticsearch', '');
+    mappedUrlComponents.pathname = joinPaths(esUrlBasePath, reqSubPath);
+
+    // querystring
+    const mappedQuery = defaults(omit(request.query, '_'), esUrlQuery || {});
+    if (Object.keys(mappedQuery).length) {
+      mappedUrlComponents.query = mappedQuery;
     }
-    const query = querystring.stringify(request.query);
-    if (query) url += '?' + query;
+
     const filteredHeaders = filterHeaders(request.headers, config.get('elasticsearch.requestHeadersWhitelist'));
-    const customHeaders = setHeaders(filteredHeaders, config.get('elasticsearch.customHeaders'));
-    done(null, url, customHeaders);
+    const mappedHeaders = setHeaders(filteredHeaders, config.get('elasticsearch.customHeaders'));
+    const mappedUrl = formatUrl(mappedUrlComponents);
+    done(null, mappedUrl, mappedHeaders);
   };
 };

src/ui/public/courier/fetch/call_client.js

@@ -4,7 +4,7 @@ import IsRequestProvider from './is_request';
 import MergeDuplicatesRequestProvider from './merge_duplicate_requests';
 import ReqStatusProvider from './req_status';
 
-export default function CourierFetchCallClient(Private, Promise, es, esShardTimeout, sessionId) {
+export default function CourierFetchCallClient(Private, Promise, es) {
 
   const isRequest = Private(IsRequestProvider);
   const mergeDuplicateRequests = Private(MergeDuplicatesRequestProvider);
@@ -94,12 +94,7 @@ export default function CourierFetchCallClient(Private, Promise, es, esShardTime
         throw ABORTED;
       }
 
-      return (esPromise = es[strategy.clientMethod]({
-        timeout: esShardTimeout,
-        ignore_unavailable: true,
-        preference: sessionId,
-        body: body
-      }));
+      return (esPromise = es[strategy.clientMethod]({ body }));
     })
     .then(function (clientResp) {
       return strategy.getResponses(clientResp);

src/ui/public/courier/fetch/strategy/search.js

@@ -3,7 +3,7 @@ import angular from 'angular';
 
 import { toJson } from 'ui/utils/aggressive_parse';
 
-export default function FetchStrategyForSearch(Private, Promise, timefilter, kbnIndex) {
+export default function FetchStrategyForSearch(Private, Promise, timefilter, kbnIndex, sessionId) {
 
   return {
     clientMethod: 'msearch',
@@ -43,7 +43,8 @@ export default function FetchStrategyForSearch(Private, Promise, timefilter, kbn
             index: indexList,
             type: fetchParams.type,
             search_type: fetchParams.search_type,
-            ignore_unavailable: true
+            ignore_unavailable: true,
+            preference: sessionId,
           })
           + '\n'
           + toJson(body, angular.toJson);

test/functional/apps/discover/_discover.js

@@ -147,29 +147,23 @@ bdd.describe('discover app', function describeIndexTests() {
       });
     });
 
-    bdd.it('should show correct data for chart interval Daily', function () {
+    bdd.it('should show correct data for chart interval Daily', async function () {
       var chartInterval = 'Daily';
       var expectedBarChartData = [
         '133.196', '129.192', '129.724'
       ];
-      return PageObjects.discover.setChartInterval(chartInterval)
-      .then(function () {
-        return PageObjects.common.sleep(4000);
-      })
-      .then(function () {
-        return verifyChartData(expectedBarChartData);
+      await PageObjects.discover.setChartInterval(chartInterval);
+      await PageObjects.common.try(async () => {
+        await verifyChartData(expectedBarChartData);
       });
     });
 
-    bdd.it('should show correct data for chart interval Weekly', function () {
+    bdd.it('should show correct data for chart interval Weekly', async function () {
       var chartInterval = 'Weekly';
       var expectedBarChartData = [ '66.598', '129.458'];
-      return PageObjects.discover.setChartInterval(chartInterval)
-      .then(function () {
-        return PageObjects.common.sleep(2000);
-      })
-      .then(function () {
-        return verifyChartData(expectedBarChartData);
+      await PageObjects.discover.setChartInterval(chartInterval);
+      await PageObjects.common.try(async () => {
+        await verifyChartData(expectedBarChartData);
       });
     });
 

test/support/utils/try.js

@@ -14,20 +14,26 @@ class Try {
     var start = Date.now();
     var retryDelay = 502;
     var lastTry = 0;
-    var tempMessage;
+    var finalMessage;
+    var prevMessage;
 
     function attempt() {
       lastTry = Date.now();
 
       if (lastTry - start > timeout) {
-        throw new Error('timeout ' + tempMessage);
+        throw new Error('tryForTime timeout: ' + finalMessage);
       }
 
       return bluebird
       .try(block)
       .catch(function tryForTimeCatch(err) {
-        Log.debug('tryForTime failure: ' + err.message);
-        tempMessage = err.message;
+        if (err.message === prevMessage) {
+          Log.debug('--- tryForTime failed again with the same message  ...');
+        } else {
+          prevMessage = err.message;
+          Log.debug('--- tryForTime failure: ' + prevMessage);
+        }
+        finalMessage = err.stack || err.message;
         return bluebird.delay(retryDelay).then(attempt);
       });
     }