Moved elasticdump methods from common to support/elastic_dump.js

test/functional/apps/dashboard/_dashboard.js

@@ -4,7 +4,8 @@ import {
   dashboardPage,
   headerPage,
   scenarioManager,
-  esClient
+  esClient,
+  elasticDump
 } from '../../../support';
 
 (function () {
@@ -20,7 +21,7 @@ import {
 
         common.debug('Starting dashboard before method');
         common.debug('navigateToApp dashboard');
-        return scenarioManager.unload('emptyKibana')
+        return esClient.delete('.kibana')
         .then(function () {
           return common.try(function () {
             return esClient.updateConfigDoc({'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*'});
@@ -29,7 +30,7 @@ import {
         // and load a set of makelogs data
         .then(function loadkibana4() {
           common.debug('load kibana index with visualizations');
-          return common.elasticLoad('kibana4.json','.kibana');
+          return elasticDump.elasticLoad('dashboard','.kibana');
         })
         .then(function () {
           return scenarioManager.loadIfEmpty('logstashFunctional');
@@ -65,7 +66,7 @@ import {
 
           return addVisualizations(visualizations)
           .then(function () {
-            console.log('all done');
+            common.debug('done adding visualizations');
           });
 
         });

test/support/elastic_dump.js

@@ -0,0 +1,113 @@
+import { common, config} from './';
+
+export default (function () {
+  var util = require('util');
+  var path = require('path');
+  var url = require('url');
+  var resolve = require('path').resolve;
+  var Elasticdump = require('elasticdump').elasticdump;
+
+  function ElasticDump() {
+  }
+
+  ElasticDump.prototype = {
+    constructor: ElasticDump,
+
+    /*
+    ** This function is basically copied from
+    ** https://github.com/taskrabbit/elasticsearch-dump/blob/master/bin/elasticdump
+    ** and allows calling elasticdump for importing or exporting data from Elasticsearch
+    */
+    elasticdumpModule: function elasticdumpModule(myinput, myoutput, index, mytype) {
+      var self = this;
+
+      var options = {
+        limit:           100,
+        offset:          0,
+        debug:           false,
+        type:            mytype,
+        delete:          false,
+        all:             false,
+        maxSockets:      null,
+        input:           myinput,
+        'input-index':   null,
+        output:          myoutput,
+        'output-index':  index,
+        inputTransport:  null,
+        outputTransport: null,
+        searchBody:      null,
+        sourceOnly:      false,
+        jsonLines:       false,
+        format:          '',
+        'ignore-errors': false,
+        scrollTime:      '10m',
+        timeout:         null,
+        skip:            null,
+        toLog:           null,
+      };
+      // common.debug(options);
+      var dumper = new Elasticdump(options.input, options.output, options);
+
+      dumper.on('log',   function (message) { common.debug(message); });
+      // dumper.on('debug', function (message) { common.debug(message); });
+      dumper.on('error', function (error)   { common.debug('error', 'Error Emitted => ' + (error.message || JSON.stringify(error))); });
+
+      var promise = new Promise(function (resolve, reject) {
+        dumper.dump(function (error, totalWrites) {
+          if (error) {
+            common.debug('THERE WAS AN ERROR :-(');
+            reject(Error(error));
+          } else {
+            resolve ('elasticdumpModule success');
+          }
+        });
+      });
+      return promise;
+    },
+
+    /*
+    ** Dumps data from Elasticsearch into json files.
+    ** Takes a simple filename as input like 'dashboard' (for dashboard tests).
+    ** Appends ''.mapping.json' and '.data.json' for the actual filenames.
+    ** Writes files to the Kibana root dir.
+    ** Fails if the files already exist, so consider appending a timestamp to filename.
+    */
+    elasticDump: function elasticDump(index, file) {
+      var self = this;
+      common.debug('Dumping mapping from ' + url.format(config.servers.elasticsearch) + '/' + index
+        + ' to (' + file + '.mapping.json)');
+      return this.elasticdumpModule(url.format(config.servers.elasticsearch),
+        file + '.mapping.json', index, 'mapping')
+      .then(function () {
+        common.debug('Dumping data from ' + url.format(config.servers.elasticsearch) + '/' + index
+          + ' to (' + file + '.data.json)');
+        return self.elasticdumpModule(url.format(config.servers.elasticsearch),
+          file + '.data.json', index, 'data');
+      });
+    },
+
+    /*
+    ** Loads data from json files into Elasticsearch.
+    ** Takes a simple filename as input like 'dashboard' (for dashboard tests).
+    ** Appends ''.mapping.json' and '.data.json' for the actual filenames.
+    ** Path /test/fixtures/dump_data is hard-coded
+    */
+    elasticLoad: function elasticLoad(file, index) {
+      // TODO: should we have a flag to delete the index first?
+      // or use scenarioManager.unload(index) ? <<- currently this
+      var self = this;
+      common.debug('Loading mapping (test/fixtures/dump_data/' + file + '.mapping.json) into '
+        + url.format(config.servers.elasticsearch) + '/' + index);
+      return this.elasticdumpModule('test/fixtures/dump_data/' + file + '.mapping.json',
+        url.format(config.servers.elasticsearch), index, 'mapping')
+      .then(function () {
+        common.debug('Loading data (test/fixtures/dump_data/' + file + '.data.json) into '
+          + url.format(config.servers.elasticsearch) + '/' + index);
+        return self.elasticdumpModule('test/fixtures/dump_data/' + file + '.data.json',
+          url.format(config.servers.elasticsearch), index, 'data');
+      });
+    },
+  };
+
+  return ElasticDump;
+}());

test/support/es_client.js

@@ -1,65 +1,96 @@
-var elasticsearch = require('elasticsearch');
-var Promise = require('bluebird');
+import { common, remote} from './';
 
-function EsClient(server) {
-  if (!server) throw new Error('No server defined');
+export default (function () {
 
-  // NOTE: some large sets of test data can take several minutes to load
-  this.client = new elasticsearch.Client({
-    host: server,
-    requestTimeout: 300000,
-    defer: function () {
-      return Promise.defer();
-    }
-  });
-}
+  var elasticsearch = require('elasticsearch');
+  var Promise = require('bluebird');
 
-/**
-* Add fields to the config doc (like setting timezone and defaultIndex)
-* @return {Promise} A promise that is resolved when elasticsearch has a response
-*/
-EsClient.prototype.updateConfigDoc = function (docMap) {
-  // first we need to get the config doc's id so we can use it in our _update call
-  var self = this;
-  var configId;
-  var docMapString = JSON.stringify(docMap);
+  function EsClient(server) {
+    this.remote = remote;
+    if (!server) throw new Error('No server defined');
 
-  return this.client.search({
-    index: '.kibana',
-    type: 'config'
-  })
-  .then(function (response) {
-    if (response.errors) {
-      throw new Error(
-        'get config failed\n' +
-          response.items
-          .map(i => i[Object.keys(i)[0]].error)
-          .filter(Boolean)
-          .map(err => '  ' + JSON.stringify(err))
-          .join('\n')
-      );
-    } else {
-      configId = response.hits.hits[0]._id;
-      console.log('config._id =' + configId);
-    }
-  })
-  // now that we have the id, we can update
-  // return scenarioManager.updateConfigDoc({'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*'});
-  .then(function (response) {
-    console.log('updating config with ' + docMapString);
-    return self.client.update({
-      index: '.kibana',
-      type: 'config',
-      id: configId,
-      body: {
-        'doc':
-          docMap
+    // NOTE: some large sets of test data can take several minutes to load
+    this.client = new elasticsearch.Client({
+      host: server,
+      requestTimeout: 300000,
+      defer: function () {
+        return Promise.defer();
       }
     });
-  })
-  .catch(function (err) {
-    throw err;
-  });
-};
+  }
 
-module.exports = EsClient;
+  EsClient.prototype = {
+    constructor: EsClient,
+
+
+    /**
+    * Delete an index
+    * @param {string} index
+    * @return {Promise} A promise that is resolved when elasticsearch has a response
+    */
+    delete: function (index) {
+
+      return this.client.indices.delete({
+        index: index
+      })
+      .catch(function (reason) {
+        // if the index never existed yet, or was already deleted it's OK
+        if (reason.message.indexOf('index_not_found_exception') < 0) {
+          common.debug('reason.message: ' + reason.message);
+          throw reason;
+        }
+      });
+    },
+
+
+    /**
+    * Add fields to the config doc (like setting timezone and defaultIndex)
+    * @return {Promise} A promise that is resolved when elasticsearch has a response
+    */
+    updateConfigDoc: function (docMap) {
+      // first we need to get the config doc's id so we can use it in our _update call
+      var self = this;
+      var configId;
+      var docMapString = JSON.stringify(docMap);
+
+      return this.client.search({
+        index: '.kibana',
+        type: 'config'
+      })
+      .then(function (response) {
+        if (response.errors) {
+          throw new Error(
+            'get config failed\n' +
+              response.items
+              .map(i => i[Object.keys(i)[0]].error)
+              .filter(Boolean)
+              .map(err => '  ' + JSON.stringify(err))
+              .join('\n')
+          );
+        } else {
+          configId = response.hits.hits[0]._id;
+          common.debug('config._id =' + configId);
+        }
+      })
+      // now that we have the id, we can update
+      // return scenarioManager.updateConfigDoc({'dateFormat:tz':'UTC', 'defaultIndex':'logstash-*'});
+      .then(function (response) {
+        common.debug('updating config with ' + docMapString);
+        return self.client.update({
+          index: '.kibana',
+          type: 'config',
+          id: configId,
+          body: {
+            'doc':
+              docMap
+          }
+        });
+      })
+      .catch(function (err) {
+        throw err;
+      });
+    }
+  };
+
+  return EsClient;
+}());

test/support/index.js

@@ -1,5 +1,6 @@
 import url from 'url';
 import EsClient from './es_client';
+import ElasticDump from './elastic_dump';
 import ScenarioManager from '../fixtures/scenario_manager';
 import Common from './pages/common';
 import DiscoverPage from './pages/discover_page';
@@ -30,6 +31,7 @@ defineDelayedExport('visualizePage', () => new VisualizePage());
 defineDelayedExport('dashboardPage', () => new DashboardPage());
 defineDelayedExport('shieldPage', () => new ShieldPage());
 defineDelayedExport('consolePage', () => new ConsolePage());
+defineDelayedExport('elasticDump', () => new ElasticDump());
 
 // creates an export for values that aren't actually avaialable until
 // until tests start to run. These getters will throw errors if the export

test/support/pages/common.js

@@ -1,4 +1,4 @@
-import { common, config, defaultTryTimeout, defaultFindTimeout, remote, shieldPage } from '../';
+import { config, defaultTryTimeout, defaultFindTimeout, remote, shieldPage } from '../';
 
 export default (function () {
   var Promise = require('bluebird');