Merge pull request #5 from spenceralger/application_boot

Application boot

src/config.js

@@ -27,6 +27,6 @@ define(function () {
      * The default ES index to use for storing Kibana specific object
      * such as stored dashboards
      */
-    kibanaIndex: 'kibana-int'
+    kibanaIndex: 'kibana4-int'
   };
 });

src/courier/courier.js

@@ -9,8 +9,9 @@ define(function (require) {
   var DocSource = require('courier/data_source/doc');
   var SearchSource = require('courier/data_source/search');
   var HastyRefresh = require('courier/errors').HastyRefresh;
+  var nextTick = require('utils/next_tick');
 
-  var Mapper = require('courier/mapper.js');
+  var Mapper = require('courier/mapper');
 
   // map constructors to type keywords
   var sourceTypes = {
@@ -31,6 +32,7 @@ define(function (require) {
         courier._refs.search,
         function (err) {
           if (err) return courier._error(err);
+          courier._activeSearchRequest = null;
         });
     },
 
@@ -38,10 +40,7 @@ define(function (require) {
     // then fetch the onces that are not
     doc: function (courier) {
       DocSource.validate(courier, courier._refs.doc, function (err, invalid) {
-        if (err) {
-          courier.stop();
-          return courier.emit('error', err);
-        }
+        if (err) return courier._error(err);
 
         // if all of the docs are up to date we don't need to do anything else
         if (invalid.length === 0) return;
@@ -56,8 +55,9 @@ define(function (require) {
   // default config values
   var defaults = {
     fetchInterval: 30000,
-    docInterval: 2500,
-    internalIndex: 'kibana4-int'
+    docInterval: 1500,
+    internalIndex: 'kibana4-int',
+    mapperCacheType: 'mappings'
   };
 
   /**
@@ -66,12 +66,13 @@ define(function (require) {
    *  search:
    *    - inherits filters, and other query properties
    *    - automatically emit results on a set interval
+   *
    *  doc:
    *    - tracks doc versions
    *    - emits same results event when the doc is updated
    *    - helps seperate versions of kibana running on the same machine stay in sync
-   *    - (NI) tracks version and uses it when new versions of a doc are reindexed
-   *    - (NI) helps deal with conflicts
+   *    - tracks version and uses it to verify that updates are safe to make
+   *    - emits conflict event when that happens
    *
    * @param {object} config
    * @param {Client} config.client - The elasticsearch.js client to use for querying. Should be
@@ -104,7 +105,10 @@ define(function (require) {
     this._onInterval = {};
 
     // make the mapper accessable
-    this._mapper = new Mapper(this);
+    this._mapper = new Mapper(this, {
+      cacheIndex: config.internalIndex,
+      cacheType: config.mapperCacheType
+    });
 
     _.each(sourceTypes, function (fn, type) {
       var courier = this;
@@ -121,7 +125,7 @@ define(function (require) {
 
       // store a quick "bound" method for triggering
       this._onInterval[type] = function () {
-        if (courier._refs[type].length) onFetch[type](courier);
+        courier.fetch(type);
         courier._schedule(type);
       };
 
@@ -151,7 +155,7 @@ define(function (require) {
 
   // is the courier currently running?
   Courier.prototype.running = function () {
-    return !!this._fetchTimer;
+    return !!_.size(this._timer);
   };
 
   // stop the courier from fetching more results
@@ -170,11 +174,18 @@ define(function (require) {
     }, this);
   };
 
-  // force a fetch of all datasources right now
-  Courier.prototype.fetch = function () {
-    _.forOwn(onFetch, function (fn, type) {
-      if (this._refs[type].length) fn(this);
-    }, this);
+  // force a fetch of all datasources right now, optionally filter by type
+  Courier.prototype.fetch = function (onlyType) {
+    var courier = this;
+    nextTick(function () {
+      _.forOwn(onFetch, function (fn, type) {
+        if (onlyType && onlyType !== type) return;
+        if (courier._refs[type].length) fn(courier);
+        courier._refs[type].forEach(function (ref) {
+          ref.fetchCount ++;
+        });
+      });
+    });
   };
 
   // data source factory
@@ -187,6 +198,7 @@ define(function (require) {
     return new Constructor(this, initialState);
   };
 
+
   /*****
    * PRIVATE API
    *****/
@@ -212,7 +224,8 @@ define(function (require) {
     var refs = this._refs[source._getType()];
     if (!_.find(refs, { source: source })) {
       refs.push({
-        source: source
+        source: source,
+        fetchCount: 0
       });
     }
   };
@@ -235,7 +248,8 @@ define(function (require) {
 
   // properly clear scheduled fetches
   Courier.prototype._clearScheduled = function (type) {
-    this._timer[type] = clearTimeout(this._timer[type]);
+    clearTimeout(this._timer[type]);
+    delete this._timer[type];
   };
 
   // alert the courior that a doc has been updated
@@ -246,18 +260,15 @@ define(function (require) {
     _.each(this._refs.doc, function (ref) {
       var state = ref.source._state;
       if (
-        state === updated
-        || (
-          state.id === updated.id
-          && state.type === updated.type
-          && state.index === updated.index
-        )
+        state.id === updated.id
+        && state.type === updated.type
+        && state.index === updated.index
       ) {
         delete ref.version;
       }
     });
 
-    onFetch.doc(this);
+    this.fetch('doc');
   };
 
   return Courier;

src/courier/data_source/data_source.js

@@ -45,6 +45,11 @@ define(function (require) {
       return courier.createSource(this._getType()).inherits(this);
     };
 
+    this.courier = function (newCourier) {
+      courier = this._courier = newCourier;
+      return this;
+    };
+
     // get/set internal state values
     this._methods.forEach(function (name) {
       this[name] = function (val) {
@@ -96,6 +101,50 @@ define(function (require) {
     return JSON.stringify(this.toJSON());
   };
 
+  /**
+   * Set the $scope for a datasource, when a datasource is bound
+   * to a scope, it's event listeners will be wrapped in a call to that
+   * scope's $apply method (safely).
+   *
+   * This also binds the DataSource to the lifetime of the scope: when the scope
+   * is destroyed, the datasource is closed
+   *
+   * @param  {AngularScope} $scope - the scope where the event emitter "occurs",
+   *   helps angular determine where to start checking for changes
+   * @return {this} - chainable
+   */
+  DataSource.prototype.$scope = function ($scope) {
+    var emitter = this;
+
+    if (emitter._emitter$scope) {
+      emitter._emitter$scope = $scope;
+      return this;
+    }
+
+    emitter._emitter$scope = $scope;
+    var origOn = emitter.on;
+
+    emitter.on = function (event, listener) {
+      var wrapped = function () {
+        var args = arguments;
+        // always use the stored ref so that it can be updated if needed
+        var $scope = emitter._emitter$scope;
+        $scope[$scope.$$phase ? '$eval' : '$apply'](function () {
+          listener.apply(emitter, args);
+        });
+      };
+      wrapped.listener = listener;
+      return origOn.call(emitter, event, wrapped);
+    };
+
+    emitter.on.restore = function () {
+      delete emitter._emitter$scope;
+      emitter.on = origOn;
+    };
+
+    return this;
+  };
+
   /*****
    * PRIVATE API
    *****/

src/courier/data_source/doc.js

@@ -1,6 +1,7 @@
 define(function (require) {
   var DataSource = require('courier/data_source/data_source');
   var inherits = require('utils/inherits');
+  var nextTick = require('utils/next_tick');
   var errors = require('courier/errors');
   var listenerCount = require('utils/event_emitter').listenerCount;
   var _ = require('lodash');
@@ -23,7 +24,7 @@ define(function (require) {
   DocSource.fetch = function (courier, refs, cb) {
     var client = courier._getClient();
     var allRefs = [];
-    var body = {
+    var getBody = {
       docs: []
     };
 
@@ -32,25 +33,30 @@ define(function (require) {
       if (source._getType() !== 'doc') return;
 
       allRefs.push(ref);
-      body.docs.push(source._flatten());
+      getBody.docs.push(source._flatten());
     });
 
-    return client.mget({ body: body }, function (err, resp) {
-      if (err) return cb(err);
+    return client.mget({ body: getBody })
+      .then(function (resp) {
+        _.each(resp.docs, function (resp, i) {
+          var ref = allRefs[i];
+          var source = ref.source;
 
-      _.each(resp.docs, function (resp, i) {
-        var ref = allRefs[i];
-        var source = ref.source;
+          if (resp.error) return source._error(new errors.DocFetchFailure(resp));
+          if (resp.found) {
+            if (ref.version === resp._version) return; // no change
+            ref.version = resp._version;
+            source._storeVersion(resp._version);
+          } else {
+            ref.version = void 0;
+            source._clearVersion();
+          }
+          source.emit('results', resp);
+        });
 
-        if (resp.error) return source._error(new errors.DocFetchFailure(resp));
-        if (ref.version === resp._version) return; // no change
-        ref.version = resp._version;
-        source._storeVersion(resp._version);
-        source.emit('results', resp);
-      });
-
-      cb(err, resp);
-    });
+        cb(void 0, resp);
+      })
+      .catch(cb);
   };
 
   /**
@@ -63,11 +69,10 @@ define(function (require) {
   DocSource.validate = function (courier, refs, cb) {
     var invalid = _.filter(refs, function (ref) {
       var storedVersion = ref.source._getVersion();
-      if (ref.version !== storedVersion) return true;
-    });
-    setTimeout(function () {
-      cb(void 0, invalid);
+      /* jshint eqeqeq: false */
+      return (!ref.fetchCount || ref.version != storedVersion);
     });
+    nextTick(cb, void 0, invalid);
   };
 
   /*****
@@ -102,6 +107,7 @@ define(function (require) {
       id: state.id,
       type: state.type,
       index: state.index,
+      version: source._getVersion(),
       body: {
         doc: fields
       }
@@ -129,7 +135,6 @@ define(function (require) {
       id: state.id,
       type: state.type,
       index: state.index,
-      version: source._getVersion(),
       body: body,
       ignore: [409]
     }, function (err, resp) {
@@ -201,8 +206,8 @@ define(function (require) {
    * @return {number} - the version number, or NaN
    */
   DocSource.prototype._getVersion = function () {
-    var id = this._versionKey();
-    return _.parseInt(localStorage.getItem(id));
+    var v = localStorage.getItem(this._versionKey());
+    return v ? _.parseInt(v) : void 0;
   };
 
   /**
@@ -212,8 +217,17 @@ define(function (require) {
    */
   DocSource.prototype._storeVersion = function (version) {
     var id = this._versionKey();
-    localStorage.setItem(id, version);
+    if (version) {
+      localStorage.setItem(id, version);
+    } else {
+      localStorage.removeItem(id);
+    }
   };
 
+  /**
+   * Clears the stored version for a DocSource
+   */
+  DocSource.prototype._clearVersion = DocSource.prototype._storeVersion;
+
   return DocSource;
 });

src/courier/errors.js

@@ -1,62 +1,101 @@
 define(function (require) {
   var listenerCount = require('utils/event_emitter').listenerCount;
+  var _ = require('lodash');
   var errors = {};
+  var inherits = require('utils/inherits');
 
-  // caused by a refresh attempting to start before the prevous is done
-  function HastyRefresh() {
-    this.name = 'HastyRefresh';
-    this.message = 'Courier attempted to start a query before the previous had finished.';
+  var canStack = (function () {
+    var err = new Error();
+    return !!err.stack;
+  }());
+
+  // abstract error class
+  function CourierError(msg, constructor) {
+    this.message = msg;
+
+    Error.call(this, this.message);
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, constructor || CourierError);
+    } else if (canStack) {
+      this.stack = (new Error()).stack;
+    } else {
+      this.stack = '';
+    }
   }
-  HastyRefresh.prototype = new Error();
-  HastyRefresh.prototype.constructor = HastyRefresh;
-  errors.HastyRefresh = HastyRefresh;
+  errors.CourierError = CourierError;
+  inherits(CourierError, Error);
 
-  // a non-critical cache write to elasticseach failed
-  function CacheWriteFailure() {
-    this.name = 'CacheWriteFailure';
-    this.message = 'A Elasticsearch cache write has failed.';
-  }
-  CacheWriteFailure.prototype = new Error();
-  CacheWriteFailure.prototype.constructor = CacheWriteFailure;
-  errors.CacheWriteFailure = CacheWriteFailure;
+  /**
+   * HastyRefresh error class
+   * @param {String} [msg] - An error message that will probably end up in a log.
+   */
+  errors.HastyRefresh = function HastyRefresh() {
+    CourierError.call(this,
+      'Courier attempted to start a query before the previous had finished.',
+      errors.HastyRefresh);
+  };
+  inherits(errors.HastyRefresh, CourierError);
 
-  // when a field mapping is requested for an unknown field
-  function FieldNotFoundInCache(name) {
-    this.name = 'FieldNotFoundInCache';
-    this.message = 'The ' + name + ' field was not found in the cached mappings';
-  }
-  FieldNotFoundInCache.prototype = new Error();
-  FieldNotFoundInCache.prototype.constructor = FieldNotFoundInCache;
-  errors.FieldNotFoundInCache = FieldNotFoundInCache;
 
-  // where there is an error getting a doc
-  function DocFetchFailure(resp) {
-    this.name = 'DocFetchFailure';
+  /**
+   * DocFetchFailure Error - where there is an error getting a doc
+   * @param {String} [msg] - An error message that will probably end up in a log.
+   */
+  errors.DocFetchFailure = function DocFetchFailure(resp) {
+    CourierError.call(this,
+      'Failed to get the doc: ' + JSON.stringify(resp),
+      errors.DocFetchFailure);
+
     this.resp = resp;
-    this.message = 'Failed to get the doc: ' + JSON.stringify(resp);
-  }
-  DocFetchFailure.prototype = new Error();
-  DocFetchFailure.prototype.constructor = DocFetchFailure;
-  errors.DocFetchFailure = DocFetchFailure;
+  };
+  inherits(errors.DocFetchFailure, CourierError);
+
+
+  /**
+   * Connection Error
+   * @param {String} [msg] - An error message that will probably end up in a log.
+   */
+  errors.VersionConflict = function VersionConflict(resp) {
+    CourierError.call(this,
+      'Failed to store document changes do to a version conflict.',
+      errors.VersionConflict);
 
-  // there was a conflict storing a doc
-  function VersionConflict(resp) {
-    this.name = 'VersionConflict';
     this.resp = resp;
-    this.message = 'Failed to store document changes due to a version conflict.';
-  }
-  VersionConflict.prototype = new Error();
-  VersionConflict.prototype.constructor = VersionConflict;
-  errors.VersionConflict = VersionConflict;
+  };
+  inherits(errors.VersionConflict, CourierError);
 
-  // there was a conflict storing a doc
-  function MappingConflict(field) {
-    this.name = 'MappingConflict';
-    this.message = 'Field ' + field + ' is defined as at least two different types in indices matching the pattern';
-  }
-  MappingConflict.prototype = new Error();
-  MappingConflict.prototype.constructor = MappingConflict;
-  errors.MappingConflict = MappingConflict;
+
+  /**
+   * there was a conflict storing a doc
+   * @param {String} field - the fields which contains the conflict
+   */
+  errors.MappingConflict = function MappingConflict(field) {
+    CourierError.call(this,
+      'Field ' + field + ' is defined as at least two different types in indices matching the pattern',
+      errors.MappingConflict);
+  };
+  inherits(errors.MappingConflict, CourierError);
+
+  /**
+   * a non-critical cache write to elasticseach failed
+   */
+  errors.CacheWriteFailure = function CacheWriteFailure() {
+    CourierError.call(this,
+      'A Elasticsearch cache write has failed.',
+      errors.CacheWriteFailure);
+  };
+  inherits(errors.CacheWriteFailure, CourierError);
+
+  /**
+   * when a field mapping is requested for an unknown field
+   * @param {String} name - the field name
+   */
+  errors.FieldNotFoundInCache = function FieldNotFoundInCache(name) {
+    CourierError.call(this,
+      'The ' + name + ' field was not found in the cached mappings',
+      errors.FieldNotFoundInCache);
+  };
+  inherits(errors.FieldNotFoundInCache, CourierError);
 
   return errors;
 });

src/courier/mapper.js

@@ -1,6 +1,7 @@
 define(function (require) {
   var _ = require('lodash');
   var Error = require('courier/errors');
+  var nextTick = require('utils/next_tick');
 
   /**
    * - Resolves index patterns
@@ -9,7 +10,7 @@ define(function (require) {
    *
    * @class Mapper
    */
-  function Mapper(courier) {
+  function Mapper(courier, config) {
 
     var client = courier._getClient();
 
@@ -23,11 +24,6 @@ define(function (require) {
     // Save a reference to this
     var self = this;
 
-    // STUB Until we have another way to get the config object.
-    var config = {
-      index: 'kibana4-int'
-    };
-
     // Store mappings we've already loaded from Elasticsearch
     var mappings = {};
 
@@ -39,7 +35,7 @@ define(function (require) {
     this.getFields = function (dataSource, callback) {
       if (self.getFieldsFromObject(dataSource)) {
         // If we already have the fields in our object, use that.
-        setTimeout(callback(undefined, self.getFieldsFromObject(dataSource)), 0);
+        nextTick(callback, void 0, self.getFieldsFromObject(dataSource));
       } else {
         // Otherwise, try to get fields from Elasticsearch cache
         self.getFieldsFromCache(dataSource, function (err, fields) {
@@ -72,7 +68,7 @@ define(function (require) {
      */
     this.getFieldMapping = function (dataSource, field, callback) {
       self.getFields(dataSource, function (err, fields) {
-        if (_.isUndefined(fields[field])) return courier._error(new Error.FieldNotFoundInCache());
+        if (_.isUndefined(fields[field])) return courier._error(new Error.FieldNotFoundInCache(field));
         callback(err, fields[field]);
       });
     };
@@ -86,7 +82,7 @@ define(function (require) {
     this.getFieldsMapping = function (dataSource, fields, callback) {
       self.getFields(dataSource, function (err, fields) {
         var _mapping = _.object(_.map(fields, function (field) {
-          if (_.isUndefined(fields[field])) return courier._error(new Error.FieldNotFoundInCache());
+          if (_.isUndefined(fields[field])) return courier._error(new Error.FieldNotFoundInCache(field));
           return [field, fields[field]];
         }));
         callback(err, _mapping);
@@ -109,8 +105,8 @@ define(function (require) {
      */
     this.getFieldsFromCache = function (dataSource, callback) {
       var params = {
-        index: config.index,
-        type: 'mapping',
+        index: config.cacheIndex,
+        type: config.cacheType,
         id: dataSource._state.index,
       };
 
@@ -161,10 +157,10 @@ define(function (require) {
      */
     var cacheFieldsToElasticsearch = function (config, index, fields, callback) {
       client.index({
-        index : config.index,
-        type: 'mapping',
-        id : index,
-        body : fields
+        index: config.cacheIndex,
+        type: config.cacheType,
+        id: index,
+        body: fields
       }, callback);
     };
 
@@ -188,9 +184,9 @@ define(function (require) {
         delete mappings[dataSource._state.index];
       }
       client.delete({
-        index : config.index,
-        type: 'mapping',
-        id : dataSource._state.index
+        index: config.cacheIndex,
+        type: config.cacheType,
+        id: dataSource._state.index
       }, callback);
     };
 

src/courier/tests/config.html

@@ -0,0 +1 @@
+<config-test></config-test>

src/courier/tests/directives.js

@@ -0,0 +1,114 @@
+define(function (require) {
+  var angular = require('angular');
+
+  angular
+    .module('kibana/directives')
+    .directive('configTest', function () {
+      return {
+        restrict: 'E',
+        template: 'My favorite number is {{favoriteNum}} <button ng-click="click()">New Favorite</button>',
+        controller: function ($scope, config) {
+          config.$bind($scope, 'favoriteNum', {
+            default: 0
+          });
+
+          $scope.click = function () {
+            $scope.favoriteNum++;
+          };
+        }
+      };
+    })
+    .directive('courierTest', function () {
+      return {
+        restrict: 'E',
+        scope: {
+          type: '@'
+        },
+        template: '<strong style="float:left">{{count}} :&nbsp;</strong><pre>{{json}}</pre>',
+        controller: function ($scope, courier) {
+          $scope.count = 0;
+          var source = courier.rootSearchSource.extend()
+            .type($scope.type)
+            .source({
+              include: 'country'
+            })
+            .$scope($scope)
+            .on('results', function (resp) {
+              $scope.count ++;
+              $scope.json = JSON.stringify(resp.hits, null, '  ');
+            });
+        }
+      };
+    })
+    .directive('courierDocTest', function () {
+      return {
+        restrict: 'E',
+        scope: {
+          id: '@',
+          type: '@',
+          index: '@'
+        },
+        template: '<strong style="float:left">{{count}} : <button ng-click="click()">reindex</button> :&nbsp;</strong><pre>{{json}}</pre>',
+        controller: function (courier, $scope) {
+          $scope.count = 0;
+          var currentSource;
+          $scope.click = function () {
+            if (currentSource) {
+              source.doIndex(currentSource);
+            }
+          };
+          var source = courier.createSource('doc')
+            .id($scope.id)
+            .type($scope.type)
+            .index($scope.index)
+            .$scope($scope)
+            .on('results', function (doc) {
+              currentSource = doc._source;
+              $scope.count ++;
+              $scope.json = JSON.stringify(doc, null, '  ');
+            });
+        }
+      };
+    })
+    .directive('mappingTest', function () {
+      return {
+        restrict: 'E',
+        scope: {
+          type: '@',
+          fields: '@'
+        },
+        template: 'Mappings:<br><div ng-repeat="(name,mapping) in mappedFields">{{name}} = {{mapping.type}}</div><hr>' +
+          '<strong style="float:left">{{count}} :&nbsp;</strong><pre>{{json}}</pre>',
+        controller: function ($rootScope, $scope, courier) {
+          $scope.count = 0;
+          $scope.mappedFields = {};
+
+          var source = courier.rootSearchSource.extend()
+            .index('logstash-*')
+            .type($scope.type)
+            .source({
+              include: 'geo'
+            })
+            .$scope($scope)
+            .on('results', function (resp) {
+              $scope.count ++;
+              $scope.json = JSON.stringify(resp.hits, null, '  ');
+            });
+
+          var fields = $scope.fields.split(',');
+
+          fields.forEach(function (field) {
+            courier._mapper.getFieldMapping(source, field, function (err, mapping) {
+              $scope.$apply(function () {
+                $scope.mappedFields[field] = mapping;
+              });
+            });
+          });
+
+          courier._mapper.getFields(source, function (err, response, status) {
+            console.log(response);
+          });
+        }
+      };
+    });
+});

src/courier/tests/mapper.html

@@ -0,0 +1 @@
+<mapping-test type="apache" fields="extension,response,request"></mapping-test>

src/index.html

@@ -12,7 +12,7 @@
     <script>require(['main'], function () {});</script>
   </head>
   <body>
-    <div ng-controller="Kibana">
+    <div ng-controller="kibana">
       <div ng-view></div>
     </div>
   </body>

src/kibana/constants/base.js

@@ -0,0 +1,15 @@
+define(function (require) {
+  var angular = require('angular');
+
+  /**
+   * broke this out so that it could be loaded before the application is
+   */
+  angular.module('kibana/constants')
+    // This stores the Kibana revision number, @REV@ is replaced by grunt.
+    .constant('kbnVersion', '@REV@')
+
+    // Use this for cache busting partials
+    .constant('cacheBust', 'cache-bust=' + Date.now())
+
+    ;
+});

src/kibana/controllers/kibana.js

@@ -1,93 +1,20 @@
 define(function (require) {
   var angular = require('angular');
   var _ = require('lodash');
+  var $ = require('jquery');
 
-  angular.module('kibana/controllers')
-    .controller('Kibana', function (courier, $scope, $rootScope) {
-      $rootScope.dataSource = courier.createSource('search')
-        .index('_all')
-        .size(5);
+  require('services/config');
+  require('services/courier');
 
-      // this should be triggered from within the controlling application
-      setTimeout(_.bindKey(courier, 'start'), 15);
-    });
+  angular
+    .module('kibana/controllers')
+    .controller('kibana', function ($scope, courier) {
+      setTimeout(function () {
+        courier.start();
+      }, 15);
 
-  angular.module('kibana/directives')
-    .directive('courierTest', function () {
-      return {
-        restrict: 'E',
-        scope: {
-          type: '@',
-          fields: '@'
-        },
-        template: 'Mappings:<br><div ng-repeat="(name,mapping) in mappedFields">{{name}} = {{mapping.type}}</div><hr>' +
-          '<strong style="float:left">{{count}} :&nbsp;</strong><pre>{{json}}</pre>',
-        controller: function ($rootScope, $scope, courier) {
-          $scope.count = 0;
-          $scope.mappedFields = {};
-
-          var source = $rootScope.dataSource.extend()
-            .index('logstash-*')
-            .type($scope.type)
-            .source({
-              include: 'geo'
-            })
-            .on('results', function (resp) {
-              $scope.count ++;
-              $scope.json = JSON.stringify(resp.hits, null, '  ');
-            });
-
-          var fields = $scope.fields.split(',');
-
-
-          _.each(fields, function (field) {
-            courier._mapper.getFieldMapping(source, field, function (err, mapping) {
-              $scope.mappedFields[field] = mapping;
-            });
-          });
-
-
-          courier._mapper.getFields(source, function (err, response, status) {
-            console.log(response);
-          });
-
-          $scope.$watch('type', source.type);
-        }
-      };
-    })
-    .directive('courierDocTest', function () {
-      return {
-        restrict: 'E',
-        scope: {
-          id: '@',
-          type: '@',
-          index: '@'
-        },
-        template: '<strong style="float:left">{{count}} : <button ng-click="click()">reindex</button> :&nbsp;</strong>' +
-          '<pre>{{json}} BEER</pre>',
-        controller: function (courier, $scope) {
-          $scope.count = 0;
-
-          console.log(courier);
-
-          var currentSource;
-          $scope.click = function () {
-            if (currentSource) {
-              source.update(currentSource);
-            }
-          };
-
-          var source = courier.createSource('doc')
-            .id($scope.id)
-            .type($scope.type)
-            .index($scope.index)
-            .on('results', function (doc) {
-              currentSource = doc._source;
-              $scope.count ++;
-              $scope.json = JSON.stringify(doc, null, '  ');
-            });
-
-        }
-      };
+      $scope.$on('$routeChangeSuccess', function () {
+        if (courier.running()) courier.fetch();
+      });
     });
 });

src/kibana/main.js

@@ -7,90 +7,72 @@ define(function (require) {
   var $ = require('jquery');
   var _ = require('lodash');
   var scopedRequire = require('require');
+  var enableAsyncModules = require('utils/async_modules');
+  var setup = require('./setup');
 
   require('elasticsearch');
   require('angular-route');
 
-  // keep a reference to each module defined before boot, so that
-  // after boot it can define new features. Also serves as a flag.
-  var preBootModules = [];
-
-  // the functions needed to register different
-  // features defined after boot
-  var registerFns = {};
+  var app = angular.module('kibana', []);
+  enableAsyncModules(app);
 
   var dependencies = [
     'elasticsearch',
+    'ngRoute',
     'kibana',
-    'ngRoute'
+    'kibana/controllers',
+    'kibana/directives',
+    'kibana/factories',
+    'kibana/services',
+    'kibana/filters',
+    'kibana/constants'
   ];
 
-  _('controllers directives factories services filters'.split(' '))
-    .map(function (type) { return 'kibana/' + type; })
-    .each(function (name) {
-      preBootModules.push(angular.module(name, []));
-      dependencies.push(name);
-    });
+  function isScope(obj) {
+    return obj && obj.$evalAsync && obj.$watch;
+  }
 
-  var app = angular.module('kibana', dependencies);
-
-  // This stores the Kibana revision number, @REV@ is replaced by grunt.
-  app.constant('kbnVersion', '@REV@');
-
-  // Use this for cache busting partials
-  app.constant('cacheBust', 'cache-bust=' + Date.now());
-
-  /**
-   * Modules that need to register components within the application after
-   * bootstrapping is complete need to pass themselves to this method.
-   *
-   * @param  {object} module - The Angular module
-   * @return {object} module
-   */
-  app.useModule = function (module) {
-    if (preBootModules) {
-      preBootModules.push(module);
-    } else {
-      _.extend(module, registerFns);
+  dependencies.forEach(function (name) {
+    if (name.indexOf('kibana/') === 0) {
+      app.useModule(angular.module(name, []));
     }
-    return module;
-  };
-
-  app.config(function ($routeProvider, $controllerProvider, $compileProvider, $filterProvider, $provide) {
-    $routeProvider
-      .when('/courier-test', {
-        templateUrl: 'courier/test.html',
-      })
-      .otherwise({
-        redirectTo: 'courier-test'
-      });
-
-    // this is how the internet told me to dynamically add modules :/
-    registerFns.controller = $controllerProvider.register;
-    registerFns.directive  = $compileProvider.directive;
-    registerFns.factory    = $provide.factory;
-    registerFns.service    = $provide.service;
-    registerFns.filter     = $filterProvider.register;
   });
 
-  // load the core components
-  require([
-    'services/courier',
-    'services/es',
-    'services/config',
-    'controllers/kibana'
-  ], function () {
+  app.requires = dependencies;
 
-    // bootstrap the app
-    $(function () {
-      angular
-        .bootstrap(document, dependencies)
-        .invoke(function ($rootScope) {
-          _.each(preBootModules, function (module) {
-            _.extend(module, registerFns);
-          });
-          preBootModules = false;
-        });
+  app.config(function ($routeProvider) {
+    $routeProvider
+      .when('/', {
+        templateUrl: 'kibana/partials/index.html'
+      })
+      .when('/config-test', {
+        templateUrl: 'courier/tests/config.html',
+      })
+      .when('/mapper-test', {
+        templateUrl: 'courier/tests/mapper.html',
+      })
+      .when('/courier-test', {
+        templateUrl: 'courier/tests/index.html',
+      })
+      .otherwise({
+        redirectTo: '/'
+      });
+  });
+
+  setup(app, function (err) {
+    if (err) throw err;
+
+    // load the elasticsearch service
+    require([
+      'courier/tests/directives',
+      'controllers/kibana',
+      'constants/base'
+    ], function () {
+      // bootstrap the app
+      $(function () {
+        angular
+          .bootstrap(document, dependencies);
+      });
     });
   });
 

src/kibana/partials/index.html

@@ -0,0 +1,5 @@
+<ul>
+  <li><a ng-href="#/courier-test">Courier Test</a></li>
+  <li><a ng-href="#/config-test">Config Test</a></li>
+  <li><a ng-href="#/mapper-test">Mapper Test</a></li>
+</ul>

src/kibana/require.config.js

@@ -16,6 +16,7 @@
   var bowerComponents = [
     'angular',
     'angular-route',
+    ['async', 'lib/async'],
     'd3',
     ['elasticsearch', 'elasticsearch.angular'],
     'jquery',

src/kibana/services/config.js

@@ -1,89 +1,149 @@
 define(function (require) {
   var angular = require('angular');
-  var configFile = require('../../config');
   var _ = require('lodash');
+  var configFile = require('../../config');
+  var nextTick = require('utils/next_tick');
+
+  require('services/courier');
 
   var module = angular.module('kibana/services');
-  module.service('config', function ($q, es, courier) {
+  // share doc and val cache between apps
+  var doc;
+  var vals = {};
 
-    var app = angular.module('kibana');
-    var config = {};
+  module.service('config', function ($q, $rootScope, courier, kbnVersion) {
     var watchers = {};
+    var unwatchers = [];
 
-    function watch(key, onChange) {
-      // probably a horrible idea
-      if (!watchers[key]) watchers[key] = [];
-      watchers[key].push(onChange);
+    if (!doc) {
+      doc = courier.createSource('doc')
+        .index(configFile.kibanaIndex)
+        .type('config')
+        .id(kbnVersion);
+    } else {
+      // clean up after previous app
+      doc
+        .removeAllListeners('results')
+        .courier(courier);
     }
 
-    function change(key, val) {
-      if (config[key] !== val) {
-        var oldVal = config[key];
-        config[key] = val;
-        if (watchers[key]) {
-          watchers[key].forEach(function (watcher) {
-            watcher(val, oldVal);
-          });
-        }
-      }
-    }
-
-    function getDoc() {
-      var defer = $q.promise();
-
-      courier.get({
-        index: config.kibanaIndex,
-        type: 'config',
-        id: app.constant('kbnVersion')
-      }, function fetchDoc(err, doc) {
-        _.assign(config, doc);
-        defer.resolve();
-      }, function onDocUpdate(doc) {
-        _.forOwn(doc, function (val, key) {
-          change(key, val);
-        });
+    doc.on('results', function (resp) {
+      if (!resp.found) return; // init should ensure it exists
+      _.forOwn(resp._source, function (val, key) {
+        if (vals[key] !== val) _change(key, val);
       });
+    });
 
+    /******
+     * PUBLIC API
+     ******/
+
+    function init() {
+      var defer = $q.defer();
+      courier.fetch();
+      doc.on('results', function completeInit(resp) {
+        // ONLY ACT IF !resp.found
+        if (!resp.found) {
+          console.log('creating empty config doc');
+          doc.doIndex({});
+          return;
+        }
+
+        console.log('fetched config doc');
+        doc.removeListener('results', completeInit);
+        defer.resolve();
+      });
       return defer.promise;
     }
 
-    return {
-      get: function (key) {
-        return config[key];
-      },
-      set: function (key, val) {
-        // sets a value in the config
-        // the es doc must be updated successfully for the update to reflect in the get api.
-
-        if (key === 'elasticsearch' || key === 'kibanaIndex') {
-          return $q.reject(new Error('These values must be updated in the config.js file.'));
-        }
+    function get(key) {
+      return vals[key];
+    }
 
+    function set(key, val) {
+      // sets a value in the config
+      // the es doc must be updated successfully for the update to reflect in the get api.
+      if (vals[key] === val) {
         var defer = $q.defer();
-
-        if (config[key] === val) {
-          defer.resolve();
-          return defer.promise;
-        }
-
-        var body = {};
-        body[key] = val;
-        courier.update({
-          index: config.kibanaIndex,
-          type: 'config',
-          id: app.constant('kbnVersion'),
-          body: body
-        }, function (err) {
-          if (err) return defer.reject(err);
-
-          change(key, val);
-          defer.resolve();
-        });
-
+        defer.resolve(true);
         return defer.promise;
-      },
-      $watch: watch,
-      init: getDoc
-    };
+      }
+
+      var update = {};
+      update[key] = val;
+
+      return doc.doUpdate(update)
+        .then(function () {
+          _change(key, val);
+          return true;
+        })
+        .catch(function (err) {
+          throw err;
+        });
+    }
+
+    function $watch(key, onChange) {
+      // probably a horrible idea
+      if (!watchers[key]) watchers[key] = [];
+      watchers[key].push(onChange);
+      _notify(onChange, vals[key]);
+    }
+
+    function $bindToScope($scope, key, opts) {
+      $watch(key, function (val) {
+        if (opts && val === void 0) val = opts['default'];
+        $scope[key] = val;
+      });
+
+      var first = true;
+      unwatchers.push($scope.$watch(key, function (newVal) {
+        if (first) return first = false;
+        set(key, newVal);
+      }));
+    }
+
+    function close() {
+      watchers = null;
+      unwatchers.forEach(function (unwatcher) {
+        unwatcher();
+      });
+    }
+
+    // expose public API on the instance
+    this.init = init;
+    this.close = close;
+    this.get = get;
+    this.set = set;
+    this.$bind = $bindToScope;
+    this.$watch = $watch;
+
+    /*******
+     * PRIVATE API
+     *******/
+
+    function _change(key, val) {
+      _notify(watchers[key], val, vals[key]);
+      vals[key] = val;
+      console.log(key, 'is now', val);
+    }
+
+    function _notify(fns, cur, prev) {
+      if ($rootScope.$$phase) {
+        // reschedule
+        nextTick(_notify, fns, cur, prev);
+        return;
+      }
+
+      var isArr = _.isArray(fns);
+      if (!fns || (isArr && !fns.length)) return;
+
+      $rootScope.$apply(function () {
+        if (!isArr) return fns(cur, prev);
+
+        fns.forEach(function (onChange) {
+          onChange(cur, prev);
+        });
+      });
+    }
   });
 });

src/kibana/services/courier.js

@@ -3,25 +3,31 @@ define(function (require) {
   var Courier = require('courier/courier');
   var DocSource = require('courier/data_source/doc');
   var errors = require('courier/errors');
+  var configFile = require('../../config');
 
   require('services/promises');
+  require('services/es');
 
+  var courier; // share the courier amoungst all of the apps
   angular.module('kibana/services')
     .service('courier', function (es, promises) {
+      if (courier) return courier;
 
       promises.playNice(DocSource.prototype, [
         'doUpdate',
         'doIndex'
       ]);
 
-      var courier = new Courier({
+      courier = new Courier({
         fetchInterval: 15000,
         client: es,
-        promises: promises
+        internalIndex: configFile.kibanaIndex
       });
 
       courier.errors = errors;
 
+      courier.rootSearchSource = courier.createSource('search');
+
       return courier;
     });
 });

src/kibana/services/es.js

@@ -1,8 +1,16 @@
 define(function (require) {
-  var angular = require('angular');
+  var configFile = require('../../config');
 
-  var module = angular.module('kibana/services');
-  module.service('es', function (esFactory) {
-    return esFactory();
-  });
+  var es; // share the client amoungst all apps
+  require('angular')
+    .module('kibana/services')
+    .service('es', function (esFactory, $q) {
+      if (es) return es;
+
+      es = esFactory({
+        host: configFile.elasticsearch
+      });
+
+      return es;
+    });
 });

src/kibana/setup.js

@@ -0,0 +1,101 @@
+define(function (require) {
+  var angular = require('angular');
+  var async = require('async');
+  var $ = require('jquery');
+  var configFile = require('../config');
+  var nextTick = require('utils/next_tick');
+
+  /**
+   * Setup the kibana application, ensuring that the kibanaIndex exists,
+   * and perform any migration of data that is required.
+   *
+   * @param {Module} app - The Kibana module
+   * @param {function} done - callback
+   */
+  return function SetupApp(app, done) {
+    // load angular deps
+    require([
+      'elasticsearch',
+      'services/es',
+      'services/config',
+      'constants/base'
+    ], function () {
+      $(function () {
+
+        var setup = angular.module('setup', [
+          'elasticsearch',
+          'kibana/services',
+          'kibana/constants'
+        ]);
+        var appEl = document.createElement('div');
+        var kibanaIndexExists;
+
+        setup.run(function (es, config) {
+          // init the setup module
+          async.series([
+            async.apply(checkForKibanaIndex, es),
+            async.apply(createKibanaIndex, es),
+            async.apply(checkForCurrentConfigDoc, es),
+            async.apply(initConfig, config)
+          ], function (err) {
+            // ready to go, remove the appEl, close services and boot be done
+            appEl.remove();
+            console.log('booting application');
+            return done(err);
+          });
+        });
+
+        angular.bootstrap(appEl, ['setup']);
+
+        function checkForKibanaIndex(es, done) {
+          console.log('look for kibana index');
+          es.indices.exists({
+            index: configFile.kibanaIndex
+          }, function (err, exists) {
+            console.log('kibana index does', (exists ? '' : 'not ') + 'exist');
+            kibanaIndexExists = exists;
+            return done(err);
+          });
+        }
+
+        // create the index if it doens't exist already
+        function createKibanaIndex(es, done) {
+          if (kibanaIndexExists) return done();
+          console.log('creating kibana index');
+          es.indices.create({
+            index: configFile.kibanaIndex,
+            body: {
+              settings: {
+                mappings: {
+                  mappings: {
+                    _source: {
+                      enabled: false
+                    },
+                    properties: {
+                      type: {
+                        type: 'string',
+                        index: 'not_analyzed'
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }, done);
+        }
+
+        // if the index is brand new, no need to see if it is out of data
+        function checkForCurrentConfigDoc(es, done) {
+          if (!kibanaIndexExists) return done();
+          console.log('checking if migration is necessary: not implemented');
+          nextTick(done);
+        }
+
+        function initConfig(config, done) {
+          console.log('initializing config service');
+          config.init().then(function () { done(); }, done);
+        }
+      });
+    });
+  };
+});

src/kibana/utils/async_modules.js

@@ -0,0 +1,58 @@
+define(function (require) {
+  var _ = require('lodash');
+
+  /* TODO: this will probably fail to work when we have multiple apps.
+   * Might need to propogate registrations to multiple providers
+   */
+  function enable(app) {
+    // keep a reference to each module defined before boot, so that
+    // after boot it can define new features. Also serves as a flag.
+    var preBootModules = [];
+
+    // the functions needed to register different
+    // features defined after boot
+    var registerFns = {};
+
+    app.config(function ($controllerProvider, $compileProvider, $filterProvider, $provide) {
+      // this is how the internet told me to dynamically add modules :/
+      registerFns = {
+        controller: $controllerProvider.register,
+        directive: $compileProvider.directive,
+        factory: $provide.factory,
+        service: $provide.service,
+        constant: $provide.constant,
+        value: $provide.value,
+        filter: $filterProvider.register
+      };
+    });
+
+    /**
+     * Modules that need to register components within the application after
+     * bootstrapping is complete need to pass themselves to this method.
+     *
+     * @param  {object} module - The Angular module
+     * @return {object} module
+     */
+    app.useModule = function (module) {
+      if (preBootModules) {
+        preBootModules.push(module);
+      } else {
+        _.extend(module, registerFns);
+      }
+      return module;
+    };
+
+    /**
+     * Called after app is bootrapped to enable asyncModules
+     * @return {[type]} [description]
+     */
+    app.run(function () {
+      _.each(preBootModules, function (module) {
+        _.extend(module, registerFns);
+      });
+      preBootModules = false;
+    });
+  }
+
+  return enable;
+});

src/kibana/utils/next_tick.js

@@ -0,0 +1,44 @@
+define(function () {
+
+  var canSetImmediate = typeof window !== 'undefined' && window.setImmediate;
+  var canPost = typeof window !== 'undefined' && window.postMessage && window.addEventListener
+  ;
+
+  if (canSetImmediate) {
+    return function (f) { return window.setImmediate(f); };
+  }
+
+  if (canPost) {
+    var queue = [];
+    window.addEventListener('message', function (ev) {
+      if (ev.source === window && ev.data === 'process-tick') {
+        ev.stopPropagation();
+        if (queue.length > 0) {
+          var fn = queue.shift();
+          if (typeof fn === 'function') {
+            fn();
+          } else {
+            // partial args were supplied
+            var args = fn;
+            fn = args.shift();
+            fn.apply(null, args);
+          }
+        }
+      }
+    }, true);
+
+    return function nextTick(fn) {
+      if (arguments.length > 1) {
+        queue.push([fn].concat([].slice.call(arguments, 1)));
+      } else {
+        queue.push(fn);
+      }
+      window.postMessage('process-tick', '*');
+    };
+  }
+
+  return function nextTick(fn) {
+    setTimeout(fn, 0);
+  };
+
+});