github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3577 from spalger/ignorePrivateFields

Browse source 
Tweak hit flattening
This commit is contained in:
Spencer 2015-05-01 12:22:33 -07:00
commit 424309cdb2
7 changed files with 159 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

75
src/kibana/components/index_patterns/_flatten_hit.js
View file

@ -1,46 +1,61 @@
// Takes a hit, merges it with any stored/scripted fields, and with the metaFields
// returns a flattened version
define(function (require) {
var _ = require('lodash');
return function FlattenHitProvider(config, $rootScope) {
function flattenHit(indexPattern, hit) {
var flat = {};
var _ = require('lodash');
// recursively merge _source
var fields = indexPattern.fields.byName;
(function flatten(obj, keyPrefix) {
keyPrefix = keyPrefix ? keyPrefix + '.' : '';
_.forOwn(obj, function (val, key) {
key = keyPrefix + key;
var metaFields = config.get('metaFields');
$rootScope.$on('change:config.metaFields', function () {
metaFields = config.get('metaFields');
});
if (flat[key] !== void 0) return;
function flattenHit(indexPattern, hit) {
var flat = {};
var hasValidMapping = (fields[key] && fields[key].type !== 'conflict');
var isValue = !_.isPlainObject(val);
// recursively merge _source
var fields = indexPattern.fields.byName;
(function flatten(obj, keyPrefix) {
keyPrefix = keyPrefix ? keyPrefix + '.' : '';
_.forOwn(obj, function (val, key) {
key = keyPrefix + key;
if (hasValidMapping || isValue) {
flat[key] = val;
return;
}
if (flat[key] !== void 0) return;
flatten(val, key);
var hasValidMapping = (fields[key] && fields[key].type !== 'conflict');
var isValue = !_.isPlainObject(val);
if (hasValidMapping || isValue) {
flat[key] = val;
return;
}
flatten(val, key);
});
}(hit._source));
// assign the meta fields
_.each(metaFields, function (meta) {
if (meta === '_source') return;
flat[meta] = hit[meta];
});
}(hit._source));
// assign the meta fields
_.each(indexPattern.metaFields, function (meta) {
flat[meta] = hit[meta];
});
// unwrap computed fields
_.forOwn(hit.fields, function (val, key) {
if (key[0] === '_' && !_.contains(metaFields, key)) return;
flat[key] = _.isArray(val) && val.length === 1 ? val[0] : val;
});
// unwrap computed fields
_.forOwn(hit.fields, function (val, key) {
flat[key] = val[0];
});
return flat;
}
return flat;
}
function cachedFlatten(indexPattern, hit) {
return hit.$$_flattened || (hit.$$_flattened = flattenHit(indexPattern, hit));
}
return function cachedFlatten(indexPattern, hit) {
return hit.$$_flattened || (hit.$$_flattened = flattenHit(indexPattern, hit));
cachedFlatten.uncached = flattenHit;
return cachedFlatten;
};
});

11
src/kibana/components/index_patterns/_index_pattern.js
View file

@ -1,5 +1,5 @@
define(function (require) {
return function IndexPatternFactory(Private, timefilter, configFile, Notifier, shortDotsFilter, config, Promise) {
return function IndexPatternFactory(Private, timefilter, Notifier, config, Promise) {
var _ = require('lodash');
var angular = require('angular');
var errors = require('errors');
@ -9,8 +9,9 @@ define(function (require) {
var fieldFormats = Private(require('components/index_patterns/_field_formats'));
var intervals = Private(require('components/index_patterns/_intervals'));
var fieldTypes = Private(require('components/index_patterns/_field_types'));
var flattenHit = require('components/index_patterns/_flatten_hit');
var flattenHit = Private(require('components/index_patterns/_flatten_hit'));
var getComputedFields = require('components/index_patterns/_get_computed_fields');
var shortDotsFilter = Private(require('filters/short_dots'));
var DocSource = Private(require('components/courier/data_source/doc_source'));
@ -42,7 +43,7 @@ define(function (require) {
self.init = function () {
// tell the docSource where to find the doc
docSource
.index(configFile.kibana_index)
.index(config.file.kibana_index)
.type(type)
.id(self.id);
@ -274,12 +275,10 @@ define(function (require) {
return '' + self.toJSON();
};
self.metaFields = config.get('metaFields');
self.flattenHit = _.partial(flattenHit, self);
self.getComputedFields = getComputedFields.bind(self);
}
return IndexPattern;
};
});

4
src/kibana/components/index_patterns/_mapper.js
View file

@ -1,5 +1,5 @@
define(function (require) {
return function MapperService(Private, Promise, es, configFile, config) {
return function MapperService(Private, Promise, es, config) {
var _ = require('lodash');
var moment = require('moment');
@ -33,7 +33,7 @@ define(function (require) {
if (!skipIndexPatternCache) {
return es.get({
index: configFile.kibana_index,
index: config.file.kibana_index,
type: 'index-pattern',
id: id,
_sourceInclude: ['fields']

23
src/kibana/filters/short_dots.js
View file

@ -3,14 +3,21 @@
// 'foo.bar.baz'.replace(/(.+?\.)/g,function(v) {return v[0]+'.';});
define(function (require) {
var _ = require('lodash');
require('modules')
.get('kibana')
.filter('shortDots', function (config) {
return function (str) {
if (!_.isString(str) || config.get('shortDots:enable') !== true) {
return str;
}
return str.replace(/(.+?\.)/g, function (v) { return v[0] + '.'; });
};
.filter('shortDots', function (Private) {
return Private(shortDotsFilterProvider);
});
});
function shortDotsFilterProvider(config) {
return function (str) {
if (!_.isString(str) || config.get('shortDots:enable') !== true) {
return str;
}
return str.replace(/(.+?\.)/g, function (v) { return v[0] + '.'; });
};
}
return shortDotsFilterProvider;
});

3
test/unit/fixtures/stubbed_logstash_index_pattern.js
View file

@ -3,7 +3,7 @@ define(function (require) {
var StubIndexPattern = Private(require('test_utils/stub_index_pattern'));
var fieldTypes = Private(require('components/index_patterns/_field_types'));
var mockLogstashFields = Private(require('fixtures/logstash_fields'));
var flattenHit = require('components/index_patterns/_flatten_hit');
var flattenHit = Private(require('components/index_patterns/_flatten_hit'));
var getComputedFields = require('components/index_patterns/_get_computed_fields');
var _ = require('lodash');
@ -21,7 +21,6 @@ define(function (require) {
indexPattern.getComputedFields = _.bind(getComputedFields, indexPattern);
indexPattern.flattenHit = _.partial(flattenHit, indexPattern);
indexPattern.metaFields = ['_id', '_type', '_source'];
indexPattern.id = 'logstash-*';
return indexPattern;

136
test/unit/specs/index_patterns/flatten_hit.js
View file

@ -1,56 +1,67 @@
define(function (require) {
var _ = require('lodash');
var flattenHit = require('components/index_patterns/_flatten_hit');
describe('IndexPattern#flattenHit()', function () {
var indexPattern = {
fields: {
byName: {
'message': { type: 'string' },
'geo.coordinates': { type: 'geo_point' },
'geo.dest': { type: 'string' },
'geo.src': { type: 'string' },
'bytes': { type: 'number' },
'@timestamp': { type: 'date' },
'team': { type: 'nested' },
'team.name': { type: 'string' },
'team.role': { type: 'string' },
'user': { type: 'conflict' },
'user.name': { type: 'string' },
'user.id': { type: 'conflict' },
'delta': { type: 'number', scripted: true }
var _ = require('lodash');
var flattenHit;
var indexPattern;
var config;
var hit;
var flat;
beforeEach(module('kibana'));
beforeEach(inject(function (Private, $injector) {
flattenHit = Private(require('components/index_patterns/_flatten_hit')).uncached;
config = $injector.get('config');
indexPattern = {
fields: {
byName: {
'message': { type: 'string' },
'geo.coordinates': { type: 'geo_point' },
'geo.dest': { type: 'string' },
'geo.src': { type: 'string' },
'bytes': { type: 'number' },
'@timestamp': { type: 'date' },
'team': { type: 'nested' },
'team.name': { type: 'string' },
'team.role': { type: 'string' },
'user': { type: 'conflict' },
'user.name': { type: 'string' },
'user.id': { type: 'conflict' },
'delta': { type: 'number', scripted: true }
}
}
}
};
};
var hit = {
_source: {
message: 'Hello World',
geo: {
coordinates: { lat: 33.4500, lon: 112.0667 },
dest: 'US',
src: 'IN'
hit = {
_source: {
message: 'Hello World',
geo: {
coordinates: { lat: 33.4500, lon: 112.0667 },
dest: 'US',
src: 'IN'
},
bytes: 10039103,
'@timestamp': (new Date()).toString(),
tags: [{ text: 'foo' }, { text: 'bar' }],
groups: ['loners'],
noMapping: true,
team: [
{ name: 'foo', role: 'leader' },
{ name: 'bar', role: 'follower' },
{ name: 'baz', role: 'party boy' },
],
user: { name: 'smith', id: 123 }
},
bytes: 10039103,
'@timestamp': (new Date()).toString(),
tags: [{ text: 'foo' }, { text: 'bar' }],
groups: ['loners'],
noMapping: true,
team: [
{ name: 'foo', role: 'leader' },
{ name: 'bar', role: 'follower' },
{ name: 'baz', role: 'party boy' },
],
user: { name: 'smith', id: 123 }
},
fields: {
delta: [42],
random: [0.12345]
}
};
fields: {
delta: [42],
random: [0.12345]
}
};
var flat = flattenHit(indexPattern, hit);
flat = flattenHit(indexPattern, hit);
}));
it('flattens keys as far down as the mapping goes', function () {
expect(flat).to.have.property('geo.coordinates', hit._source.geo.coordinates);
@ -94,5 +105,38 @@ define(function (require) {
it('assumes that all fields are "computed fields"', function () {
expect(flat).to.have.property('random', 0.12345);
});
it('ignores fields that start with an _ and are not in the metaFields', function () {
config.set('metaFields', ['_metaKey']);
hit.fields._notMetaKey = [100];
flat = flattenHit(indexPattern, hit);
expect(flat).to.not.have.property('_notMetaKey');
});
it('includes underscore-prefixed keys that are in the metaFields', function () {
config.set('metaFields', ['_metaKey']);
hit.fields._metaKey = [100];
flat = flattenHit(indexPattern, hit);
expect(flat).to.have.property('_metaKey', 100);
});
it('adapts to changes in the metaFields', function () {
hit.fields._metaKey = [100];
config.set('metaFields', ['_metaKey']);
flat = flattenHit(indexPattern, hit);
expect(flat).to.have.property('_metaKey', 100);
config.set('metaFields', []);
flat = flattenHit(indexPattern, hit);
expect(flat).to.not.have.property('_metaKey');
});
it('handles fields that are not arrays, like _timestamp', function () {
hit.fields._metaKey = 20000;
config.set('metaFields', ['_metaKey']);
flat = flattenHit(indexPattern, hit);
expect(flat).to.have.property('_metaKey', 20000);
});
});
});

3
test/utils/stub_index_pattern.js
View file

@ -4,7 +4,7 @@ define(function (require) {
var sinon = require('sinon/sinon');
var IndexedArray = require('utils/indexed_array/index');
var fieldFormats = Private(require('components/index_patterns/_field_formats'));
var flattenHit = require('components/index_patterns/_flatten_hit');
var flattenHit = Private(require('components/index_patterns/_flatten_hit'));
var getComputedFields = require('components/index_patterns/_get_computed_fields');
@ -32,7 +32,6 @@ define(function (require) {
this.toIndexList = _.constant([pattern]);
this.getComputedFields = getComputedFields;
this.flattenHit = _.partial(flattenHit, this);
this.metaFields = ['_id', '_type', '_source'];
}
return StubIndexPattern;