Merge pull request #3583 from spalger/fix/3410/tempFieldsInDiscover

Fix/3410/temp fields in discover
This commit is contained in:
Spencer 2015-05-08 13:58:05 -07:00
commit 1242a03552
11 changed files with 190 additions and 189 deletions

View file

@ -0,0 +1,20 @@
define(function (require) {
return function FieldListProvider(Private) {
var Field = Private(require('components/index_patterns/_field'));
var IndexedArray = require('utils/indexed_array/index');
var _ = require('lodash');
_(FieldList).inherits(IndexedArray);
function FieldList(indexPattern, specs) {
FieldList.Super.call(this, {
index: ['name'],
group: ['type'],
initialSet: specs.map(function (field) {
return new Field(indexPattern, field);
})
});
}
return FieldList;
};
});

View file

@ -8,11 +8,10 @@ define(function (require) {
var getIds = Private(require('components/index_patterns/_get_ids'));
var mapper = Private(require('components/index_patterns/_mapper'));
var intervals = Private(require('components/index_patterns/_intervals'));
var Field = Private(require('components/index_patterns/_field'));
var getComputedFields = require('components/index_patterns/_get_computed_fields');
var DocSource = Private(require('components/courier/data_source/doc_source'));
var mappingSetup = Private(require('utils/mapping_setup'));
var IndexedArray = require('utils/indexed_array/index');
var FieldList = Private(require('components/index_patterns/_field_list'));
var flattenHit = Private(require('components/index_patterns/_flatten_hit'));
var formatHit = require('components/index_patterns/_format_hit');
@ -107,14 +106,7 @@ define(function (require) {
};
function initFields(fields) {
fields = fields || self.fields || [];
self.fields = new IndexedArray({
index: ['name'],
group: ['type'],
initialSet: fields.map(function (field) {
return new Field(self, field);
})
});
self.fields = new FieldList(self, fields || self.fields || []);
}
self._indexFields = function () {

View file

@ -1,7 +1,5 @@
define(function (require) {
var module = require('modules').get('kibana');
var $ = require('jquery');
var _ = require('lodash');
require('filters/short_dots');
module.directive('fieldName', function ($compile, $rootScope, $filter) {
@ -12,7 +10,7 @@ define(function (require) {
'fieldName': '=',
'fieldType': '='
},
link: function ($scope, $el, attrs) {
link: function ($scope, $el) {
var typeIcon = function (fieldType) {
switch (fieldType) {
@ -41,12 +39,12 @@ define(function (require) {
'field',
'fieldName',
'fieldType',
'field.inData'
'field.rowCount'
], function () {
var type = $scope.field ? $scope.field.type : $scope.fieldType;
var name = $scope.field ? $scope.field.name : $scope.fieldName;
var results = $scope.field ? !$scope.field.inData && !$scope.field.scripted : false;
var results = $scope.field ? !$scope.field.rowCount && !$scope.field.scripted : false;
var scripted = $scope.field ? $scope.field.scripted : false;
var displayName = $filter('shortDots')(name);
@ -61,4 +59,4 @@ define(function (require) {
}
};
});
});
});

View file

@ -74,13 +74,9 @@ define(function (require) {
if (_.isUndefined(field.details) || recompute) {
// This is inherited from fieldChooser
$scope.details(field, recompute);
var fieldMapping = $scope.indexPattern.fields.byName[$scope.field.name];
detailScope.$destroy();
detailScope = $scope.$new();
detailScope.warnings = getWarnings(fieldMapping);
detailScope.warnings = getWarnings(field);
detailsElem = $(detailsHtml);
$compile(detailsElem)(detailScope);

View file

@ -23,8 +23,7 @@
<div class="sidebar-list-header">
<h5>Selected Fields</h5>
</div>
<ul
bindonce class="list-unstyled discover-selected-fields" >
<ul class="list-unstyled discover-selected-fields" >
<discover-field ng-repeat="field in fields.raw|filter:{display:true}">
</discover-field>
</ul>
@ -104,16 +103,23 @@
</div>
</div>
<ul bindonce
<ul
ng-show="(popularFields | filter:filter.isFieldFiltered).length > 0"
class="list-unstyled sidebar-well discover-popular-fields" ng-class="{ 'hidden-sm': !showFields, 'hidden-xs': !showFields }">
<li class="sidebar-item sidebar-list-header"><h6>Popular</h6></li>
<discover-field ng-repeat="field in popularFields | filter:filter.isFieldFiltered">
ng-class="{ 'hidden-sm': !showFields, 'hidden-xs': !showFields }"
class="list-unstyled sidebar-well discover-popular-fields">
<li class="sidebar-item sidebar-list-header">
<h6>Popular</h6>
</li>
<discover-field
ng-repeat="field in popularFields | filter:filter.isFieldFiltered">
</discover-field>
</ul>
<ul bindonce class="list-unstyled discover-unpopular-fields" ng-class="{ 'hidden-sm': !showFields, 'hidden-xs': !showFields }">
<discover-field ng-repeat="field in unpopularFields | filter:filter.isFieldFiltered">
<ul
ng-class="{ 'hidden-sm': !showFields, 'hidden-xs': !showFields }"
class="list-unstyled discover-unpopular-fields">
<discover-field
ng-repeat="field in unpopularFields | filter:filter.isFieldFiltered">
</discover-field>
</ul>

View file

@ -1,31 +1,30 @@
define(function (require) {
var app = require('modules').get('apps/discover');
var html = require('text!plugins/discover/components/field_chooser/field_chooser.html');
var _ = require('lodash');
var rison = require('utils/rison');
var qs = require('utils/query_string');
var fieldCalculator = require('plugins/discover/components/field_chooser/lib/field_calculator');
var IndexedArray = require('utils/indexed_array/index');
require('directives/css_truncate');
require('directives/field_name');
require('filters/unique');
require('plugins/discover/components/field_chooser/discover_field');
app.directive('discFieldChooser', function ($location, globalState, config) {
app.directive('discFieldChooser', function ($location, globalState, config, $route, Private) {
var _ = require('lodash');
var rison = require('utils/rison');
var fieldCalculator = require('plugins/discover/components/field_chooser/lib/field_calculator');
var FieldList = Private(require('components/index_patterns/_field_list'));
return {
restrict: 'E',
scope: {
columns: '=',
data: '=',
hits: '=',
fieldCounts: '=',
state: '=',
indexPattern: '=',
indexPatternList: '=',
updateFilterInQuery: '=filter'
},
template: html,
controller: function ($scope, $route) {
template: require('text!plugins/discover/components/field_chooser/field_chooser.html'),
link: function ($scope) {
$scope.setIndexPattern = function (indexPattern) {
$scope.state.index = indexPattern;
$scope.state.save();
@ -62,14 +61,14 @@ define(function (require) {
var matchFilter = (filter.vals.type == null || field.type === filter.vals.type);
var isAnalyzed = (filter.vals.analyzed == null || field.analyzed === filter.vals.analyzed);
var isIndexed = (filter.vals.indexed == null || field.indexed === filter.vals.indexed);
var rowsScritpedOrMissing = (!filter.vals.missing || field.scripted || field.inData);
var scriptedOrMissing = (!filter.vals.missing || field.scripted || field.rowCount > 0);
var matchName = (!filter.vals.name || field.name.indexOf(filter.vals.name) !== -1);
return !field.display
&& matchFilter
&& isAnalyzed
&& isIndexed
&& rowsScritpedOrMissing
&& scriptedOrMissing
&& matchName
;
},
@ -86,7 +85,7 @@ define(function (require) {
// set the initial values to the defaults
filter.reset();
$scope.$watchCollection('filter.vals', function (newFieldFilters) {
$scope.$watchCollection('filter.vals', function () {
filter.active = filter.getActive();
});
@ -95,66 +94,52 @@ define(function (require) {
_.toggleInOut($scope.columns, fieldName);
};
var calculateFields = function (newFields) {
// Find the top N most popular fields
$scope.popularFields = _(newFields)
.where(function (field) {
return field.count > 0;
$scope.$watchMulti([
'[]fieldCounts',
'[]columns',
'[]hits'
], function (cur, prev) {
var newHits = cur[2] !== prev[2];
var fields = $scope.fields;
var columns = $scope.columns || [];
var fieldCounts = $scope.fieldCounts;
if (!fields || newHits) {
$scope.fields = fields = getFields();
}
if (!fields) return;
// group the fields into popular and up-popular lists
_.chain(fields)
.each(function (field) {
field.displayOrder = _.indexOf(columns, field.name) + 1;
field.display = !!field.displayOrder;
field.rowCount = fieldCounts[field.name];
})
.sortBy('count')
.reverse()
.slice(0, config.get('fields:popularLimit'))
.sortBy('name')
.value();
.sortBy(function (field) {
return (field.count || 0) * -1;
})
.groupBy(function (field) {
if (field.display) return 'selected';
return field.count > 0 ? 'popular' : 'unpopular';
})
.tap(function (groups) {
groups.selected = _.sortBy(groups.selected || [], 'displayOrder');
// Find the top N most popular fields
$scope.unpopularFields = _.sortBy(_.sortBy(newFields, 'count')
.reverse()
.slice($scope.popularFields.length), 'name');
groups.popular = groups.popular || [];
groups.unpopular = groups.unpopular || [];
$scope.fieldTypes = _.unique(_.pluck(newFields, 'type'));
// push undefined so the user can clear the filter
$scope.fieldTypes.unshift(undefined);
};
$scope.$watch('fields', calculateFields);
$scope.$watch('indexPattern', function (indexPattern) {
$scope.fields = new IndexedArray ({
index: ['name'],
initialSet: _($scope.indexPattern.fields)
.sortBy('name')
.transform(function (fields, field) {
// clone the field with Object.create so that its getters
// and non-enumerable props are preserved
var clone = Object.create(field);
clone.display = _.contains($scope.columns, field.name);
fields.push(clone);
}, [])
.value()
// move excess popular fields to un-popular list
var extras = groups.popular.splice(config.get('fields:popularLimit'));
groups.unpopular = extras.concat(groups.unpopular);
})
.each(function (group, name) {
$scope[name + 'Fields'] = _.sortBy(group, name === 'selected' ? 'display' : 'name');
});
});
$scope.$watchCollection('columns', function (columns, oldColumns) {
_.each($scope.fields, function (field) {
field.display = _.contains(columns, field.name) ? true : false;
});
});
$scope.$watch('data', function () {
// Get all fields current in data set
var currentFields = _.chain($scope.data).map(function (d) {
return _.keys($scope.indexPattern.flattenHit(d));
}).flatten().unique().sort().value();
_.each($scope.fields, function (field) {
field.inData = _.contains(currentFields, field.name) ? true : false;
if (field.details) {
$scope.details(field, true);
}
});
// include undefined so the user can clear the filter
$scope.fieldTypes = _.union([undefined], _.pluck(fields, 'type'));
});
$scope.increaseFieldCounter = function (fieldName) {
@ -218,7 +203,7 @@ define(function (require) {
$scope.details = function (field, recompute) {
if (_.isUndefined(field.details) || recompute) {
field.details = fieldCalculator.getFieldValueCounts({
data: $scope.data,
hits: $scope.hits,
field: field,
count: 5,
grouped: false
@ -232,6 +217,37 @@ define(function (require) {
}
};
function getFields() {
var prevFields = $scope.fields;
var indexPattern = $scope.indexPattern;
var hits = $scope.hits;
var fieldCounts = $scope.fieldCounts;
if (!indexPattern || !hits || !fieldCounts) return;
var fieldSpecs = indexPattern.fields.slice(0);
var fieldNamesInDocs = _.keys(fieldCounts);
var fieldNamesInIndexPattern = _.keys(indexPattern.fields.byName);
_.difference(fieldNamesInDocs, fieldNamesInIndexPattern)
.forEach(function (unknownFieldName) {
fieldSpecs.push({
name: unknownFieldName,
type: 'unknown'
});
});
var fields = new FieldList(indexPattern, fieldSpecs);
if (prevFields) {
fields.forEach(function (field) {
field.details = _.get(prevFields, ['byName', field.name, 'details']);
});
}
return fields;
}
}
};
});

View file

@ -1,11 +1,11 @@
define(function (require) {
var _ = require('lodash');
var getFieldValues = function (data, field) {
var getFieldValues = function (hits, field) {
var name = field.name;
return _.map(data, function (row) {
return row.$$_flattened[name] == null ? row[name] : row.$$_flattened[name];
var flattenHit = field.indexPattern.flattenHit;
return _.map(hits, function (hit) {
return flattenHit(hit)[name];
});
};
@ -23,10 +23,8 @@ define(function (require) {
return { error: 'Analysis is not available for geo fields.' };
}
var allValues = getFieldValues(params.data, params.field);
var exists = 0;
var allValues = getFieldValues(params.hits, params.field);
var counts;
var missing = _countMissing(allValues);
try {
@ -37,11 +35,11 @@ define(function (require) {
return {
value: bucket.value,
count: bucket.count,
percent: (bucket.count / (params.data.length - missing) * 100).toFixed(1)
percent: (bucket.count / (params.hits.length - missing) * 100).toFixed(1)
};
});
if (params.data.length - missing === 0) {
if (params.hits.length - missing === 0) {
return {
error: 'This field is present in your elasticsearch mapping' +
' but not in any documents in the search results.' +
@ -50,8 +48,8 @@ define(function (require) {
}
return {
total: params.data.length,
exists: params.data.length - missing,
total: params.hits.length,
exists: params.hits.length - missing,
missing: missing,
buckets: counts,
};
@ -69,7 +67,6 @@ define(function (require) {
var _groupValues = function (allValues, params) {
var groups = {};
var value;
var k;
allValues.forEach(function (value) {
@ -104,4 +101,4 @@ define(function (require) {
getFieldValues: getFieldValues,
getFieldValueCounts: getFieldValueCounts
};
});
});

View file

@ -122,12 +122,10 @@ define(function (require) {
$state.index = $scope.indexPattern.id;
$state.sort = getSort.array($state.sort, $scope.indexPattern);
$scope.$watchCollection('state.columns', function (columns) {
$scope.$watchCollection('state.columns', function () {
$state.save();
});
var metaFields = config.get('metaFields');
$scope.opts = {
// number of records to fetch, then paginate through
sampleSize: config.get('discover:sampleSize'),
@ -185,7 +183,7 @@ define(function (require) {
$scope.fetch();
});
$scope.$watch('vis.aggs', function (aggs) {
$scope.$watch('vis.aggs', function () {
var buckets = $scope.vis.aggs.bySchemaGroup.buckets;
if (buckets && buckets.length === 1) {
@ -206,7 +204,7 @@ define(function (require) {
NO_RESULTS: 'none' // no results came back
};
function pick(rows, oldRows, fetchStatus, oldFetchStatus) {
function pick(rows, oldRows, fetchStatus) {
// initial state, pretend we are loading
if (rows == null && oldRows == null) return status.LOADING;
@ -297,7 +295,7 @@ define(function (require) {
$scope.hits = 0;
$scope.faliures = [];
$scope.rows = [];
$scope.rows.fieldCounts = {};
$scope.fieldCounts = {};
}
if (!$scope.rows) flushResponseData();
@ -349,7 +347,6 @@ define(function (require) {
}
var rows = $scope.rows;
var counts = rows.fieldCounts || (rows.fieldCounts = {});
var indexPattern = $scope.searchSource.get('index');
// merge the rows and the hits, use a new array to help watchers
@ -359,11 +356,12 @@ define(function (require) {
notify.event('resort rows', function () {
rows.sort(sortFn);
rows = $scope.rows = rows.slice(0, totalSize);
counts = rows.fieldCounts = {};
$scope.fieldCounts = {};
});
}
notify.event('flatten hit and count fields', function () {
var counts = $scope.fieldCounts;
$scope.rows.forEach(function (hit) {
// skip this work if we have already done it and we are NOT sorting.
// ---
@ -438,16 +436,6 @@ define(function (require) {
$window.scrollTo(0, 0);
};
// TODO: Move to utility class
var addSlashes = function (str) {
if (!_.isString(str)) return str;
str = str.replace(/\\/g, '\\\\');
str = str.replace(/\'/g, '\\\'');
str = str.replace(/\"/g, '\\"');
str = str.replace(/\0/g, '\\0');
return str;
};
var loadingVis;
var setupVisualization = function () {
// If we're not setting anything up we need to return an empty promise
@ -479,7 +467,6 @@ define(function (require) {
return Promise.resolve($scope.vis);
}
// TODO: a legit way to update the index pattern
$scope.vis = new Vis($scope.indexPattern, {
type: 'histogram',
params: {

View file

@ -64,7 +64,8 @@
<disc-field-chooser
columns="state.columns"
refresh="refreshFieldList"
data="rows"
hits="rows"
field-counts="fieldCounts"
filter="filterQuery"
index-pattern="searchSource.get('index')"
index-pattern-list="opts.indexPatternList"

View file

@ -1,8 +1,5 @@
define(function (require) {
var angular = require('angular');
var $ = require('jquery');
var _ = require('lodash');
var sinon = require('test_utils/auto_release_sinon');
var fieldCalculator = require('plugins/discover/components/field_chooser/lib/field_calculator');
// Load the kibana app dependencies.
@ -10,7 +7,7 @@ define(function (require) {
var indexPattern;
describe('fieldCalculator', function (done) {
describe('fieldCalculator', function () {
beforeEach(module('kibana'));
beforeEach(function () {
inject(function (Private) {
@ -19,10 +16,9 @@ define(function (require) {
});
it('should have a _countMissing that counts nulls & undefineds in an array', function (done) {
it('should have a _countMissing that counts nulls & undefineds in an array', function () {
var values = [['foo', 'bar'], 'foo', 'foo', undefined, ['foo', 'bar'], 'bar', 'baz', null, null, null, 'foo', undefined];
expect(fieldCalculator._countMissing(values)).to.be(5);
done();
});
describe('_groupValues', function () {
@ -33,55 +29,48 @@ define(function (require) {
groups = fieldCalculator._groupValues(values, params);
});
it('should have a _groupValues that counts values', function (done) {
it('should have a _groupValues that counts values', function () {
expect(groups).to.be.an(Object);
done();
});
it('should throw an error if any value is a plain object', function (done) {
it('should throw an error if any value is a plain object', function () {
expect(function () { fieldCalculator._groupValues([{}, true, false], params); })
.to.throwError();
done();
});
it('should have a a key for value in the array when not grouping array terms', function (done) {
it('should have a a key for value in the array when not grouping array terms', function () {
expect(_.keys(groups).length).to.be(3);
expect(groups.foo).to.be.a(Object);
expect(groups.bar).to.be.a(Object);
expect(groups.baz).to.be.a(Object);
done();
});
it('should count array terms independently', function (done) {
it('should count array terms independently', function () {
expect(groups['foo,bar']).to.be(undefined);
expect(groups.foo.count).to.be(5);
expect(groups.bar.count).to.be(3);
expect(groups.baz.count).to.be(1);
done();
});
describe('grouped array terms', function (done) {
describe('grouped array terms', function () {
beforeEach(function () {
params.grouped = true;
groups = fieldCalculator._groupValues(values, params);
});
it('should group array terms when passed params.grouped', function (done) {
it('should group array terms when passed params.grouped', function () {
expect(_.keys(groups).length).to.be(4);
expect(groups['foo,bar']).to.be.a(Object);
done();
});
it('should contain the original array as the value', function (done) {
it('should contain the original array as the value', function () {
expect(groups['foo,bar'].value).to.eql(['foo', 'bar']);
done();
});
it('should count the pairs seperately from the values they contain', function (done) {
it('should count the pairs seperately from the values they contain', function () {
expect(groups['foo,bar'].count).to.be(2);
expect(groups.foo.count).to.be(3);
expect(groups.bar.count).to.be(1);
done();
});
});
});
@ -113,7 +102,7 @@ define(function (require) {
var params;
beforeEach(function () {
params = {
data: require('fixtures/real_hits.js'),
hits: require('fixtures/real_hits.js'),
field: indexPattern.fields.byName.extension,
count: 3
};
@ -139,13 +128,13 @@ define(function (require) {
expect(fieldCalculator.getFieldValueCounts(params).error).to.not.be(undefined);
});
it('fails to analyze fields that are in the mapping, but not the data', function () {
it('fails to analyze fields that are in the mapping, but not the hits', function () {
params.field = indexPattern.fields.byName.ip;
expect(fieldCalculator.getFieldValueCounts(params).error).to.not.be(undefined);
});
it('counts the total hits', function () {
expect(fieldCalculator.getFieldValueCounts(params).total).to.be(params.data.length);
expect(fieldCalculator.getFieldValueCounts(params).total).to.be(params.hits.length);
});
it('counts the hits the field exists in', function () {
@ -154,4 +143,4 @@ define(function (require) {
});
});
});
});
});

View file

@ -42,7 +42,8 @@ define(function (require) {
'<disc-field-chooser' +
' columns="columns"' +
' toggle="toggle"' +
' data="data"' +
' hits="hits"' +
' field-counts="fieldCounts"' +
' filter="filter"' +
' index-pattern="indexPattern"' +
' index-pattern-list="indexPatternList"' +
@ -58,12 +59,17 @@ define(function (require) {
indexPatternList = [ 'b', 'a', 'c' ];
});
var flatHits = _.each(hits, indexPattern.flattenHit);
var fieldCounts = _.transform(hits, function (counts, hit) {
_(indexPattern.flattenHit(hit)).keys().each(function (key) {
counts[key] = (counts[key] || 0) + 1;
});
}, {});
init($elem, {
columns: [],
toggle: sinon.spy(),
data: flatHits,
hits: hits,
fieldCounts: fieldCounts,
filter: sinon.spy(),
indexPattern: indexPattern,
indexPatternList: indexPatternList
@ -126,11 +132,11 @@ define(function (require) {
// Re-init
destroy();
_.each(indexPattern.fields, function (field) { field.count = 0;}); // Reset the popular fields
_.each(indexPattern.fields, function (field) { field.$$spec.count = 0;}); // Reset the popular fields
init($elem, {
columns: [],
toggle: sinon.spy(),
data: require('fixtures/hits'),
hits: require('fixtures/hits'),
filter: sinon.spy(),
indexPattern: indexPattern
});
@ -164,72 +170,65 @@ define(function (require) {
describe('details processing', function () {
var field;
function getField() { return _.find($scope.fields, { name: 'bytes' }); }
beforeEach(function () {
field = indexPattern.fields.byName.bytes;
field = getField();
});
afterEach(function () {
delete field.details;
});
it('should have a details function', function (done) {
it('should have a details function', function () {
expect($scope.details).to.be.a(Function);
done();
});
it('should increase the field popularity when called', function (done) {
it('should increase the field popularity when called', function () {
indexPattern.popularizeField = sinon.spy();
$scope.details(field);
expect(indexPattern.popularizeField.called).to.be(true);
done();
});
it('should append a details object to the field', function (done) {
it('should append a details object to the field', function () {
$scope.details(field);
expect(field.details).to.not.be(undefined);
done();
});
it('should delete the field details if they already exist', function (done) {
it('should delete the field details if they already exist', function () {
$scope.details(field);
expect(field.details).to.not.be(undefined);
$scope.details(field);
expect(field.details).to.be(undefined);
done();
});
it('... unless recompute is true', function (done) {
it('... unless recompute is true', function () {
$scope.details(field);
expect(field.details).to.not.be(undefined);
$scope.details(field, true);
expect(field.details).to.not.be(undefined);
done();
});
it('should create buckets with formatted and raw values', function (done) {
it('should create buckets with formatted and raw values', function () {
$scope.details(field);
expect(field.details.buckets).to.not.be(undefined);
expect(field.details.buckets[0].value).to.be(40.141592);
expect(field.details.buckets[0].display).to.be('40.142');
done();
});
it('should recalculate the details on open fields if the data changes', function () {
$scope.details(field);
sinon.stub($scope, 'details');
$scope.data = [];
it('should recalculate the details on open fields if the hits change', function () {
$scope.hits = [
{ _source: { bytes: 1024 } }
];
$scope.$apply();
expect($scope.details.called).to.be(true);
$scope.details.restore();
// close the field, make sure details isnt called again
field = getField();
$scope.details(field);
sinon.stub($scope, 'details');
$scope.data = ['foo'];
expect(getField().details.total).to.be(1);
$scope.hits = [
{ _source: { notbytes: 1024 } }
];
$scope.$apply();
expect($scope.details.called).to.be(false);
field = getField();
expect(field.details).to.not.have.property('total');
});
});
});