mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
added source filtering
This commit is contained in:
Stéphane Campinas
parent
420f1fa092
commit
e7204bef63
11 changed files with 317 additions and 11 deletions
|
|
@ -0,0 +1,60 @@
|
|||
var expect = require('expect.js');
|
||||
|
||||
define(function (require) {
|
||||
var isRetrieved = require('src/plugins/kibana/public/settings/sections/indices/retrieved_field');
|
||||
|
||||
describe('Settings', function () {
|
||||
describe('Indices', function () {
|
||||
describe('isRetrieved(sourceFiltering, name)', function () {
|
||||
it('should be a function', function () {
|
||||
expect(isRetrieved).to.be.a(Function);
|
||||
});
|
||||
|
||||
it('should retrieve john', function () {
|
||||
var sourceFiltering = {
|
||||
include: 'john'
|
||||
};
|
||||
|
||||
expect(isRetrieved(sourceFiltering, 'john')).to.be(true);
|
||||
});
|
||||
|
||||
it('should not retrieve connor', function () {
|
||||
var sourceFiltering = {
|
||||
exclude: 'connor'
|
||||
};
|
||||
|
||||
expect(isRetrieved(sourceFiltering, 'connor')).to.be(false);
|
||||
});
|
||||
|
||||
it('should retrieve connor', function () {
|
||||
var sourceFiltering = {
|
||||
exclude: '*.connor'
|
||||
};
|
||||
|
||||
expect(isRetrieved(sourceFiltering, 'connor')).to.be(true);
|
||||
expect(isRetrieved(sourceFiltering, 'john.connor')).to.be(false);
|
||||
});
|
||||
|
||||
it('should not retrieve neither john nor connor', function () {
|
||||
var sourceFiltering = {
|
||||
exclude: [ 'john', 'connor' ]
|
||||
};
|
||||
|
||||
expect(isRetrieved(sourceFiltering, 'connor')).to.be(false);
|
||||
expect(isRetrieved(sourceFiltering, 'john')).to.be(false);
|
||||
expect(isRetrieved(sourceFiltering, 'toto')).to.be(true);
|
||||
});
|
||||
|
||||
it('should not retrieve john.*.connor', function () {
|
||||
var sourceFiltering = {
|
||||
exclude: 'john.*.connor'
|
||||
};
|
||||
|
||||
expect(isRetrieved(sourceFiltering, 'john.j.connor')).to.be(false);
|
||||
expect(isRetrieved(sourceFiltering, 'john.t.connor')).to.be(false);
|
||||
expect(isRetrieved(sourceFiltering, 'john.j.watterson')).to.be(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -41,13 +41,14 @@
|
|||
<li class="kbn-settings-tab" ng-class="{ active: state.tab === fieldType.index }" ng-repeat="fieldType in fieldTypes">
|
||||
<a ng-click="changeTab(fieldType)">
|
||||
{{ fieldType.title }}
|
||||
<small>({{ fieldType.count }})</small>
|
||||
<small ng-if="fieldType.count !== undefined">({{ fieldType.count }})</small>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<indexed-fields ng-show="state.tab == 'indexedFields'" class="fields indexed-fields"></indexed-fields>
|
||||
<scripted-fields ng-show="state.tab == 'scriptedFields'" class="fields scripted-fields"></scripted-fields>
|
||||
<source-filtering ng-show="state.tab == 'sourceFiltering'" class="fields source-filtering"></source-filtering>
|
||||
|
||||
</div>
|
||||
</kbn-settings-indices>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import _ from 'lodash';
|
||||
import 'plugins/kibana/settings/sections/indices/_indexed_fields';
|
||||
import 'plugins/kibana/settings/sections/indices/_scripted_fields';
|
||||
import 'plugins/kibana/settings/sections/indices/_source_filtering';
|
||||
import 'plugins/kibana/settings/sections/indices/_index_header';
|
||||
import PluginsKibanaSettingsSectionsIndicesRefreshKibanaIndexProvider from 'plugins/kibana/settings/sections/indices/_refresh_kibana_index';
|
||||
import UrlProvider from 'ui/url';
|
||||
|
|
|
|||
|
|
@ -11,14 +11,21 @@ export default function GetFieldTypes() {
|
|||
scripted: 0
|
||||
});
|
||||
|
||||
return [{
|
||||
title: 'fields',
|
||||
index: 'indexedFields',
|
||||
count: fieldCount.indexed
|
||||
}, {
|
||||
title: 'scripted fields',
|
||||
index: 'scriptedFields',
|
||||
count: fieldCount.scripted
|
||||
}];
|
||||
return [
|
||||
{
|
||||
title: 'fields',
|
||||
index: 'indexedFields',
|
||||
count: fieldCount.indexed
|
||||
},
|
||||
{
|
||||
title: 'scripted fields',
|
||||
index: 'scriptedFields',
|
||||
count: fieldCount.scripted
|
||||
},
|
||||
{
|
||||
title: 'Retrieved Fields',
|
||||
index: 'sourceFiltering'
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import _ from 'lodash';
|
||||
import isRetrieved from 'plugins/kibana/settings/sections/indices/retrieved_field';
|
||||
import 'ui/paginated_table';
|
||||
import nameHtml from 'plugins/kibana/settings/sections/indices/_field_name.html';
|
||||
import typeHtml from 'plugins/kibana/settings/sections/indices/_field_type.html';
|
||||
|
|
@ -25,6 +26,7 @@ uiModules.get('apps/settings')
|
|||
{ title: 'format' },
|
||||
{ title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
|
||||
{ title: 'indexed', info: 'Fields that are not indexed are unavailable for search' },
|
||||
{ title: 'retrieved', info: 'Fields that are not retrieved as part of the _source object per hit' },
|
||||
{ title: 'controls', sortable: false }
|
||||
];
|
||||
|
||||
|
|
@ -34,6 +36,7 @@ uiModules.get('apps/settings')
|
|||
// clear and destroy row scopes
|
||||
_.invoke(rowScopes.splice(0), '$destroy');
|
||||
|
||||
const sourceFiltering = $scope.indexPattern.getSourceFiltering();
|
||||
const fields = filter($scope.indexPattern.getNonScriptedFields(), $scope.fieldFilter);
|
||||
_.find($scope.fieldTypes, {index: 'indexedFields'}).count = fields.length; // Update the tab count
|
||||
|
||||
|
|
@ -41,6 +44,7 @@ uiModules.get('apps/settings')
|
|||
const childScope = _.assign($scope.$new(), { field: field });
|
||||
rowScopes.push(childScope);
|
||||
|
||||
<<<<<<< HEAD
|
||||
return [
|
||||
{
|
||||
markup: nameHtml,
|
||||
|
|
@ -61,6 +65,9 @@ uiModules.get('apps/settings')
|
|||
markup: field.indexed ? yesTemplate : noTemplate,
|
||||
value: field.indexed
|
||||
},
|
||||
{
|
||||
markup: isRetrieved(sourceFiltering, field.displayName) ? yesTemplate : noTemplate
|
||||
},
|
||||
{
|
||||
markup: controlsHtml,
|
||||
scope: childScope
|
||||
|
|
|
|||
|
|
@ -0,0 +1,114 @@
|
|||
<h3>Retrieved fields
|
||||
|
||||
<span class="pull-right text-info hintbox-label" ng-click="showHelp = !showHelp">
|
||||
<h4><i class="fa fa-info"></i> Retrieved Fields Help</h4>
|
||||
</span>
|
||||
</h3>
|
||||
|
||||
<div class="hintbox" ng-if="showHelp">
|
||||
<h4 class="hintbox-heading">
|
||||
<i class="fa fa-question-circle text-info"></i> Retrieved Fields Help
|
||||
</h4>
|
||||
|
||||
<p>
|
||||
All fields are by default retrieved and are available inside the "_source" object of each hit. Thanks to the
|
||||
<a target="_window" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html">
|
||||
source filtering API
|
||||
<i aria-hidden="true" class="fa-link fa"></i>
|
||||
</a>
|
||||
of Elasticsearch, you can choose which fields are actually retrieved. The value needs to be a JSON object.
|
||||
</p>
|
||||
|
||||
<br/>
|
||||
<strong>Examples</strong>
|
||||
|
||||
<p style="color: #b4bcc2;">Exclusion of a single field</p>
|
||||
|
||||
<div
|
||||
readonly
|
||||
ui-ace="{
|
||||
advanced: {
|
||||
highlightActiveLine: false
|
||||
},
|
||||
useWrapMode: true,
|
||||
rendererOptions: {
|
||||
showPrintMargin: false,
|
||||
maxLines: 4294967296
|
||||
},
|
||||
mode: 'json'
|
||||
}">{
|
||||
"exclude": "user"
|
||||
}</div>
|
||||
|
||||
<p style="color: #b4bcc2;">Exclusion with a path pattern</p>
|
||||
<div
|
||||
readonly
|
||||
ui-ace="{
|
||||
advanced: {
|
||||
highlightActiveLine: false
|
||||
},
|
||||
useWrapMode: true,
|
||||
rendererOptions: {
|
||||
showPrintMargin: false,
|
||||
maxLines: 4294967296
|
||||
},
|
||||
mode: 'json'
|
||||
}">{
|
||||
"exclude": [
|
||||
"obj1.*",
|
||||
"*.value"
|
||||
]
|
||||
}</div>
|
||||
|
||||
<p style="color: #b4bcc2;">Complete control with exclusions and inclusion</p>
|
||||
<div
|
||||
readonly
|
||||
ui-ace="{
|
||||
advanced: {
|
||||
highlightActiveLine: false
|
||||
},
|
||||
useWrapMode: true,
|
||||
rendererOptions: {
|
||||
showPrintMargin: false,
|
||||
maxLines: 4294967296
|
||||
},
|
||||
mode: 'json'
|
||||
}">{
|
||||
"exclude": "obj1.*",
|
||||
"include": "obj2.*.val"
|
||||
}</div>
|
||||
</div>
|
||||
|
||||
<p>
|
||||
By default, all fields are retrieved to populate results table.
|
||||
Sometimes however some fields can be very large and seriously affect performance. This can be often the case when you have materialized one to many relationships, e.g., an entity having many nested entities. Those fields are still useful for searching or analytics but not in a result table.<br/><br/>
|
||||
Use this setting to decide what to include or exclude in the data retrieved from the Elasticsearch index.<br/>
|
||||
If empty, all fields are retrieved.
|
||||
</p>
|
||||
|
||||
<form ng-submit="save()" name="form">
|
||||
<div
|
||||
id="json-ace"
|
||||
ng-model="sourceFiltering"
|
||||
ui-ace="{
|
||||
useWrapMode: true,
|
||||
advanced: {
|
||||
highlightActiveLine: false
|
||||
},
|
||||
rendererOptions: {
|
||||
showPrintMargin: false,
|
||||
maxLines: 4294967296
|
||||
},
|
||||
mode: 'json'
|
||||
}">{{ sourceFiltering | json }}</div>
|
||||
|
||||
<div class="form-group">
|
||||
<button
|
||||
ng-disabled="form.$invalid"
|
||||
type="submit"
|
||||
aria-label="Submit"
|
||||
class="btn btn-success">
|
||||
Submit
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
define(function (require) {
|
||||
require('ui/modules').get('apps/settings')
|
||||
.directive('sourceFiltering', function (Notifier, $window) {
|
||||
var notify = new Notifier();
|
||||
return {
|
||||
restrict: 'E',
|
||||
template: require('plugins/kibana/settings/sections/indices/_source_filtering.html'),
|
||||
link: function ($scope) {
|
||||
$scope.showHelp = false;
|
||||
$scope.sourceFiltering = JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' ');
|
||||
$scope.save = function () {
|
||||
try {
|
||||
var sourceFiltering;
|
||||
|
||||
if ($scope.sourceFiltering) {
|
||||
sourceFiltering = JSON.parse($scope.sourceFiltering);
|
||||
if (sourceFiltering.constructor !== Object) {
|
||||
throw 'You must enter a JSON object with exclude/include field(s)';
|
||||
}
|
||||
for (var att in sourceFiltering) {
|
||||
if (sourceFiltering.hasOwnProperty(att) && att !== 'exclude' && att !== 'include') {
|
||||
throw 'The JSON object should have only either an exclude or an include field';
|
||||
}
|
||||
}
|
||||
$scope.indexPattern.setSourceFiltering(sourceFiltering);
|
||||
notify.info('Updated the set of retrieved fields');
|
||||
} else if ($scope.indexPattern.getSourceFiltering()) {
|
||||
var confirmIfEmpty = 'The following configuration will be deleted:\n\n' +
|
||||
JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' ');
|
||||
if ($window.confirm(confirmIfEmpty)) {
|
||||
$scope.indexPattern.setSourceFiltering(undefined);
|
||||
notify.info('All fields are now retrieved');
|
||||
} else {
|
||||
$scope.sourceFiltering = JSON.stringify($scope.indexPattern.getSourceFiltering(), null, ' ');
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
notify.error(e);
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
define(function (require) {
|
||||
/**
|
||||
* Returns true if the path matches the given pattern
|
||||
*/
|
||||
function matchPath(pathPattern, path) {
|
||||
var pattern = pathPattern.split('.');
|
||||
var pathArr = path.split('.');
|
||||
|
||||
if (pattern.length !== pathArr.length) {
|
||||
return false;
|
||||
}
|
||||
for (var i = 0; i < pattern.length; i++) {
|
||||
if (pattern[i] !== '*' && pattern[i] !== pathArr[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function process(val, name) {
|
||||
if (val.constructor === Array) {
|
||||
for (var i = 0; i < val.length; i++) {
|
||||
if (matchPath(val[i], name)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else {
|
||||
return matchPath(val, name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the field named "name" should be retrieved as part of
|
||||
* the _source object for each hit.
|
||||
*/
|
||||
return function isRetrieved(sourceFiltering, name) {
|
||||
if (sourceFiltering === undefined) {
|
||||
return true;
|
||||
}
|
||||
if (sourceFiltering.include) {
|
||||
var inc = sourceFiltering.include;
|
||||
return process(inc, name);
|
||||
} else if (sourceFiltering.exclude) {
|
||||
var exc = sourceFiltering.exclude;
|
||||
return !process(exc, name);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
});
|
||||
|
|
@ -72,6 +72,10 @@ describe('docTable', function () {
|
|||
expect($elem.text()).to.not.be.empty();
|
||||
});
|
||||
|
||||
it('should set the source filtering defintion', function () {
|
||||
expect($scope.indexPattern.getSourceFiltering.called).to.be(true);
|
||||
});
|
||||
|
||||
it('should set the indexPattern to that of the searchSource', function () {
|
||||
expect($scope.indexPattern).to.be(searchSource.get('index'));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -80,6 +80,10 @@ uiModules.get('kibana')
|
|||
|
||||
$scope.searchSource.size(config.get('discover:sampleSize'));
|
||||
$scope.searchSource.sort(getSort($scope.sorting, $scope.indexPattern));
|
||||
const sourceFiltering = $scope.indexPattern.getSourceFiltering();
|
||||
if (sourceFiltering) {
|
||||
$scope.searchSource.source(sourceFiltering);
|
||||
}
|
||||
|
||||
// Set the watcher after initialization
|
||||
$scope.$watchCollection('sorting', function (newSort, oldSort) {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
<<<<<<< HEAD
|
||||
import _ from 'lodash';
|
||||
import errors from 'ui/errors';
|
||||
import angular from 'angular';
|
||||
|
|
@ -35,6 +36,7 @@ export default function IndexPatternFactory(Private, timefilter, Notifier, confi
|
|||
timeFieldName: 'string',
|
||||
notExpandable: 'boolean',
|
||||
intervalName: 'string',
|
||||
sourceFiltering: 'json',
|
||||
fields: 'json',
|
||||
fieldFormatMap: {
|
||||
type: 'string',
|
||||
|
|
@ -119,6 +121,17 @@ export default function IndexPatternFactory(Private, timefilter, Notifier, confi
|
|||
});
|
||||
};
|
||||
|
||||
// Set the source filtering configuration for that index
|
||||
self.setSourceFiltering = function (config) {
|
||||
self.sourceFiltering = config;
|
||||
self.save();
|
||||
};
|
||||
|
||||
// Get the source filtering configuration for that index
|
||||
self.getSourceFiltering = function () {
|
||||
return self.sourceFiltering;
|
||||
};
|
||||
|
||||
function initFields(fields) {
|
||||
self.fields = new FieldList(self, fields || self.fields || []);
|
||||
}
|
||||
|
|
@ -328,7 +341,8 @@ export default function IndexPatternFactory(Private, timefilter, Notifier, confi
|
|||
edit: '/settings/indices/{{id}}',
|
||||
addField: '/settings/indices/{{id}}/create-field',
|
||||
indexedFields: '/settings/indices/{{id}}?_a=(tab:indexedFields)',
|
||||
scriptedFields: '/settings/indices/{{id}}?_a=(tab:scriptedFields)'
|
||||
scriptedFields: '/settings/indices/{{id}}?_a=(tab:scriptedFields)',
|
||||
sourceFiltering: '/settings/indices/{{id}}?_a=(tab:sourceFiltering)'
|
||||
};
|
||||
|
||||
return IndexPattern;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue