github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[indexPatterns/field] remove indexed/analyzed/doc_values awareness (#11969)

Browse source 
* [indexPatterns/field] replace mapping awareness with readFromDocValues

* move link to more details to proper location

* Address kjbekkelund's feedback

* let the prs speak for themselves
This commit is contained in:
Spencer 2017-05-25 12:58:23 -07:00 • committed by GitHub
parent aa3fa06f61
commit 84ea502205
16 changed files with 111 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/core_plugins/kibana/public/discover/components/field_chooser/discover_field.js
View file

@ -34,23 +34,6 @@ app.directive('discoverField', function ($compile) {
const getWarnings = function (field) {
let warnings = [];
if (!field.scripted) {
if (!field.doc_values && field.type !== 'boolean' && !(field.analyzed && field.type === 'string')) {
warnings.push('Doc values are not enabled on this field. This may lead to excess heap consumption when visualizing.');
}
if (field.analyzed && field.type === 'string') {
warnings.push('This is an analyzed string field.' +
' Analyzed strings are highly unique and can use a lot of memory to visualize.' +
' Values such as foo-bar will be broken into foo and bar.');
}
if (!field.indexed && !field.searchable) {
warnings.push('This field is not indexed and might not be usable in visualizations.');
}
}
if (field.scripted) {
warnings.push('Scripted fields can take a long time to execute.');
}

8
src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.html
View file

@ -62,21 +62,21 @@
<form role="form" class="discover-field-details">
<div class="form-group">
<label>
Analyzed
Aggregatable
</label>
<select
ng-options="opt.value as opt.label for opt in filter.boolOpts"
ng-model="filter.vals.analyzed"
ng-model="filter.vals.aggregatable"
class="form-control">
</select>
</div>
<div class="form-group">
<label>
Indexed
Searchable
</label>
<select
ng-options="opt.value as opt.label for opt in filter.boolOpts"
ng-model="filter.vals.indexed"
ng-model="filter.vals.searchable"
class="form-control">
</select>
</div>

19
src/core_plugins/kibana/public/discover/components/field_chooser/field_chooser.js
View file

@ -45,13 +45,14 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
const filter = $scope.filter = {
props: [
'type',
'indexed',
'analyzed',
'aggregatable',
'searchable',
'missing',
'name'
],
defaults: {
missing: true
missing: true,
type: 'any'
},
boolOpts: [
{ label: 'any', value: undefined },
@ -69,16 +70,16 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
return field.display;
},
isFieldFiltered: function (field) {
const matchFilter = (filter.vals.type == null || field.type === filter.vals.type);
const isAnalyzed = (filter.vals.analyzed == null || field.analyzed === filter.vals.analyzed);
const isIndexed = (filter.vals.indexed == null || field.indexed === filter.vals.indexed);
const matchFilter = (filter.vals.type === 'any' || field.type === filter.vals.type);
const isAggregatable = (filter.vals.aggregatable == null || field.aggregatable === filter.vals.aggregatable);
const isSearchable = (filter.vals.searchable == null || field.searchable === filter.vals.searchable);
const scriptedOrMissing = (!filter.vals.missing || field.scripted || field.rowCount > 0);
const matchName = (!filter.vals.name || field.name.indexOf(filter.vals.name) !== -1);
return !field.display
&& matchFilter
&& isAnalyzed
&& isIndexed
&& isAggregatable
&& isSearchable
&& scriptedOrMissing
&& matchName
;
@ -146,7 +147,7 @@ app.directive('discFieldChooser', function ($location, globalState, config, $rou
.commit();
// include undefined so the user can clear the filter
$scope.fieldTypes = _.union([undefined], _.pluck(fields, 'type'));
$scope.fieldTypes = _.union(['any'], _.pluck(fields, 'type'));
});
$scope.increaseFieldCounter = function (fieldName) {

5
src/core_plugins/kibana/public/management/sections/indices/edit_index_pattern/indexed_fields_table/indexed_fields_table.js
View file

@ -27,7 +27,6 @@ uiModules.get('apps/management')
{ title: 'format' },
{ title: 'searchable', info: 'These fields can be used in the filter bar' },
{ title: 'aggregatable' , info: 'These fields can be used in visualization aggregations' },
{ title: 'analyzed', info: 'Analyzed fields may require extra memory to visualize' },
{ title: 'excluded', info: 'Fields that are excluded from _source when it is fetched' },
{ title: 'controls', sortable: false }
];
@ -77,10 +76,6 @@ uiModules.get('apps/management')
markup: field.aggregatable ? yesTemplate : noTemplate,
value: field.aggregatable
},
{
markup: field.analyzed ? yesTemplate : noTemplate,
value: field.analyzed
},
{
markup: excluded ? yesTemplate : noTemplate,
value: excluded

8
src/core_plugins/kibana/server/routes/api/ingest/register_field_capabilities.js
View file

@ -1,5 +1,7 @@
import _ from 'lodash';
import handleESError from '../../../lib/handle_es_error';
import { shouldReadFieldFromDocValues } from './should_read_field_from_doc_values';
export function registerFieldCapabilities(server) {
server.route({
@ -19,7 +21,11 @@ export function registerFieldCapabilities(server) {
(res) => {
const fields = _.get(res, 'indices._all.fields', {});
const fieldsFilteredValues = _.mapValues(fields, (value) => {
return _.pick(value, ['searchable', 'aggregatable']);
return {
searchable: value.searchable,
aggregatable: value.aggregatable,
readFromDocValues: shouldReadFieldFromDocValues(value.aggregatable, value.type)
};
});
const retVal = { fields: fieldsFilteredValues };

4
src/core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values.js Normal file
View file

@ -0,0 +1,4 @@
// see https://github.com/elastic/kibana/issues/11141 for details
export function shouldReadFieldFromDocValues(aggregatable, esType) {
return aggregatable && esType !== 'text' && !esType.startsWith('_');
}

70
src/fixtures/logstash_fields.js
View file

@ -1,52 +1,48 @@
import { castEsToKbnFieldTypeName } from '../utils';
import { shouldReadFieldFromDocValues } from '../core_plugins/kibana/server/routes/api/ingest/should_read_field_from_doc_values';
function stubbedLogstashFields() {
return [
// |indexed
// | |analyzed
// | | |aggregatable
// | | | |searchable
// name esType | | | | |metadata
['bytes', 'long', true, true, true, true, { count: 10, docValues: true } ],
['ssl', 'boolean', true, true, true, true, { count: 20 } ],
['@timestamp', 'date', true, true, true, true, { count: 30 } ],
['time', 'date', true, true, true, true, { count: 30 } ],
['@tags', 'keyword', true, true, true, true ],
['utc_time', 'date', true, true, true, true ],
['phpmemory', 'integer', true, true, true, true ],
['ip', 'ip', true, true, true, true ],
['request_body', 'attachment', true, true, true, true ],
['point', 'geo_point', true, true, true, true ],
['area', 'geo_shape', true, true, true, true ],
['hashed', 'murmur3', true, true, false, true ],
['geo.coordinates', 'geo_point', true, true, true, true ],
['extension', 'keyword', true, true, true, true ],
['machine.os', 'text', true, true, true, true ],
['machine.os.raw', 'keyword', true, false, true, true, { docValues: true } ],
['geo.src', 'keyword', true, true, true, true ],
['_id', 'keyword', false, false, true, true ],
['_type', 'keyword', false, false, true, true ],
['_source', 'keyword', false, false, true, true ],
['non-filterable', 'text', false, false, true, false],
['non-sortable', 'text', false, false, false, false],
['custom_user_field', 'conflict', false, false, true, true ],
['script string', 'text', false, false, true, false, { script: '\'i am a string\'' } ],
['script number', 'long', false, false, true, false, { script: '1234' } ],
['script date', 'date', false, false, true, false, { script: '1234', lang: 'painless' } ],
['script murmur3', 'murmur3', false, false, true, false, { script: '1234' } ],
// |aggregatable
// | |searchable
// name esType | | |metadata
['bytes', 'long', true, true, { count: 10 } ],
['ssl', 'boolean', true, true, { count: 20 } ],
['@timestamp', 'date', true, true, { count: 30 } ],
['time', 'date', true, true, { count: 30 } ],
['@tags', 'keyword', true, true ],
['utc_time', 'date', true, true ],
['phpmemory', 'integer', true, true ],
['ip', 'ip', true, true ],
['request_body', 'attachment', true, true ],
['point', 'geo_point', true, true ],
['area', 'geo_shape', true, true ],
['hashed', 'murmur3', false, true ],
['geo.coordinates', 'geo_point', true, true ],
['extension', 'keyword', true, true ],
['machine.os', 'text', true, true ],
['machine.os.raw', 'keyword', true, true ],
['geo.src', 'keyword', true, true ],
['_id', '_id', true, true ],
['_type', '_type', true, true ],
['_source', '_source', true, true ],
['non-filterable', 'text', true, false],
['non-sortable', 'text', false, false],
['custom_user_field', 'conflict', true, true ],
['script string', 'text', true, false, { script: '\'i am a string\'' } ],
['script number', 'long', true, false, { script: '1234' } ],
['script date', 'date', true, false, { script: '1234', lang: 'painless' } ],
['script murmur3', 'murmur3', true, false, { script: '1234' } ],
].map(function (row) {
const [
name,
esType,
indexed,
analyzed,
aggregatable,
searchable,
metadata = {}
] = row;
const {
docValues = false,
count = 0,
script,
lang = script ? 'expression' : undefined,
@ -60,9 +56,7 @@ function stubbedLogstashFields() {
return {
name,
type,
doc_values: docValues,
indexed,
analyzed,
readFromDocValues: shouldReadFieldFromDocValues(aggregatable, esType),
aggregatable,
searchable,
count,

20
src/ui/public/agg_types/__tests__/metrics/top_hit.js
View file

@ -71,15 +71,27 @@ describe('Top hit metric', function () {
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});
it('should request both for the source and doc_values fields', function () {
it('requests both source and docvalues_fields for non-text aggregatable fields', function () {
init({ field: 'bytes' });
expect(aggDsl.top_hits._source).to.be('bytes');
expect(aggDsl.top_hits.docvalue_fields).to.eql([ 'bytes' ]);
});
it('should only request for the source if the field does not have the doc_values property', function () {
init({ field: 'ssl' });
expect(aggDsl.top_hits._source).to.be('ssl');
it('requests just source for aggregatable text fields', function () {
init({ field: 'machine.os' });
expect(aggDsl.top_hits._source).to.be('machine.os');
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});
it('requests just source for not-aggregatable text fields', function () {
init({ field: 'non-sortable' });
expect(aggDsl.top_hits._source).to.be('non-sortable');
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});
it('requests just source for not-aggregatable, non-text fields', function () {
init({ field: 'hashed' });
expect(aggDsl.top_hits._source).to.be('hashed');
expect(aggDsl.top_hits.docvalue_fields).to.be(undefined);
});

19
src/ui/public/agg_types/controls/field.html
View file

@ -3,25 +3,6 @@
Field
</label>
<span class="pull-right text-warning hintbox-label" ng-show="aggParam.showAnalyzedWarning && agg.params.field.analyzed"
ng-click="showAnalyzedFieldWarning = !showAnalyzedFieldWarning">
<i class="fa fa-warning"></i> Analyzed Field
</span>
<div class="hintbox" ng-show="showAnalyzedFieldWarning && agg.params.field.analyzed">
<p>
<strong>Careful!</strong> The field selected contains analyzed strings. Analyzed strings are highly unique and can use a lot of memory to visualize. Values such as <i>foo-bar</i> will be broken into <i>foo</i> and <i>bar</i>. See <a href="http://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html" target="_blank">Mapping Types</a> for more information on setting this field as <i>keyword</i> (not analyzed).
</p>
<p ng-show="indexedFields.byName[agg.params.field.name + '.keyword'].analyzed == false">
<strong>Tip:</strong> <i>{{agg.params.field.name + '.keyword'}}</i> may be a non analyzed version of this field.
</p>
<p ng-show="indexedFields.byName[agg.params.field.name + '.raw'].analyzed == false">
<strong>Tip:</strong> <i>{{agg.params.field.name + '.raw'}}</i> may be a non analyzed version of this field.
</p>
</div>
<ui-select
name="field"
required

3
src/ui/public/agg_types/metrics/top_hit.js
View file

@ -24,7 +24,6 @@ export function AggTypesMetricsTopHitProvider(Private) {
{
name: 'field',
onlyAggregatable: false,
showAnalyzedWarning: false,
filterFieldTypes: function (vis, value) {
if (vis.type.name === 'table' || vis.type.name === 'metric') {
return true;
@ -45,7 +44,7 @@ export function AggTypesMetricsTopHitProvider(Private) {
}
};
} else {
if (field.doc_values) {
if (field.readFromDocValues) {
output.params.docvalue_fields = [ field.name ];
}
output.params._source = field.name === '_source' ? true : field.name;

2
src/ui/public/agg_types/param_types/field.js
View file

@ -20,8 +20,6 @@ export function AggTypesParamTypesFieldProvider(Private, $filter) {
FieldAggParam.prototype.filterFieldTypes = '*';
// retain only the fields with the aggregatable property if the onlyAggregatable option is true
FieldAggParam.prototype.onlyAggregatable = true;
// show a warning about the field being analyzed
FieldAggParam.prototype.showAnalyzedWarning = true;
/**
* Called to serialize values for saving an aggConfig object

3
src/ui/public/index_patterns/__tests__/_index_pattern.js
View file

@ -157,7 +157,8 @@ describe('index pattern', function () {
sortable: true,
type: 'number',
aggregatable: true,
searchable: false
searchable: false,
readFromDocValues: true
},
{
name: 'script number',

25
src/ui/public/index_patterns/_field.js
View file

@ -42,6 +42,7 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
const scripted = !!spec.scripted;
const searchable = !!spec.searchable || scripted;
const aggregatable = !!spec.aggregatable || scripted;
const readFromDocValues = !!spec.readFromDocValues && !scripted;
const sortable = spec.name === '_score' || ((indexed || aggregatable) && type.sortable);
const filterable = spec.name === '_id' || scripted || ((indexed || searchable) && type.filterable);
const visualizable = aggregatable;
@ -55,14 +56,10 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
obj.writ('script', scripted ? spec.script : null);
obj.writ('lang', scripted ? (spec.lang || 'painless') : null);
// mapping info
obj.fact('indexed', indexed);
obj.fact('analyzed', !!spec.analyzed);
obj.fact('doc_values', !!spec.doc_values);
// stats
obj.fact('searchable', searchable);
obj.fact('aggregatable', aggregatable);
obj.fact('readFromDocValues', readFromDocValues);
// usage flags, read-only and won't be saved
obj.comp('format', format);
@ -81,6 +78,24 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
return obj.create();
}
Object.defineProperties(Field.prototype, {
indexed: {
get() {
throw new Error('field.indexed has been removed, see https://github.com/elastic/kibana/pull/11969');
}
},
analyzed: {
get() {
throw new Error('field.analyzed has been removed, see https://github.com/elastic/kibana/pull/11969');
}
},
doc_values: {
get() {
throw new Error('field.doc_values has been removed, see https://github.com/elastic/kibana/pull/11969');
}
},
});
Field.prototype.routes = {
edit: '/management/kibana/indices/{{indexPattern.id}}/field/{{name}}'
};

18
src/ui/public/index_patterns/_index_pattern.js
View file

@ -95,9 +95,19 @@ export function IndexPatternProvider(Private, Notifier, config, kbnIndex, Promis
return promise;
}
function containsFieldCapabilities(fields) {
return _.any(fields, (field) => {
return _.has(field, 'aggregatable') && _.has(field, 'searchable');
function isFieldRefreshRequired(indexPattern) {
if (!indexPattern.fields) {
return true;
}
return indexPattern.fields.every(field => {
// See https://github.com/elastic/kibana/pull/8421
const hasFieldCaps = ('aggregatable' in field) && ('searchable' in field);
// See https://github.com/elastic/kibana/pull/11969
const hasDocValuesFlag = ('readFromDocValues' in field);
return !hasFieldCaps || !hasDocValuesFlag;
});
}
@ -108,7 +118,7 @@ export function IndexPatternProvider(Private, Notifier, config, kbnIndex, Promis
return promise;
}
if (!indexPattern.fields || !containsFieldCapabilities(indexPattern.fields)) {
if (isFieldRefreshRequired(indexPattern)) {
promise = indexPattern.refreshFields();
}
return promise.then(() => {initFields(indexPattern);});

2
src/utils/kbn_field_types.js
View file

@ -22,7 +22,7 @@ const KBN_FIELD_TYPES = [
name: 'string',
sortable: true,
filterable: true,
esTypes: ['string', 'text', 'keyword'],
esTypes: ['string', 'text', 'keyword', '_type', '_id'],
}),
new KbnFieldType({
name: 'number',

1
test/functional/apps/management/_index_pattern_create_delete.js
View file

@ -52,7 +52,6 @@ export default function ({ getService, getPageObjects }) {
'format',
'searchable',
'aggregatable',
'analyzed',
'excluded',
'controls'
];