github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 17:59:23 -04:00
Code Issues Projects Releases Packages Wiki Activity

[utils] add util for converting between es and kibana types (#11967)

Browse source 
* [utils] add util for converting between es and kibana types

* [utils/kbnFieldTypes] use random strings instead of numbers

* [utils/kbnFieldTypes] ensure that castEsToKbnFieldType() returns correct instance

* [utils/kbnFieldTypes] change getEsTypes() -> getKbnTypeNames()

* [fieldEditor] update test to validate limited type support

* [fieldEditor] unexpected scripted field langs should list all kbn types

* [test/stubbedLogstashIndexPattern] fix kbnFieldType use

* [utils/kbnFieldTypes] remove unused castEsToKbnFieldType() fn

* [utils/kbnFieldTypes] don't use Object.freeze()

Object.freeze() keeps properties of an object from being redefined or removed, which we want for the kbnFieldTypes, but it also prevents them from being iterated by angular since angular needs to add a unique `$hashKey` property to each object. To keep the properties read only but allow extension KbnFieldType uses Object.defineProperties() instead.

* [fixtures/logstashFields] fix use of "unknown" and "conflict" types

* [stubs/logstashFields] mention why "conflict" is special

* [utils/kbnFieldTypes] check complete output of getKbnTypeNames()
This commit is contained in:
Spencer 2017-05-24 16:10:52 -07:00 • committed by GitHub
parent dde4e9a127
commit c9afc8b16f
14 changed files with 272 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/core_plugins/kibana/public/management/sections/objects/_view.js
View file

@ -3,9 +3,9 @@ import angular from 'angular';
import rison from 'rison-node';
import { savedObjectManagementRegistry } from 'plugins/kibana/management/saved_object_registry';
import objectViewHTML from 'plugins/kibana/management/sections/objects/_view.html';
import { IndexPatternsCastMappingTypeProvider } from 'ui/index_patterns/_cast_mapping_type';
import uiRoutes from 'ui/routes';
import { uiModules } from 'ui/modules';
import { castEsToKbnFieldTypeName } from '../../../../../../utils';
uiRoutes
.when('/management/kibana/objects/:service/:id', {
@ -16,9 +16,8 @@ uiModules.get('apps/management')
.directive('kbnManagementObjectsView', function (kbnIndex, Notifier, confirmModal) {
return {
restrict: 'E',
controller: function ($scope, $injector, $routeParams, $location, $window, $rootScope, esAdmin, Private) {
controller: function ($scope, $injector, $routeParams, $location, $window, $rootScope, esAdmin) {
const notify = new Notifier({ location: 'SavedObject view' });
const castMappingType = Private(IndexPatternsCastMappingTypeProvider);
const serviceObj = savedObjectManagementRegistry.get($routeParams.service);
const service = $injector.get(serviceObj.service);
@ -81,7 +80,7 @@ uiModules.get('apps/management')
fields.push({
name: name,
type: (function () {
switch (castMappingType(esType)) {
switch (castEsToKbnFieldTypeName(esType)) {
case 'string': return 'text';
case 'number': return 'number';
case 'boolean': return 'boolean';

38
src/fixtures/logstash_fields.js
View file

@ -1,41 +1,43 @@
import { castEsToKbnFieldTypeName } from '../utils';
function stubbedLogstashFields() {
return [
// |indexed
// | |analyzed
// | | |aggregatable
// | | | |searchable
// name type | | | | |metadata
['bytes', 'number', true, true, true, true, { count: 10, docValues: true } ],
// name esType | | | | |metadata
['bytes', 'long', true, true, true, true, { count: 10, docValues: true } ],
['ssl', 'boolean', true, true, true, true, { count: 20 } ],
['@timestamp', 'date', true, true, true, true, { count: 30 } ],
['time', 'date', true, true, true, true, { count: 30 } ],
['@tags', 'string', true, true, true, true ],
['@tags', 'keyword', true, true, true, true ],
['utc_time', 'date', true, true, true, true ],
['phpmemory', 'number', true, true, true, true ],
['phpmemory', 'integer', true, true, true, true ],
['ip', 'ip', true, true, true, true ],
['request_body', 'attachment', true, true, true, true ],
['point', 'geo_point', true, true, true, true ],
['area', 'geo_shape', true, true, true, true ],
['hashed', 'murmur3', true, true, false, true ],
['geo.coordinates', 'geo_point', true, true, true, true ],
['extension', 'string', true, true, true, true ],
['machine.os', 'string', true, true, true, true ],
['machine.os.raw', 'string', true, false, true, true, { docValues: true } ],
['geo.src', 'string', true, true, true, true ],
['_id', 'string', false, false, true, true ],
['_type', 'string', false, false, true, true ],
['_source', 'string', false, false, true, true ],
['non-filterable', 'string', false, false, true, false],
['non-sortable', 'string', false, false, false, false],
['extension', 'keyword', true, true, true, true ],
['machine.os', 'text', true, true, true, true ],
['machine.os.raw', 'keyword', true, false, true, true, { docValues: true } ],
['geo.src', 'keyword', true, true, true, true ],
['_id', 'keyword', false, false, true, true ],
['_type', 'keyword', false, false, true, true ],
['_source', 'keyword', false, false, true, true ],
['non-filterable', 'text', false, false, true, false],
['non-sortable', 'text', false, false, false, false],
['custom_user_field', 'conflict', false, false, true, true ],
['script string', 'string', false, false, true, false, { script: '\'i am a string\'' } ],
['script number', 'number', false, false, true, false, { script: '1234' } ],
['script string', 'text', false, false, true, false, { script: '\'i am a string\'' } ],
['script number', 'long', false, false, true, false, { script: '1234' } ],
['script date', 'date', false, false, true, false, { script: '1234', lang: 'painless' } ],
['script murmur3', 'murmur3', false, false, true, false, { script: '1234' } ],
].map(function (row) {
const [
name,
type,
esType,
indexed,
analyzed,
aggregatable,
@ -51,6 +53,10 @@ function stubbedLogstashFields() {
scripted = !!script,
} = metadata;
// the conflict type is actually a kbnFieldType, we
// don't have any other way to represent it here
const type = esType === 'conflict' ? esType : castEsToKbnFieldTypeName(esType);
return {
name,
type,

23
src/fixtures/stubbed_logstash_index_pattern.js
View file

@ -1,21 +1,24 @@
import _ from 'lodash';
import TestUtilsStubIndexPatternProvider from 'test_utils/stub_index_pattern';
import { IndexPatternsFieldTypesProvider } from 'ui/index_patterns/_field_types';
import FixturesLogstashFieldsProvider from 'fixtures/logstash_fields';
import { getKbnFieldType } from '../utils';
export default function stubbedLogstashIndexPatternService(Private) {
const StubIndexPattern = Private(TestUtilsStubIndexPatternProvider);
const fieldTypes = Private(IndexPatternsFieldTypesProvider);
const mockLogstashFields = Private(FixturesLogstashFieldsProvider);
const fields = mockLogstashFields.map(function (field) {
field.displayName = field.name;
const type = fieldTypes.byName[field.type];
if (!type) throw new TypeError('unknown type ' + field.type);
if (!_.has(field, 'sortable')) field.sortable = type.sortable;
if (!_.has(field, 'filterable')) field.filterable = type.filterable;
return field;
const kbnType = getKbnFieldType(field.type);
if (kbnType.name === 'unknown') {
throw new TypeError(`unknown type ${field.type}`);
}
return {
...field,
sortable: ('sortable' in field) ? !!field.sortable : kbnType.sortable,
filterable: ('filterable' in field) ? !!field.filterable : kbnType.filterable,
displayName: field.name,
};
});
const indexPattern = new StubIndexPattern('logstash-*', 'time', fields);

32
src/ui/public/field_editor/__tests__/field_editor.js
View file

@ -16,13 +16,13 @@ describe('FieldEditor directive', function () {
let $el;
let $httpBackend;
let getScriptedLangsResponse;
beforeEach(ngMock.module('kibana'));
beforeEach(ngMock.inject(function ($compile, $injector, Private) {
$httpBackend = $injector.get('$httpBackend');
$httpBackend
.when('GET', '/api/kibana/scripts/languages')
.respond(['expression', 'painless']);
getScriptedLangsResponse = $httpBackend.when('GET', '/api/kibana/scripts/languages');
getScriptedLangsResponse.respond(['expression', 'painless']);
$rootScope = $injector.get('$rootScope');
Field = Private(IndexPatternsFieldProvider);
@ -153,12 +153,15 @@ describe('FieldEditor directive', function () {
expect(field.lang).to.be('expression');
});
it('provides lang options based on what is enabled for inline use in ES', function () {
it('limits lang options to "expression" and "painless"', function () {
getScriptedLangsResponse
.respond(['expression', 'painless', 'groovy']);
$httpBackend.flush();
expect(_.isEqual(editor.scriptingLangs, ['expression', 'painless'])).to.be.ok();
expect(editor.scriptingLangs).to.eql(['expression', 'painless']);
});
it('provides curated type options based on language', function () {
it('provides specific type when language is painless', function () {
$rootScope.$apply();
expect(editor.fieldTypes).to.have.length(1);
expect(editor.fieldTypes[0]).to.be('number');
@ -170,6 +173,23 @@ describe('FieldEditor directive', function () {
expect(_.isEqual(editor.fieldTypes, ['number', 'string', 'date', 'boolean'])).to.be.ok();
});
it('provides all kibana types when language is groovy (only possible in 5.x)', function () {
$rootScope.$apply();
expect(editor.fieldTypes).to.have.length(1);
expect(editor.fieldTypes[0]).to.be('number');
editor.field.lang = 'groovy';
$rootScope.$apply();
expect(editor.fieldTypes).to.contain('number');
expect(editor.fieldTypes).to.contain('string');
expect(editor.fieldTypes).to.contain('geo_point');
expect(editor.fieldTypes).to.contain('ip');
expect(editor.fieldTypes).to.not.contain('text');
expect(editor.fieldTypes).to.not.contain('keyword');
expect(editor.fieldTypes).to.not.contain('attachement');
});
it('updates formatter options based on field type', function () {
field.lang = 'painless';

4
src/ui/public/field_editor/field_editor.js
View file

@ -6,10 +6,10 @@ import { RegistryFieldFormatsProvider } from 'ui/registry/field_formats';
import { IndexPatternsFieldProvider } from 'ui/index_patterns/_field';
import { uiModules } from 'ui/modules';
import fieldEditorTemplate from 'ui/field_editor/field_editor.html';
import { IndexPatternsCastMappingTypeProvider } from 'ui/index_patterns/_cast_mapping_type';
import { documentationLinks } from '../documentation_links/documentation_links';
import './field_editor.less';
import { GetEnabledScriptingLanguagesProvider, getSupportedScriptingLanguages } from '../scripting_languages';
import { getKbnTypeNames } from '../../../utils';
uiModules
.get('kibana', ['colorpicker.module'])
@ -21,7 +21,7 @@ uiModules
const fieldTypesByLang = {
painless: ['number', 'string', 'date', 'boolean'],
expression: ['number'],
default: _.keys(Private(IndexPatternsCastMappingTypeProvider).types.byType)
default: getKbnTypeNames()
};
return {

68
src/ui/public/index_patterns/__tests__/_cast_mapping_type.js
View file

@ -1,68 +0,0 @@
import _ from 'lodash';
import ngMock from 'ng_mock';
import expect from 'expect.js';
import { IndexPatternsCastMappingTypeProvider } from 'ui/index_patterns/_cast_mapping_type';
describe('type normalizer (castMappingType)', function () {
let fn;
beforeEach(ngMock.module('kibana'));
beforeEach(ngMock.inject(function (Private) {
fn = Private(IndexPatternsCastMappingTypeProvider);
}));
it('should be a function', function () {
expect(fn).to.be.a(Function);
});
it('should have a types property', function () {
expect(fn).to.have.property('types');
});
it('should cast numeric types to "number"', function () {
const types = [
'float',
'double',
'integer',
'long',
'short',
'byte',
'token_count'
];
_.each(types, function (type) {
expect(fn(type)).to.be('number');
});
});
it('should treat non-numeric known types as what they are', function () {
const types = [
'date',
'boolean',
'ip',
'attachment',
'geo_point',
'geo_shape',
'murmur3',
'string'
];
_.each(types, function (type) {
expect(fn(type)).to.be(type);
});
});
it('should cast text and keyword types to "string"', function () {
const types = [
'keyword',
'text'
];
_.each(types, function (type) {
expect(fn(type)).to.be('string');
});
});
it('should treat everything else as a string', function () {
expect(fn('fooTypeIsNotReal')).to.be('string');
});
});

1
src/ui/public/index_patterns/__tests__/index.js
View file

@ -1,5 +1,4 @@
import './_index_pattern';
import './_cast_mapping_type';
import './_map_field';
import './_pattern_to_wildcard';
import './_get_computed_fields';

45
src/ui/public/index_patterns/_cast_mapping_type.js
View file

@ -1,45 +0,0 @@
import { IndexedArray } from 'ui/indexed_array';
export function IndexPatternsCastMappingTypeProvider() {
castMappingType.types = new IndexedArray({
index: ['name'],
group: ['type'],
immutable: true,
initialSet: [
{ name: 'string', type: 'string', group: 'base' },
{ name: 'text', type: 'string', group: 'base' },
{ name: 'keyword', type: 'string', group: 'base' },
{ name: 'date', type: 'date', group: 'base' },
{ name: 'boolean', type: 'boolean', group: 'base' },
{ name: 'float', type: 'number', group: 'number' },
{ name: 'half_float', type: 'number', group: 'number' },
{ name: 'scaled_float', type: 'number', group: 'number' },
{ name: 'double', type: 'number', group: 'number' },
{ name: 'integer', type: 'number', group: 'number' },
{ name: 'long', type: 'number', group: 'number' },
{ name: 'short', type: 'number', group: 'number' },
{ name: 'byte', type: 'number', group: 'number' },
{ name: 'token_count', type: 'number', group: 'number' },
{ name: 'geo_point', type: 'geo_point', group: 'geo' },
{ name: 'geo_shape', type: 'geo_shape', group: 'geo' },
{ name: 'ip', type: 'ip', group: 'other' },
{ name: 'attachment', type: 'attachment', group: 'other' },
{ name: 'murmur3', type: 'murmur3', group: 'hash' }
]
});
/**
* Accepts a mapping type, and converts it into it's js equivilent
* @param {String} type - the type from the mapping's 'type' field
* @return {String} - the most specific type that we care for
*/
function castMappingType(name) {
if (!name) return 'unknown';
const match = castMappingType.types.byName[name];
return match ? match.type : 'string';
}
return castMappingType;
}

7
src/ui/public/index_patterns/_field.js
View file

@ -1,12 +1,11 @@
import { ObjDefine } from 'ui/utils/obj_define';
import { IndexPatternsFieldFormatProvider } from 'ui/index_patterns/_field_format/field_format';
import { IndexPatternsFieldTypesProvider } from 'ui/index_patterns/_field_types';
import { RegistryFieldFormatsProvider } from 'ui/registry/field_formats';
import { getKbnFieldType } from '../../../utils';
export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope, Notifier) {
const notify = new Notifier({ location: 'IndexPattern Field' });
const FieldFormat = Private(IndexPatternsFieldFormatProvider);
const fieldTypes = Private(IndexPatternsFieldTypesProvider);
const fieldFormats = Private(RegistryFieldFormatsProvider);
function Field(indexPattern, spec) {
@ -23,7 +22,7 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
}
// find the type for this field, fallback to unknown type
let type = fieldTypes.byName[spec.type];
let type = getKbnFieldType(spec.type);
if (spec.type && !type) {
notify.error(
'Unknown field type "' + spec.type + '"' +
@ -32,7 +31,7 @@ export function IndexPatternsFieldProvider(Private, shortDotsFilter, $rootScope,
);
}
if (!type) type = fieldTypes.byName.unknown;
if (!type) type = getKbnFieldType('unknown');
let format = spec.format;
if (!format || !(format instanceof FieldFormat)) {

24
src/ui/public/index_patterns/_field_types.js
View file

@ -1,24 +0,0 @@
import { IndexedArray } from 'ui/indexed_array';
export function IndexPatternsFieldTypesProvider() {
return new IndexedArray({
index: ['name'],
group: ['sortable', 'filterable'],
immutable: true,
initialSet: [
{ name: 'ip', sortable: true, filterable: true },
{ name: 'date', sortable: true, filterable: true },
{ name: 'string', sortable: true, filterable: true },
{ name: 'number', sortable: true, filterable: true },
{ name: 'boolean', sortable: true, filterable: true },
{ name: 'conflict', sortable: false, filterable: false },
{ name: 'geo_point', sortable: false, filterable: false },
{ name: 'geo_shape', sortable: false, filterable: false },
{ name: 'attachment', sortable: false, filterable: false },
{ name: 'murmur3', sortable: false, filterable: false },
{ name: 'unknown', sortable: false, filterable: false },
{ name: '_source', sortable: false, filterable: false },
]
});
}

7
src/ui/public/index_patterns/_map_field.js
View file

@ -1,9 +1,8 @@
import _ from 'lodash';
import { IndexPatternsCastMappingTypeProvider } from 'ui/index_patterns/_cast_mapping_type';
import { castEsToKbnFieldTypeName } from '../../../utils';
export function IndexPatternsMapFieldProvider(Private, config) {
const castMappingType = Private(IndexPatternsCastMappingTypeProvider);
/**
* Accepts a field object and its name, and tries to give it a mapping
* @param {Object} field - the field mapping returned by elasticsearch
@ -41,7 +40,7 @@ export function IndexPatternsMapFieldProvider(Private, config) {
mapping.analyzed = mapping.index === 'analyzed' || mapping.type === 'text';
mapping.type = castMappingType(mapping.type);
mapping.type = castEsToKbnFieldTypeName(mapping.type);
if (mappingOverrides[name]) {
_.merge(mapping, mappingOverrides[name]);

90
src/utils/__tests__/kbn_field_types.js Normal file
View file

@ -0,0 +1,90 @@
import expect from 'expect.js';
import Chance from 'chance';
const chance = new Chance();
import {
KbnFieldType,
getKbnFieldType,
castEsToKbnFieldTypeName,
getKbnTypeNames
} from '../kbn_field_types';
describe('utils/kbn_field_types', () => {
describe('KbnFieldType', () => {
it('defaults', () => {
expect(new KbnFieldType())
.to.have.property('name', undefined)
.and.have.property('sortable', false)
.and.have.property('filterable', false)
.and.have.property('esTypes').eql([]);
});
it('assigns name, sortable, filterable, and esTypes options to itself', () => {
const name = chance.word();
const sortable = chance.bool();
const filterable = chance.bool();
const esTypes = chance.n(chance.word, 3);
expect(new KbnFieldType({ name, sortable, filterable, esTypes }))
.to.have.property('name', name)
.and.have.property('sortable', sortable)
.and.have.property('filterable', filterable)
.and.have.property('esTypes').eql(esTypes);
});
it('prevents modification', () => {
const type = new KbnFieldType();
expect(() => type.name = null).to.throwError();
expect(() => type.sortable = null).to.throwError();
expect(() => type.filterable = null).to.throwError();
expect(() => type.esTypes = null).to.throwError();
expect(() => type.esTypes.push(null)).to.throwError();
});
it('allows extension', () => {
const type = new KbnFieldType();
type.$hashKey = '123';
expect(type).to.have.property('$hashKey', '123');
});
});
describe('getKbnFieldType()', () => {
it('returns a KbnFieldType instance by name', () => {
expect(getKbnFieldType('string')).to.be.a(KbnFieldType);
});
it('returns undefined for invalid name', () => {
expect(getKbnFieldType(chance.sentence())).to.be(undefined);
});
});
describe('castEsToKbnFieldTypeName()', () => {
it('returns the kbnFieldType name that matches the esType', () => {
expect(castEsToKbnFieldTypeName('keyword')).to.be('string');
expect(castEsToKbnFieldTypeName('float')).to.be('number');
});
it('returns unknown for unknown es types', () => {
expect(castEsToKbnFieldTypeName(chance.sentence())).to.be('unknown');
});
});
describe('getKbnTypeNames()', () => {
it('returns a list of all kbnFieldType names', () => {
expect(getKbnTypeNames().sort()).to.eql([
'_source',
'attachment',
'boolean',
'conflict',
'date',
'geo_point',
'geo_shape',
'ip',
'murmur3',
'number',
'string',
'unknown',
]);
});
});
});

6
src/utils/index.js
View file

@ -8,6 +8,12 @@ export { encodeQueryComponent } from './encode_query_component';
export { modifyUrl } from './modify_url';
export { createToolingLog } from './tooling_log';
export {
getKbnTypeNames,
getKbnFieldType,
castEsToKbnFieldTypeName,
} from './kbn_field_types';
export {
createConcatStream,
createIntersperseStream,

104
src/utils/kbn_field_types.js Normal file
View file

@ -0,0 +1,104 @@
export class KbnFieldType {
constructor(options = {}) {
const {
name,
sortable = false,
filterable = false,
esTypes = []
} = options;
Object.defineProperties(this, {
name: { value: name },
sortable: { value: sortable },
filterable: { value: filterable },
esTypes: { value: Object.freeze(esTypes.slice()) },
});
}
}
const KBN_FIELD_TYPES = [
new KbnFieldType({
name: 'string',
sortable: true,
filterable: true,
esTypes: ['string', 'text', 'keyword'],
}),
new KbnFieldType({
name: 'number',
sortable: true,
filterable: true,
esTypes: ['float', 'half_float', 'scaled_float', 'double', 'integer', 'long', 'short', 'byte', 'token_count'],
}),
new KbnFieldType({
name: 'date',
sortable: true,
filterable: true,
esTypes: ['date'],
}),
new KbnFieldType({
name: 'ip',
sortable: true,
filterable: true,
esTypes: ['ip'],
}),
new KbnFieldType({
name: 'boolean',
sortable: true,
filterable: true,
esTypes: ['boolean'],
}),
new KbnFieldType({
name: 'geo_point',
esTypes: ['geo_point'],
}),
new KbnFieldType({
name: 'geo_shape',
esTypes: ['geo_shape'],
}),
new KbnFieldType({
name: 'attachment',
esTypes: ['attachment'],
}),
new KbnFieldType({
name: 'murmur3',
esTypes: ['murmur3'],
}),
new KbnFieldType({
name: '_source',
esTypes: ['_source'],
}),
new KbnFieldType({
name: 'unknown',
}),
new KbnFieldType({
name: 'conflict',
}),
];
/**
* Get a type object by name
* @param {string} typeName
* @return {KbnFieldType}
*/
export function getKbnFieldType(typeName) {
return KBN_FIELD_TYPES.find(type => type.name === typeName);
}
/**
* Get the KbnFieldType name for an esType string
* @param {string} esType
* @return {string}
*/
export function castEsToKbnFieldTypeName(esType) {
const type = KBN_FIELD_TYPES.find(type => type.esTypes.includes(esType));
return type ? type.name : 'unknown';
}
/**
* Get the esTypes known by all kbnFieldTypes
* @return {Array<string>}
*/
export function getKbnTypeNames() {
return KBN_FIELD_TYPES.map(type => type.name);
}