Merge pull request #7084 from BigFunger/add-data-split-converters

[add data] split processor converters and types
This commit is contained in:
Jim Unger 2016-05-09 09:19:48 -05:00
commit 1b8adbd0e7
88 changed files with 759 additions and 686 deletions

View file

@ -2,12 +2,13 @@ import uiModules from 'ui/modules';
import _ from 'lodash';
import Pipeline from '../lib/pipeline';
import angular from 'angular';
import * as ProcessorTypes from '../lib/processor_types';
import * as ProcessorTypes from '../processors/view_models';
import IngestProvider from 'ui/ingest';
import '../styles/_pipeline_setup.less';
import './pipeline_output';
import './source_data';
import './processor_ui';
import './processor_ui_container';
import '../processors';
import pipelineSetupTemplate from '../views/pipeline_setup.html';
const app = uiModules.get('kibana');

View file

@ -1,15 +0,0 @@
import './processor_ui_container';
import './processor_ui_append';
import './processor_ui_convert';
import './processor_ui_date';
import './processor_ui_geoip';
import './processor_ui_grok';
import './processor_ui_gsub';
import './processor_ui_join';
import './processor_ui_lowercase';
import './processor_ui_remove';
import './processor_ui_rename';
import './processor_ui_set';
import './processor_ui_split';
import './processor_ui_trim';
import './processor_ui_uppercase';

View file

@ -3,7 +3,7 @@ import _ from 'lodash';
import '../styles/_processor_ui_container.less';
import './output_preview';
import './processor_ui_container_header';
import processorUiContainerTemplate from '../views/processor_ui_container.html';
import template from '../views/processor_ui_container.html';
const app = uiModules.get('kibana');
@ -14,7 +14,7 @@ app.directive('processorUiContainer', function ($compile) {
pipeline: '=',
processor: '='
},
template: processorUiContainerTemplate,
template: template,
link: function ($scope, $el) {
const processor = $scope.processor;
const pipeline = $scope.pipeline;

View file

@ -2,7 +2,7 @@ import _ from 'lodash';
import expect from 'expect.js';
import sinon from 'sinon';
import Pipeline from '../pipeline';
import * as processorTypes from '../processor_types';
import * as processorTypes from '../../processors/view_models';
describe('processor pipeline', function () {

View file

@ -1,354 +0,0 @@
import _ from 'lodash';
import keysDeep from './keys_deep';
class Processor {
constructor(processorId, typeId, title) {
if (!typeId || !title) {
throw new Error('Cannot instantiate the base Processor class.');
}
this.processorId = processorId;
this.title = title;
this.typeId = typeId;
this.collapsed = false;
this.parent = undefined;
this.inputObject = undefined;
this.outputObject = undefined;
this.error = undefined;
}
setParent(newParent) {
const oldParent = this.parent;
this.parent = newParent;
return (oldParent !== this.parent);
}
}
export class Append extends Processor {
constructor(processorId) {
super(processorId, 'append', 'Append');
this.targetField = '';
this.values = [];
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
values: this.values || []
};
}
};
export class Convert extends Processor {
constructor(processorId) {
super(processorId, 'convert', 'Convert');
this.sourceField = '';
this.targetField = '';
this.type = 'auto';
}
get description() {
const source = this.sourceField || '?';
const type = this.type || '?';
const target = this.targetField ? ` -> [${this.targetField}]` : '';
return `[${source}] to ${type}${target}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
type: this.type || 'auto'
};
}
};
export class Date extends Processor {
constructor(processorId) {
super(processorId, 'date', 'Date');
this.sourceField = '';
this.targetField = '@timestamp';
this.formats = [];
this.timezone = 'Etc/UTC';
this.locale = 'ENGLISH';
this.customFormat = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
formats: this.formats || [],
timezone: this.timezone || '',
locale: this.locale || '',
customFormat: this.customFormat || ''
};
}
};
export class GeoIp extends Processor {
constructor(processorId) {
super(processorId, 'geoip', 'Geo IP');
this.sourceField = '';
this.targetField = '';
this.databaseFile = '';
this.databaseFields = [];
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
databaseFile: this.databaseFile || '',
databaseFields: this.databaseFields || []
};
}
};
export class Grok extends Processor {
constructor(processorId) {
super(processorId, 'grok', 'Grok');
this.sourceField = '';
this.pattern = '';
}
get description() {
const inputKeys = keysDeep(this.inputObject);
const outputKeys = keysDeep(this.outputObject);
const addedKeys = _.difference(outputKeys, inputKeys);
const added = addedKeys.sort().map(field => `[${field}]`).join(', ');
const source = this.sourceField || '?';
return `[${source}] -> ${added}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || ''
};
}
};
export class Gsub extends Processor {
constructor(processorId) {
super(processorId, 'gsub', 'Gsub');
this.sourceField = '';
this.pattern = '';
this.replacement = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}] - /${this.pattern}/ -> '${this.replacement}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || '',
replacement: this.replacement || ''
};
}
};
export class Join extends Processor {
constructor(processorId) {
super(processorId, 'join', 'Join');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator ? ` on '${this.separator}'` : '';
return `[${source}]${separator}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};
export class Lowercase extends Processor {
constructor(processorId) {
super(processorId, 'lowercase', 'Lowercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};
export class Remove extends Processor {
constructor(processorId) {
super(processorId, 'remove', 'Remove');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};
export class Rename extends Processor {
constructor(processorId) {
super(processorId, 'rename', 'Rename');
this.sourceField = '';
this.targetField = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || ''
};
}
};
export class Set extends Processor {
constructor(processorId) {
super(processorId, 'set', 'Set');
this.targetField = '';
this.value = '';
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
value: this.value || ''
};
}
};
export class Split extends Processor {
constructor(processorId) {
super(processorId, 'split', 'Split');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator || '?';
return `[${source}] on '${separator}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};
export class Trim extends Processor {
constructor(processorId) {
super(processorId, 'trim', 'Trim');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};
export class Uppercase extends Processor {
constructor(processorId) {
super(processorId, 'uppercase', 'Uppercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,5 +1,5 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_append.html';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,23 @@
import Processor from '../base/view_model';
export class Append extends Processor {
constructor(processorId) {
super(processorId, 'append', 'Append');
this.targetField = '';
this.values = [];
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
values: this.values || []
};
}
};

View file

@ -0,0 +1,23 @@
export default class Processor {
constructor(processorId, typeId, title) {
if (!typeId || !title) {
throw new Error('Cannot instantiate the base Processor class.');
}
this.processorId = processorId;
this.title = title;
this.typeId = typeId;
this.collapsed = false;
this.parent = undefined;
this.inputObject = undefined;
this.outputObject = undefined;
this.error = undefined;
}
setParent(newParent) {
const oldParent = this.parent;
this.parent = newParent;
return (oldParent !== this.parent);
}
}

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_convert.html';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,28 @@
import _ from 'lodash';
import Processor from '../base/view_model';
export class Convert extends Processor {
constructor(processorId) {
super(processorId, 'convert', 'Convert');
this.sourceField = '';
this.targetField = '';
this.type = 'auto';
}
get description() {
const source = this.sourceField || '?';
const type = this.type || '?';
const target = this.targetField ? ` -> [${this.targetField}]` : '';
return `[${source}] to ${type}${target}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
type: this.type || 'auto'
};
}
};

View file

@ -1,9 +1,9 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_date.html';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
const createMultiSelectModel = require('../lib/create_multi_select_model');
import '../styles/_processor_ui_date.less';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import createMultiSelectModel from '../../lib/create_multi_select_model';
import template from './view.html';
import './styles.less';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,32 @@
import Processor from '../base/view_model';
export class Date extends Processor {
constructor(processorId) {
super(processorId, 'date', 'Date');
this.sourceField = '';
this.targetField = '@timestamp';
this.formats = [];
this.timezone = 'Etc/UTC';
this.locale = 'ENGLISH';
this.customFormat = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
formats: this.formats || [],
timezone: this.timezone || '',
locale: this.locale || '',
customFormat: this.customFormat || ''
};
}
};

View file

@ -1,8 +1,8 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_geoip.html';
import '../styles/_processor_ui_geoip.less';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
import './styles.less';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,28 @@
import Processor from '../base/view_model';
export class GeoIp extends Processor {
constructor(processorId) {
super(processorId, 'geoip', 'Geo IP');
this.sourceField = '';
this.targetField = '';
this.databaseFile = '';
this.databaseFields = [];
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
databaseFile: this.databaseFile || '',
databaseFields: this.databaseFields || []
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_grok.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,30 @@
import _ from 'lodash';
import keysDeep from '../../lib/keys_deep';
import Processor from '../base/view_model';
export class Grok extends Processor {
constructor(processorId) {
super(processorId, 'grok', 'Grok');
this.sourceField = '';
this.pattern = '';
}
get description() {
const inputKeys = keysDeep(this.inputObject);
const outputKeys = keysDeep(this.outputObject);
const addedKeys = _.difference(outputKeys, inputKeys);
const added = addedKeys.sort().map(field => `[${field}]`).join(', ');
const source = this.sourceField || '?';
return `[${source}] -> ${added}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_gsub.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,25 @@
import Processor from '../base/view_model';
export class Gsub extends Processor {
constructor(processorId) {
super(processorId, 'gsub', 'Gsub');
this.sourceField = '';
this.pattern = '';
this.replacement = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}] - /${this.pattern}/ -> '${this.replacement}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || '',
replacement: this.replacement || ''
};
}
};

View file

@ -0,0 +1,14 @@
import './append/directive';
import './convert/directive';
import './date/directive';
import './geoip/directive';
import './grok/directive';
import './gsub/directive';
import './join/directive';
import './lowercase/directive';
import './remove/directive';
import './rename/directive';
import './set/directive';
import './split/directive';
import './trim/directive';
import './uppercase/directive';

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_join.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,24 @@
import Processor from '../base/view_model';
export class Join extends Processor {
constructor(processorId) {
super(processorId, 'join', 'Join');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator ? ` on '${this.separator}'` : '';
return `[${source}]${separator}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_lowercase.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Lowercase extends Processor {
constructor(processorId) {
super(processorId, 'lowercase', 'Lowercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_remove.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Remove extends Processor {
constructor(processorId) {
super(processorId, 'remove', 'Remove');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_rename.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,24 @@
import Processor from '../base/view_model';
export class Rename extends Processor {
constructor(processorId) {
super(processorId, 'rename', 'Rename');
this.sourceField = '';
this.targetField = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || ''
};
}
};

View file

@ -1,5 +1,5 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_set.html';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,23 @@
import Processor from '../base/view_model';
export class Set extends Processor {
constructor(processorId) {
super(processorId, 'set', 'Set');
this.targetField = '';
this.value = '';
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
value: this.value || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_split.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,24 @@
import Processor from '../base/view_model';
export class Split extends Processor {
constructor(processorId) {
super(processorId, 'split', 'Split');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator || '?';
return `[${source}] on '${separator}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_trim.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Trim extends Processor {
constructor(processorId) {
super(processorId, 'trim', 'Trim');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_uppercase.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Uppercase extends Processor {
constructor(processorId) {
super(processorId, 'uppercase', 'Uppercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -0,0 +1,14 @@
export { Append } from './append/view_model';
export { Convert } from './convert/view_model';
export { Date } from './date/view_model';
export { GeoIp } from './geoip/view_model';
export { Grok } from './grok/view_model';
export { Gsub } from './gsub/view_model';
export { Join } from './join/view_model';
export { Lowercase } from './lowercase/view_model';
export { Remove } from './remove/view_model';
export { Rename } from './rename/view_model';
export { Set } from './set/view_model';
export { Split } from './split/view_model';
export { Trim } from './trim/view_model';
export { Uppercase } from './uppercase/view_model';

View file

@ -1,5 +1,5 @@
import _ from 'lodash';
import * as ingestProcessorApiKibanaToEsConverters from './ingest_processor_api_kibana_to_es_converters';
import * as ingestProcessorApiKibanaToEsConverters from '../processors/converters';
export default function ingestPipelineApiKibanaToEsConverter(pipelineApiDocument) {
return {

View file

@ -1,175 +0,0 @@
import _ from 'lodash';
export function append(processorApiDocument) {
return {
append: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.values
}
};
}
export function convert(processorApiDocument) {
const types = {
//<kibana type>: <ingest type>,
auto: 'auto',
number: 'float',
string: 'string',
boolean: 'boolean'
};
const processor = {
convert: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
type: types[processorApiDocument.type]
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.convert.target_field = processorApiDocument.target_field;
}
return processor;
}
export function date(processorApiDocument) {
const formats = [];
processorApiDocument.formats.forEach((format) => {
if (format.toUpperCase() === 'CUSTOM') {
if (processorApiDocument.custom_format) {
formats.push(processorApiDocument.custom_format);
}
} else {
formats.push(format);
}
});
return {
date: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field,
formats: formats,
timezone: processorApiDocument.timezone,
locale: processorApiDocument.locale
}
};
}
export function geoip(processorApiDocument) {
const processor = {
geoip: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.geoip.target_field = processorApiDocument.target_field;
}
if (!_.isEmpty(processorApiDocument.database_file)) {
processor.geoip.database_file = processorApiDocument.database_file;
}
if (!_.isEmpty(processorApiDocument.database_fields)) {
processor.geoip.properties = processorApiDocument.database_fields;
}
return processor;
}
export function grok(processorApiDocument) {
return {
grok: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern
}
};
}
export function gsub(processorApiDocument) {
return {
gsub: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern,
replacement: processorApiDocument.replacement
}
};
}
export function join(processorApiDocument) {
return {
join: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}
export function lowercase(processorApiDocument) {
return {
lowercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}
export function remove(processorApiDocument) {
return {
remove: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}
export function rename(processorApiDocument) {
return {
rename: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field
}
};
}
export function set(processorApiDocument) {
return {
set: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.value
}
};
}
export function split(processorApiDocument) {
return {
split: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}
export function trim(processorApiDocument) {
return {
trim: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}
export function uppercase(processorApiDocument) {
return {
uppercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,9 @@
export default function append(processorApiDocument) {
return {
append: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.values
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const append = base.keys({
type_id: Joi.string().only('append').required(),
target_field: Joi.string().allow(''),
values: Joi.array().items(Joi.string().allow(''))
});

View file

@ -0,0 +1,5 @@
import Joi from 'joi';
export const base = Joi.object({
processor_id: Joi.string().required()
});

View file

@ -0,0 +1,24 @@
import _ from 'lodash';
export default function convert(processorApiDocument) {
const types = {
//<kibana type>: <ingest type>,
auto: 'auto',
number: 'float',
string: 'string',
boolean: 'boolean'
};
const processor = {
convert: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
type: types[processorApiDocument.type]
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.convert.target_field = processorApiDocument.target_field;
}
return processor;
}

View file

@ -0,0 +1,9 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const convert = base.keys({
type_id: Joi.string().only('convert').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
type: Joi.string()
});

View file

@ -0,0 +1,14 @@
export append from '../processors/append/kibana_to_es_converter';
export convert from '../processors/convert/kibana_to_es_converter';
export date from '../processors/date/kibana_to_es_converter';
export geoip from '../processors/geoip/kibana_to_es_converter';
export grok from '../processors/grok/kibana_to_es_converter';
export gsub from '../processors/gsub/kibana_to_es_converter';
export join from '../processors/join/kibana_to_es_converter';
export lowercase from '../processors/lowercase/kibana_to_es_converter';
export remove from '../processors/remove/kibana_to_es_converter';
export rename from '../processors/rename/kibana_to_es_converter';
export set from '../processors/set/kibana_to_es_converter';
export split from '../processors/split/kibana_to_es_converter';
export trim from '../processors/trim/kibana_to_es_converter';
export uppercase from '../processors/uppercase/kibana_to_es_converter';

View file

@ -0,0 +1,23 @@
export default function date(processorApiDocument) {
const formats = [];
processorApiDocument.formats.forEach((format) => {
if (format.toUpperCase() === 'CUSTOM') {
if (processorApiDocument.custom_format) {
formats.push(processorApiDocument.custom_format);
}
} else {
formats.push(format);
}
});
return {
date: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field,
formats: formats,
timezone: processorApiDocument.timezone,
locale: processorApiDocument.locale
}
};
}

View file

@ -0,0 +1,12 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const date = base.keys({
type_id: Joi.string().only('date').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
formats: Joi.array().items(Joi.string().allow('')),
timezone: Joi.string().allow(''),
locale: Joi.string().allow(''),
custom_format: Joi.string().allow('')
});

View file

@ -0,0 +1,21 @@
import _ from 'lodash';
export default function geoip(processorApiDocument) {
const processor = {
geoip: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.geoip.target_field = processorApiDocument.target_field;
}
if (!_.isEmpty(processorApiDocument.database_file)) {
processor.geoip.database_file = processorApiDocument.database_file;
}
if (!_.isEmpty(processorApiDocument.database_fields)) {
processor.geoip.properties = processorApiDocument.database_fields;
}
return processor;
}

View file

@ -0,0 +1,10 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const geoip = base.keys({
type_id: Joi.string().only('geoip').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
database_file: Joi.string().allow(''),
database_fields: Joi.array().items(Joi.string().allow('')),
});

View file

@ -0,0 +1,9 @@
export default function grok(processorApiDocument) {
return {
grok: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const grok = base.keys({
type_id: Joi.string().only('grok').required(),
source_field: Joi.string().allow(''),
pattern: Joi.string().allow('')
});

View file

@ -0,0 +1,10 @@
export default function gsub(processorApiDocument) {
return {
gsub: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern,
replacement: processorApiDocument.replacement
}
};
}

View file

@ -0,0 +1,9 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const gsub = base.keys({
type_id: Joi.string().only('gsub').required(),
source_field: Joi.string().allow(''),
pattern: Joi.string().allow(''),
replacement: Joi.string().allow('')
});

View file

@ -0,0 +1,9 @@
export default function join(processorApiDocument) {
return {
join: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const join = base.keys({
type_id: Joi.string().only('join').required(),
source_field: Joi.string().allow(''),
separator: Joi.string().allow('')
});

View file

@ -0,0 +1,8 @@
export default function lowercase(processorApiDocument) {
return {
lowercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,7 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const lowercase = base.keys({
type_id: Joi.string().only('lowercase').required(),
source_field: Joi.string().allow('')
});

View file

@ -0,0 +1,8 @@
export default function remove(processorApiDocument) {
return {
remove: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,7 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const remove = base.keys({
type_id: Joi.string().only('remove').required(),
source_field: Joi.string().allow('')
});

View file

@ -0,0 +1,9 @@
export default function rename(processorApiDocument) {
return {
rename: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const rename = base.keys({
type_id: Joi.string().only('rename').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow('')
});

View file

@ -0,0 +1,14 @@
export { append } from './append/schema';
export { convert } from './convert/schema';
export { date } from './date/schema';
export { geoip } from './geoip/schema';
export { grok } from './grok/schema';
export { gsub } from './gsub/schema';
export { join } from './join/schema';
export { lowercase } from './lowercase/schema';
export { remove } from './remove/schema';
export { rename } from './rename/schema';
export { set } from './set/schema';
export { split } from './split/schema';
export { trim } from './trim/schema';
export { uppercase } from './uppercase/schema';

View file

@ -0,0 +1,9 @@
export default function set(processorApiDocument) {
return {
set: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.value
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const set = base.keys({
type_id: Joi.string().only('set').required(),
target_field: Joi.string().allow(''),
value: Joi.string().allow('')
});

View file

@ -0,0 +1,9 @@
export default function split(processorApiDocument) {
return {
split: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const split = base.keys({
type_id: Joi.string().only('split').required(),
source_field: Joi.string().allow(''),
separator: Joi.string().allow('')
});

View file

@ -0,0 +1,8 @@
export default function trim(processorApiDocument) {
return {
trim: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,7 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const trim = base.keys({
type_id: Joi.string().only('trim').required(),
source_field: Joi.string().allow('')
});

View file

@ -0,0 +1,8 @@
export default function uppercase(processorApiDocument) {
return {
uppercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,7 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const uppercase = base.keys({
type_id: Joi.string().only('uppercase').required(),
source_field: Joi.string().allow('')
});

View file

@ -1,93 +0,0 @@
import Joi from 'joi';
const base = Joi.object({
processor_id: Joi.string().required()
});
export const append = base.keys({
type_id: Joi.string().only('append').required(),
target_field: Joi.string().allow(''),
values: Joi.array().items(Joi.string().allow(''))
});
export const convert = base.keys({
type_id: Joi.string().only('convert').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
type: Joi.string()
});
export const date = base.keys({
type_id: Joi.string().only('date').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
formats: Joi.array().items(Joi.string().allow('')),
timezone: Joi.string().allow(''),
locale: Joi.string().allow(''),
custom_format: Joi.string().allow('')
});
export const geoip = base.keys({
type_id: Joi.string().only('geoip').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
database_file: Joi.string().allow(''),
database_fields: Joi.array().items(Joi.string().allow('')),
});
export const grok = base.keys({
type_id: Joi.string().only('grok').required(),
source_field: Joi.string().allow(''),
pattern: Joi.string().allow('')
});
export const gsub = base.keys({
type_id: Joi.string().only('gsub').required(),
source_field: Joi.string().allow(''),
pattern: Joi.string().allow(''),
replacement: Joi.string().allow('')
});
export const join = base.keys({
type_id: Joi.string().only('join').required(),
source_field: Joi.string().allow(''),
separator: Joi.string().allow('')
});
export const lowercase = base.keys({
type_id: Joi.string().only('lowercase').required(),
source_field: Joi.string().allow('')
});
export const remove = base.keys({
type_id: Joi.string().only('remove').required(),
source_field: Joi.string().allow('')
});
export const rename = base.keys({
type_id: Joi.string().only('rename').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow('')
});
export const set = base.keys({
type_id: Joi.string().only('set').required(),
target_field: Joi.string().allow(''),
value: Joi.string().allow('')
});
export const split = base.keys({
type_id: Joi.string().only('split').required(),
source_field: Joi.string().allow(''),
separator: Joi.string().allow('')
});
export const trim = base.keys({
type_id: Joi.string().only('trim').required(),
source_field: Joi.string().allow('')
});
export const uppercase = base.keys({
type_id: Joi.string().only('uppercase').required(),
source_field: Joi.string().allow('')
});

View file

@ -1,5 +1,5 @@
import _ from 'lodash';
import Joi from 'joi';
import * as ingestProcessorSchemas from './ingest_processor_schemas';
import * as ingestProcessorSchemas from '../../processors/schemas';
module.exports = Joi.array().items(_.values(ingestProcessorSchemas));

View file

@ -1,5 +1,5 @@
import Joi from 'joi';
import * as ingestProcessorSchemas from './resources/ingest_processor_schemas';
import * as ingestProcessorSchemas from '../processors/schemas';
import _ from 'lodash';
export default Joi.object({