Merge branch 'feature/ingest' into ingest/bulkAPI

This commit is contained in:
Matthew Bargar 2016-05-10 18:27:51 -04:00
commit 05b198a819
140 changed files with 1431 additions and 804 deletions

View file

@ -55,7 +55,7 @@ Please make sure you have signed the [Contributor License Agreement](http://www.
npm run elasticsearch
```
- Start the development server. _On Windows, you'll need you use Git Bash, Cygwin, or a similar shell that exposes the `sh` command._
- Start the development server. _On Windows, you'll need you use Git Bash, Cygwin, or a similar shell that exposes the `sh` command. And to successfully build you'll need Cygwin optional packages zip, tar, and shasum._
```sh
npm start

View file

@ -6,7 +6,6 @@ Kibana is an open source ([Apache Licensed](https://github.com/elastic/kibana/bl
- Elasticsearch master
- Kibana binary package
- 512 MB of available RAM
## Installation
@ -15,6 +14,7 @@ Kibana is an open source ([Apache Licensed](https://github.com/elastic/kibana/bl
* Run `bin/kibana` on unix, or `bin\kibana.bat` on Windows.
* Visit [http://localhost:5601](http://localhost:5601)
## Upgrade from previous version
* Move any custom configurations in your old kibana.yml to your new one

View file

@ -21,9 +21,4 @@ if [ ! -x "$NODE" ]; then
exit 1
fi
# sets V8 defaults while still allowing them to be overridden
if echo "${@}" | grep -qv "\-\-dev"; then
NODE_OPTIONS="--max-old-space-size=256 $NODE_OPTIONS"
fi
exec "${NODE}" $NODE_OPTIONS "${DIR}/src/cli" ${@}

View file

@ -21,10 +21,8 @@ If Not Exist "%NODE%" (
)
)
echo.%* | findstr /V /C:"--dev" && set NODE_OPTIONS=--max-old-space-size=256 %NODE_OPTIONS%
TITLE Kibana Server
call "%NODE%" %NODE_OPTIONS% "%DIR%\src\cli" %*
"%NODE%" %NODE_OPTIONS% "%DIR%\src\cli" %*
:finally

View file

@ -4,7 +4,6 @@
* <<enabling-ssl, Enabling SSL>>
* <<controlling-access, Controlling Access>>
* <<load-balancing, Load Balancing Across Multiple Elasticsearch Nodes>>
* <<memory-management, Memory management>>
How you deploy Kibana largely depends on your use case. If you are the only user,
you can run Kibana on your local machine and configure it to point to whatever
@ -133,10 +132,3 @@ cluster.name: "my_cluster"
# The Elasticsearch instance to use for all your queries.
elasticsearch_url: "http://localhost:9200"
--------
[float]
[[memory-management]]
=== Memory management
Kibana is built on Node.js which doesn't tune its heap size based on the amount of memory available. To combat this, we set defaults based on the requirements of Kibana needs, while allowing overhead for additional plugins. These defaults can be overridden at runtime, for example `NODE_OPTIONS="--max-old-space-size=512" bin/kibana`.

View file

@ -5,7 +5,6 @@ All you need is:
* Elasticsearch {esversion}
* A modern web browser - http://www.elastic.co/subscriptions/matrix#matrix_browsers[Supported Browsers].
* 512 MB of available RAM
* Information about your Elasticsearch installation:
** URL of the Elasticsearch instance you want to connect to.
** Which Elasticsearch indices you want to search.

View file

@ -87,8 +87,9 @@ async function mergePackageData(settings, packages) {
*/
async function extractArchive(settings) {
const filter = {
paths: [ settings.plugins[0].folder ]
paths: [ `kibana/${settings.plugins[0].folder}` ]
};
await extractFiles(settings.tempArchiveFile, settings.workingPath, 2, filter);
}

View file

@ -106,6 +106,8 @@ describe('docViews', function () {
expect($scope.filter.calledOnce).to.be(true);
cell.find('.fa-search-minus').first().click();
expect($scope.filter.calledTwice).to.be(true);
cell.find('.fa-asterisk').first().click();
expect($scope.filter.calledThrice).to.be(true);
});
it('should NOT apply a filter when clicking non-filterable fields', function () {
@ -115,6 +117,8 @@ describe('docViews', function () {
expect($scope.filter.calledOnce).to.be(false);
cell.find('.fa-search-minus').first().click();
expect($scope.filter.calledTwice).to.be(false);
cell.find('.fa-asterisk').first().click();
expect($scope.filter.calledOnce).to.be(true);
});
});

View file

@ -27,6 +27,15 @@
tooltip-append-to-body="1"
class="fa fa-columns"></i>
</span>
<span ng-if="!indexPattern.metaFields.includes(field)">
<i ng-click="filter('_exists_', field, '+')"
tooltip="Filter for field present"
tooltip-append-to-body="1"
class="fa fa-asterisk"></i>
</span>
<span ng-if="indexPattern.metaFields.includes(field)" tooltip="Unable to filter for presence of meta fields">
<i class="fa fa-asterisk text-muted"></i>
</span>
</td>
<td>

View file

@ -147,7 +147,7 @@ dashboard-grid {
.panel-content {
display: flex;
flex: 1 1 100%;
height: 100%;
height: auto;
}
}
}

View file

@ -7,7 +7,7 @@ Now that you've got a fresh pipeline and index pattern, let's throw some data at
<li>
<span>
<strong>Install Filebeat</strong> on all servers on which you want to tail logs &nbsp;
<a target="_blank" href="https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-installation.html">
<a target="_blank" ng-href="{{installStep.docLinks.installation}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a>
</span>
@ -16,7 +16,7 @@ Now that you've got a fresh pipeline and index pattern, let's throw some data at
<li>
<span>
<strong>Point Filebeat</strong> at the log files you want to tail &nbsp;
<a target="_blank" href="https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-configuration.html">
<a target="_blank" ng-href="{{installStep.docLinks.configuration}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a>
</span>
@ -25,12 +25,12 @@ Now that you've got a fresh pipeline and index pattern, let's throw some data at
<li ng-if="installStep.results.pipeline.processors.length">
<span>
<strong>Configure Filebeat</strong> to send data through your new Elasticsearch pipeline &nbsp;
<a target="_blank" href="https://www.elastic.co/guide/en/beats/filebeat/master/elasticsearch-output.html">
<a target="_blank" ng-href="{{installStep.docLinks.elasticsearchOutput}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a><br/>
At minimum you'll need to configure Filebeat's Elasticsearch output with a hostname, an index name, and a
<a target="_blank"
href="https://www.elastic.co/guide/en/beats/filebeat/master/elasticsearch-output.html#_parameters">
ng-href="{{installStep.docLinks.elasticsearchOutputAnchorParameters}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> paramaters
</a> block. Your config should end up looking something like this:<br/>
<pre>
@ -49,7 +49,7 @@ output:
<li ng-if="!installStep.results.pipeline.processors.length">
<span>
<strong>Configure Filebeat</strong> to send data to Elasticsearch &nbsp;
<a target="_blank" href="https://www.elastic.co/guide/en/beats/filebeat/master/elasticsearch-output.html">
<a target="_blank" ng-href="{{installStep.docLinks.elasticsearchOutput}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a><br/>
At minimum you'll need to configure Filebeat's Elasticsearch output with a hostname and an index name.
@ -67,7 +67,7 @@ output:
<li>
<span>
<strong>Run Filebeat</strong> on each server &nbsp;
<a target="_blank" href="https://www.elastic.co/guide/en/beats/filebeat/master/_step_5_starting_filebeat.html">
<a target="_blank" ng-href="{{installStep.docLinks.startup}}">
<i aria-hidden="true" class="fa fa-info-circle"></i> instructions
</a>
</span>

View file

@ -2,6 +2,7 @@ import modules from 'ui/modules';
import template from './install_filebeat_step.html';
import 'ui/pattern_checker';
import { patternToIngest } from '../../../../../../common/lib/convert_pattern_and_ingest_name';
import { filebeat as docLinks } from '../../../../../../../../ui/public/documentation_links/documentation_links';
import './styles/_add_data_install_filebeat_step.less';
modules.get('apps/settings')
@ -15,6 +16,7 @@ modules.get('apps/settings')
controllerAs: 'installStep',
controller: function ($scope) {
this.pipelineId = patternToIngest(this.results.indexPattern.id);
this.docLinks = docLinks;
}
};
});

View file

@ -2,7 +2,7 @@
Paste in one or more lines from the file you intend to tail. We'll use these samples in the following steps to help
you build an ingest pipeline and configure a Kibana index pattern. Log lines can be raw strings or
formatted as JSON. If your logs are raw strings but you intend to use
<a target="_window" href="https://www.elastic.co/guide/en/beats/filebeat/current/exported-fields.html">Filebeat's metadata</a>,
<a target="_window" ng-href="{{pasteStep.docLinks.exportedFields}}">Filebeat's metadata</a>,
you'll want to paste the JSON as it will come out of Filebeat.
</h2>

View file

@ -1,5 +1,6 @@
import modules from 'ui/modules';
import template from './paste_samples_step.html';
import { filebeat as docLinks } from '../../../../../../../../ui/public/documentation_links/documentation_links';
import _ from 'lodash';
import './styles/_add_data_paste_samples_step.less';
@ -14,6 +15,8 @@ modules.get('apps/settings')
bindToController: true,
controllerAs: 'pasteStep',
controller: function ($scope) {
this.docLinks = docLinks;
if (_.isUndefined(this.rawSamples)) {
this.rawSamples = '';
}

View file

@ -0,0 +1,64 @@
import forEachField from '../lib/for_each_field';
import sinon from 'auto-release-sinon';
import expect from 'expect.js';
describe('forEachField', function () {
let testDoc;
beforeEach(function () {
testDoc = {
foo: [
{bar: [{'baz': 1}]},
{bat: 'boo'}
]
};
});
it('should require a plain object argument', function () {
expect(forEachField).withArgs([], () => {}).to.throwException(/first argument must be a plain object/);
});
it('should not invoke iteratee if collection is null or empty', function () {
const iteratee = sinon.spy();
forEachField({}, iteratee);
expect(iteratee.called).to.not.be.ok();
});
it('should call iteratee for each item in an array field, but not for the array itself', function () {
const iteratee = sinon.spy();
forEachField({foo: [1, 2, 3]}, iteratee);
expect(iteratee.callCount).to.be(3);
expect(iteratee.calledWith(1, 'foo')).to.be.ok();
expect(iteratee.calledWith(2, 'foo')).to.be.ok();
expect(iteratee.calledWith(3, 'foo')).to.be.ok();
});
it('should call iteratee for flattened inner object properties, as well as the object itself', function () {
const iteratee = sinon.spy();
forEachField(testDoc, iteratee);
expect(iteratee.callCount).to.be(5);
expect(iteratee.calledWith(testDoc.foo[0], 'foo')).to.be.ok();
expect(iteratee.calledWith(testDoc.foo[1], 'foo')).to.be.ok();
expect(iteratee.calledWith(testDoc.foo[0].bar[0], 'foo.bar')).to.be.ok();
expect(iteratee.calledWith(1, 'foo.bar.baz')).to.be.ok();
expect(iteratee.calledWith('boo', 'foo.bat')).to.be.ok();
});
it('should detect geo_point fields and should not invoke iteratee for its lat and lon sub properties', function () {
const iteratee = sinon.spy();
const geo = {lat: 38.6631, lon: -90.5771};
forEachField({ geo }, iteratee);
expect(iteratee.callCount).to.be(1);
expect(iteratee.calledWith(geo, 'geo')).to.be.ok();
});
});

View file

@ -0,0 +1,21 @@
import isGeoPointObject from '../lib/is_geo_point_object';
import expect from 'expect.js';
describe('isGeoPointObject', function () {
it('should return true if an object has lat and lon properties', function () {
expect(isGeoPointObject({lat: 38.6631, lon: -90.5771})).to.be(true);
});
it('should return false if the value is not an object', function () {
expect(isGeoPointObject('foo')).to.be(false);
expect(isGeoPointObject(1)).to.be(false);
expect(isGeoPointObject(true)).to.be(false);
expect(isGeoPointObject(null)).to.be(false);
});
it('should return false if the value is an object without lat an lon properties', function () {
expect(isGeoPointObject({foo: 'bar'})).to.be(false);
});
});

View file

@ -0,0 +1,91 @@
import expect from 'expect.js';
import ngMock from 'ng_mock';
describe('pattern review directive', function () {
let $rootScope;
let $compile;
beforeEach(ngMock.module('kibana'));
beforeEach(ngMock.inject(function ($injector, Private) {
$compile = $injector.get('$compile');
$rootScope = $injector.get('$rootScope');
}));
describe('handling geopoints', function () {
it('should detect geo_point fields when they\'re expressed as an object', function () {
const scope = $rootScope.$new();
scope.sampleDoc = {
geoip: {
location: {
lat: 38.6631,
lon: -90.5771
}
}
};
$compile('<pattern-review-step sample-doc="sampleDoc" index-pattern="indexPattern"></pattern-review-step>')(scope);
scope.$digest();
expect(scope).to.have.property('indexPattern');
expect(scope.indexPattern.fields[0].type).to.be('geo_point');
});
it('should not count the lat and lon properties as their own fields', function () {
const scope = $rootScope.$new();
scope.sampleDoc = {
geoip: {
location: {
lat: 38.6631,
lon: -90.5771
}
}
};
$compile('<pattern-review-step sample-doc="sampleDoc" index-pattern="indexPattern"></pattern-review-step>')(scope);
scope.$digest();
expect(scope).to.have.property('indexPattern');
expect(scope.indexPattern.fields[0].type).to.be('geo_point');
expect(scope.indexPattern.fields.length).to.be(1);
});
});
describe('detecting date fields', function () {
it('should detect sample strings in ISO 8601 format as date fields', function () {
const scope = $rootScope.$new();
scope.sampleDoc = {
isodate: '2004-03-08T00:05:49.000Z'
};
$compile('<pattern-review-step sample-doc="sampleDoc" index-pattern="indexPattern"></pattern-review-step>')(scope);
scope.$digest();
expect(scope).to.have.property('indexPattern');
expect(scope.indexPattern.fields[0].type).to.be('date');
});
});
describe('conflicting array values', function () {
it('should detect heterogeneous arrays and flag them with an error message', function () {
const scope = $rootScope.$new();
scope.sampleDoc = {
badarray: ['foo', 42]
};
const element = $compile('<pattern-review-step sample-doc="sampleDoc" index-pattern="indexPattern"></pattern-review-step>')(scope);
const controller = element.controller('patternReviewStep');
scope.$digest();
expect(controller).to.have.property('errors');
// error message should mentioned the conflicting field
expect(controller.errors[0]).to.contain('badarray');
});
});
});

View file

@ -0,0 +1,58 @@
import _ from 'lodash';
import isGeoPointObject from './is_geo_point_object';
// This function recursively traverses an object, visiting each node that elasticsearch would index as a field.
// Iteratee is invoked with two arguments: (value, fieldName). fieldName is the name of the field as elasticsearch
// would see it. For example:
//
// const testDoc = {
// foo: [
// {bar: [{'baz': 1}]},
// {bat: 'boo'}
// ],
// geo: {
// lat: 38.6631,
// lon: -90.5771
// }
// };
//
// forEachField(testDoc, function(value, fieldName) { ... });
//
// The iteratee would be invoked six times, with the following parameters:
// 1. fieldName = 'foo' value = {bar: [{'baz': 1}]}
// 2. fieldName = 'foo' value = {bat: 'boo'}
// 3. fieldName = 'foo.bar' value = {'baz': 1}
// 4. fieldName = 'foo.bar.baz' value = 1
// 5. fieldName = 'foo.bat' value = 'boo'
// 6. fieldName = 'geo' value = {lat: 38.6631, lon: -90.5771}
//
// forEachField handles arrays, objects, and geo_points as elasticsearch would. It does not currently handle nested
// type fields.
function forEachFieldAux(value, iteratee, fieldName) {
if (!_.isObject(value) || isGeoPointObject(value)) {
iteratee(value, fieldName);
}
else if (_.isPlainObject(value)) {
if (!_.isEmpty(fieldName)) {
iteratee(value, fieldName);
fieldName += '.';
}
_.forEach(value, (subValue, key) => {
forEachFieldAux(subValue, iteratee, fieldName + key);
});
}
else if (_.isArray(value)) {
_.forEach(value, (subValue) => {
forEachFieldAux(subValue, iteratee, fieldName);
});
}
}
export default function forEachField(object, iteratee) {
if (!_.isPlainObject(object)) {
throw new Error('first argument must be a plain object');
}
forEachFieldAux(object, iteratee, '');
}

View file

@ -0,0 +1,15 @@
import _ from 'lodash';
export default function isGeoPointObject(object) {
let retVal = false;
if (_.isPlainObject(object)) {
const keys = _.keys(object);
if (keys.length === 2 && _.contains(keys, 'lat') && _.contains(keys, 'lon')) {
retVal = true;
}
}
return retVal;
}

View file

@ -4,6 +4,9 @@
</h2>
<div class="pattern-review form-inline">
<div ng-show="reviewStep.errors.length" class="alert alert-danger">
<div ng-repeat="error in reviewStep.errors">{{ error }}</div>
</div>
<label>Index name or pattern</label>
<span id="pattern-help" class="help-block">Patterns allow you to define dynamic index names using * as a wildcard. Example: filebeat-*</span>
<input ng-model="reviewStep.indexPattern.id" class="pattern-input form-control" aria-describedby="pattern-help"/>
@ -27,3 +30,4 @@
rows="reviewStep.rows"
per-page="10">
</paginated-table>

View file

@ -1,8 +1,11 @@
import modules from 'ui/modules';
import template from './pattern_review_step.html';
import _ from 'lodash';
import editFieldTypeHTML from 'plugins/kibana/settings/sections/indices/partials/_edit_field_type.html';
import editFieldTypeHTML from '../../partials/_edit_field_type.html';
import isGeoPointObject from './lib/is_geo_point_object';
import forEachField from './lib/for_each_field';
import './styles/_add_data_pattern_review_step.less';
import moment from 'moment';
function pickDefaultTimeFieldName(dateFields) {
if (_.isEmpty(dateFields)) {
@ -12,6 +15,10 @@ function pickDefaultTimeFieldName(dateFields) {
return _.includes(dateFields, '@timestamp') ? '@timestamp' : dateFields[0];
}
function findFieldsByType(indexPatternFields, type) {
return _.map(_.filter(indexPatternFields, {type}), 'name');
}
modules.get('apps/settings')
.directive('patternReviewStep', function () {
return {
@ -24,39 +31,47 @@ modules.get('apps/settings')
controllerAs: 'reviewStep',
bindToController: true,
controller: function ($scope, Private) {
this.errors = [];
const sampleFields = {};
if (_.isUndefined(this.indexPattern)) {
this.indexPattern = {};
}
const knownFieldTypes = {};
this.dateFields = [];
this.pipeline.model.processors.forEach((processor) => {
if (processor.typeId === 'geoip') {
const field = processor.targetField || 'geoip';
knownFieldTypes[field] = 'geo_point';
forEachField(this.sampleDoc, (value, fieldName) => {
let type = typeof value;
if (isGeoPointObject(value)) {
type = 'geo_point';
}
else if (processor.typeId === 'date') {
const field = processor.targetField || '@timestamp';
knownFieldTypes[field] = 'date';
this.dateFields.push(field);
if (type === 'string' && moment(value, moment.ISO_8601).isValid()) {
type = 'date';
}
if (value === null) {
type = 'string';
}
if (!_.isUndefined(sampleFields[fieldName]) && (sampleFields[fieldName].type !== type)) {
this.errors.push(`Error in field ${fieldName} - conflicting types '${sampleFields[fieldName].type}' and '${type}'`);
}
else {
sampleFields[fieldName] = {type, value};
}
});
_.defaults(this.indexPattern, {
id: 'filebeat-*',
title: 'filebeat-*',
timeFieldName: pickDefaultTimeFieldName(this.dateFields),
fields: _.map(this.sampleDoc, (value, key) => {
let type = knownFieldTypes[key] || typeof value;
if (type === 'object' && _.isArray(value) && !_.isEmpty(value)) {
type = typeof value[0];
}
return {name: key, type: type};
})
fields: _(sampleFields)
.map((field, fieldName) => {
return {name: fieldName, type: field.type};
})
.reject({type: 'object'})
.value()
});
this.isTimeBased = !!this.indexPattern.timeFieldName;
$scope.$watch('reviewStep.indexPattern.id', (value) => {
this.indexPattern.title = value;
});
@ -69,17 +84,21 @@ modules.get('apps/settings')
}
});
$scope.$watch('reviewStep.indexPattern.fields', (fields) => {
this.dateFields = _.map(_.filter(fields, {type: 'date'}), 'name');
this.dateFields = findFieldsByType(fields, 'date');
}, true);
this.dateFields = findFieldsByType(this.indexPattern.fields, 'date');
this.isTimeBased = !_.isEmpty(this.dateFields);
const buildRows = () => {
this.rows = _.map(this.indexPattern.fields, (field) => {
const sampleValue = this.sampleDoc[field.name];
const {type: detectedType, value: sampleValue} = sampleFields[field.name];
return [
_.escape(field.name),
{
markup: editFieldTypeHTML,
scope: _.assign($scope.$new(), {field: field, knownFieldTypes: knownFieldTypes, buildRows: buildRows}),
scope: _.assign($scope.$new(), {field: field, detectedType: detectedType, buildRows: buildRows}),
value: field.type
},
typeof sampleValue === 'object' ? _.escape(JSON.stringify(sampleValue)) : _.escape(sampleValue)

View file

@ -2,12 +2,13 @@ import uiModules from 'ui/modules';
import _ from 'lodash';
import Pipeline from '../lib/pipeline';
import angular from 'angular';
import * as ProcessorTypes from '../lib/processor_types';
import * as ProcessorTypes from '../processors/view_models';
import IngestProvider from 'ui/ingest';
import '../styles/_pipeline_setup.less';
import './pipeline_output';
import './source_data';
import './processor_ui';
import './processor_ui_container';
import '../processors';
import pipelineSetupTemplate from '../views/pipeline_setup.html';
const app = uiModules.get('kibana');

View file

@ -1,15 +0,0 @@
import './processor_ui_container';
import './processor_ui_append';
import './processor_ui_convert';
import './processor_ui_date';
import './processor_ui_geoip';
import './processor_ui_grok';
import './processor_ui_gsub';
import './processor_ui_join';
import './processor_ui_lowercase';
import './processor_ui_remove';
import './processor_ui_rename';
import './processor_ui_set';
import './processor_ui_split';
import './processor_ui_trim';
import './processor_ui_uppercase';

View file

@ -3,7 +3,7 @@ import _ from 'lodash';
import '../styles/_processor_ui_container.less';
import './output_preview';
import './processor_ui_container_header';
import processorUiContainerTemplate from '../views/processor_ui_container.html';
import template from '../views/processor_ui_container.html';
const app = uiModules.get('kibana');
@ -14,7 +14,7 @@ app.directive('processorUiContainer', function ($compile) {
pipeline: '=',
processor: '='
},
template: processorUiContainerTemplate,
template: template,
link: function ($scope, $el) {
const processor = $scope.processor;
const pipeline = $scope.pipeline;

View file

@ -2,7 +2,7 @@ import _ from 'lodash';
import expect from 'expect.js';
import sinon from 'sinon';
import Pipeline from '../pipeline';
import * as processorTypes from '../processor_types';
import * as processorTypes from '../../processors/view_models';
describe('processor pipeline', function () {

View file

@ -1,354 +0,0 @@
import _ from 'lodash';
import keysDeep from './keys_deep';
class Processor {
constructor(processorId, typeId, title) {
if (!typeId || !title) {
throw new Error('Cannot instantiate the base Processor class.');
}
this.processorId = processorId;
this.title = title;
this.typeId = typeId;
this.collapsed = false;
this.parent = undefined;
this.inputObject = undefined;
this.outputObject = undefined;
this.error = undefined;
}
setParent(newParent) {
const oldParent = this.parent;
this.parent = newParent;
return (oldParent !== this.parent);
}
}
export class Append extends Processor {
constructor(processorId) {
super(processorId, 'append', 'Append');
this.targetField = '';
this.values = [];
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
values: this.values || []
};
}
};
export class Convert extends Processor {
constructor(processorId) {
super(processorId, 'convert', 'Convert');
this.sourceField = '';
this.targetField = '';
this.type = 'auto';
}
get description() {
const source = this.sourceField || '?';
const type = this.type || '?';
const target = this.targetField ? ` -> [${this.targetField}]` : '';
return `[${source}] to ${type}${target}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
type: this.type || 'auto'
};
}
};
export class Date extends Processor {
constructor(processorId) {
super(processorId, 'date', 'Date');
this.sourceField = '';
this.targetField = '@timestamp';
this.formats = [];
this.timezone = 'Etc/UTC';
this.locale = 'ENGLISH';
this.customFormat = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
formats: this.formats || [],
timezone: this.timezone || '',
locale: this.locale || '',
customFormat: this.customFormat || ''
};
}
};
export class GeoIp extends Processor {
constructor(processorId) {
super(processorId, 'geoip', 'Geo IP');
this.sourceField = '';
this.targetField = '';
this.databaseFile = '';
this.databaseFields = [];
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
databaseFile: this.databaseFile || '',
databaseFields: this.databaseFields || []
};
}
};
export class Grok extends Processor {
constructor(processorId) {
super(processorId, 'grok', 'Grok');
this.sourceField = '';
this.pattern = '';
}
get description() {
const inputKeys = keysDeep(this.inputObject);
const outputKeys = keysDeep(this.outputObject);
const addedKeys = _.difference(outputKeys, inputKeys);
const added = addedKeys.sort().map(field => `[${field}]`).join(', ');
const source = this.sourceField || '?';
return `[${source}] -> ${added}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || ''
};
}
};
export class Gsub extends Processor {
constructor(processorId) {
super(processorId, 'gsub', 'Gsub');
this.sourceField = '';
this.pattern = '';
this.replacement = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}] - /${this.pattern}/ -> '${this.replacement}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || '',
replacement: this.replacement || ''
};
}
};
export class Join extends Processor {
constructor(processorId) {
super(processorId, 'join', 'Join');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator ? ` on '${this.separator}'` : '';
return `[${source}]${separator}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};
export class Lowercase extends Processor {
constructor(processorId) {
super(processorId, 'lowercase', 'Lowercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};
export class Remove extends Processor {
constructor(processorId) {
super(processorId, 'remove', 'Remove');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};
export class Rename extends Processor {
constructor(processorId) {
super(processorId, 'rename', 'Rename');
this.sourceField = '';
this.targetField = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || ''
};
}
};
export class Set extends Processor {
constructor(processorId) {
super(processorId, 'set', 'Set');
this.targetField = '';
this.value = '';
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
value: this.value || ''
};
}
};
export class Split extends Processor {
constructor(processorId) {
super(processorId, 'split', 'Split');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator || '?';
return `[${source}] on '${separator}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};
export class Trim extends Processor {
constructor(processorId) {
super(processorId, 'trim', 'Trim');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};
export class Uppercase extends Processor {
constructor(processorId) {
super(processorId, 'uppercase', 'Uppercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,5 +1,5 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_append.html';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -4,5 +4,5 @@
</div>
<div class="form-group">
<label>Values:</label><span> (line delimited)</span>
<textarea ng-model="values" style="display:block; width:100%; height:150px;"></textarea>
<textarea ng-model="values" class="form-control"></textarea>
</div>

View file

@ -0,0 +1,23 @@
import Processor from '../base/view_model';
export class Append extends Processor {
constructor(processorId) {
super(processorId, 'append', 'Append');
this.targetField = '';
this.values = [];
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
values: this.values || []
};
}
};

View file

@ -0,0 +1,23 @@
export default class Processor {
constructor(processorId, typeId, title) {
if (!typeId || !title) {
throw new Error('Cannot instantiate the base Processor class.');
}
this.processorId = processorId;
this.title = title;
this.typeId = typeId;
this.collapsed = false;
this.parent = undefined;
this.inputObject = undefined;
this.outputObject = undefined;
this.error = undefined;
}
setParent(newParent) {
const oldParent = this.parent;
this.parent = newParent;
return (oldParent !== this.parent);
}
}

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_convert.html';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,28 @@
import _ from 'lodash';
import Processor from '../base/view_model';
export class Convert extends Processor {
constructor(processorId) {
super(processorId, 'convert', 'Convert');
this.sourceField = '';
this.targetField = '';
this.type = 'auto';
}
get description() {
const source = this.sourceField || '?';
const type = this.type || '?';
const target = this.targetField ? ` -> [${this.targetField}]` : '';
return `[${source}] to ${type}${target}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
type: this.type || 'auto'
};
}
};

View file

@ -1,9 +1,9 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_date.html';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
const createMultiSelectModel = require('../lib/create_multi_select_model');
import '../styles/_processor_ui_date.less';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import createMultiSelectModel from '../../lib/create_multi_select_model';
import template from './view.html';
import './styles.less';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,32 @@
import Processor from '../base/view_model';
export class Date extends Processor {
constructor(processorId) {
super(processorId, 'date', 'Date');
this.sourceField = '';
this.targetField = '@timestamp';
this.formats = [];
this.timezone = 'Etc/UTC';
this.locale = 'ENGLISH';
this.customFormat = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
formats: this.formats || [],
timezone: this.timezone || '',
locale: this.locale || '',
customFormat: this.customFormat || ''
};
}
};

View file

@ -1,8 +1,8 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_geoip.html';
import '../styles/_processor_ui_geoip.less';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
import './styles.less';
const app = uiModules.get('kibana');

View file

@ -36,7 +36,7 @@
</div>
<div class="form-group">
<label>Data Fields:</label><span> (line delimited)</span>
<textarea ng-model="databaseFields"></textarea>
<textarea ng-model="databaseFields" class="form-control"></textarea>
</div>
</div>
</div>

View file

@ -0,0 +1,28 @@
import Processor from '../base/view_model';
export class GeoIp extends Processor {
constructor(processorId) {
super(processorId, 'geoip', 'Geo IP');
this.sourceField = '';
this.targetField = '';
this.databaseFile = '';
this.databaseFields = [];
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || '',
databaseFile: this.databaseFile || '',
databaseFields: this.databaseFields || []
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_grok.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,30 @@
import _ from 'lodash';
import keysDeep from '../../lib/keys_deep';
import Processor from '../base/view_model';
export class Grok extends Processor {
constructor(processorId) {
super(processorId, 'grok', 'Grok');
this.sourceField = '';
this.pattern = '';
}
get description() {
const inputKeys = keysDeep(this.inputObject);
const outputKeys = keysDeep(this.outputObject);
const addedKeys = _.difference(outputKeys, inputKeys);
const added = addedKeys.sort().map(field => `[${field}]`).join(', ');
const source = this.sourceField || '?';
return `[${source}] -> ${added}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_gsub.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,25 @@
import Processor from '../base/view_model';
export class Gsub extends Processor {
constructor(processorId) {
super(processorId, 'gsub', 'Gsub');
this.sourceField = '';
this.pattern = '';
this.replacement = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}] - /${this.pattern}/ -> '${this.replacement}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
pattern: this.pattern || '',
replacement: this.replacement || ''
};
}
};

View file

@ -0,0 +1,14 @@
import './append/directive';
import './convert/directive';
import './date/directive';
import './geoip/directive';
import './grok/directive';
import './gsub/directive';
import './join/directive';
import './lowercase/directive';
import './remove/directive';
import './rename/directive';
import './set/directive';
import './split/directive';
import './trim/directive';
import './uppercase/directive';

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_join.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,24 @@
import Processor from '../base/view_model';
export class Join extends Processor {
constructor(processorId) {
super(processorId, 'join', 'Join');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator ? ` on '${this.separator}'` : '';
return `[${source}]${separator}`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_lowercase.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Lowercase extends Processor {
constructor(processorId) {
super(processorId, 'lowercase', 'Lowercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_remove.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Remove extends Processor {
constructor(processorId) {
super(processorId, 'remove', 'Remove');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_rename.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,24 @@
import Processor from '../base/view_model';
export class Rename extends Processor {
constructor(processorId) {
super(processorId, 'rename', 'Rename');
this.sourceField = '';
this.targetField = '';
}
get description() {
const source = this.sourceField || '?';
const target = this.targetField || '?';
return `[${source}] -> [${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
targetField: this.targetField || ''
};
}
};

View file

@ -1,5 +1,5 @@
import uiModules from 'ui/modules';
import template from '../views/processor_ui_set.html';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,23 @@
import Processor from '../base/view_model';
export class Set extends Processor {
constructor(processorId) {
super(processorId, 'set', 'Set');
this.targetField = '';
this.value = '';
}
get description() {
const target = this.targetField || '?';
return `[${target}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
targetField: this.targetField || '',
value: this.value || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_split.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,24 @@
import Processor from '../base/view_model';
export class Split extends Processor {
constructor(processorId) {
super(processorId, 'split', 'Split');
this.sourceField = '';
this.separator = '';
}
get description() {
const source = this.sourceField || '?';
const separator = this.separator || '?';
return `[${source}] on '${separator}'`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || '',
separator: this.separator || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_trim.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Trim extends Processor {
constructor(processorId) {
super(processorId, 'trim', 'Trim');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -1,7 +1,7 @@
import uiModules from 'ui/modules';
import _ from 'lodash';
import keysDeep from '../lib/keys_deep';
import template from '../views/processor_ui_uppercase.html';
import uiModules from 'ui/modules';
import keysDeep from '../../lib/keys_deep';
import template from './view.html';
const app = uiModules.get('kibana');

View file

@ -0,0 +1,21 @@
import Processor from '../base/view_model';
export class Uppercase extends Processor {
constructor(processorId) {
super(processorId, 'uppercase', 'Uppercase');
this.sourceField = '';
}
get description() {
const source = this.sourceField || '?';
return `[${source}]`;
}
get model() {
return {
processorId: this.processorId,
typeId: this.typeId,
sourceField: this.sourceField || ''
};
}
};

View file

@ -0,0 +1,14 @@
export { Append } from './append/view_model';
export { Convert } from './convert/view_model';
export { Date } from './date/view_model';
export { GeoIp } from './geoip/view_model';
export { Grok } from './grok/view_model';
export { Gsub } from './gsub/view_model';
export { Join } from './join/view_model';
export { Lowercase } from './lowercase/view_model';
export { Remove } from './remove/view_model';
export { Rename } from './rename/view_model';
export { Set } from './set/view_model';
export { Split } from './split/view_model';
export { Trim } from './trim/view_model';
export { Uppercase } from './uppercase/view_model';

View file

@ -67,4 +67,8 @@ pipeline-setup {
}
}
}
textarea.form-control {
min-height: 150px;
}
}

View file

@ -1,13 +1,12 @@
<select ng-if="knownFieldTypes[field.name] !== 'geo_point'" name="field_type" ng-model="field.type" ng-change="buildRows()">
<select ng-if="detectedType !== 'geo_point'" name="field_type" ng-model="field.type" ng-change="buildRows()">
<option value="string">string</option>
<option value="number">number</option>
<option value="boolean">boolean</option>
<option value="date">date</option>
<option value="geo_point">geo_point</option>
<option value="geo_shape">geo_shape</option>
<option value="ip">ip</option>
</select>
<span ng-if="knownFieldTypes[field.name] === 'geo_point'">
<span ng-if="detectedType === 'geo_point'">
geo_point
</span>

View file

@ -1,13 +1,13 @@
import { saveAs } from '@spalger/filesaver';
import _ from 'lodash';
import { extend, find, flattenDeep, partialRight, pick, pluck, sortBy } from 'lodash';
import angular from 'angular';
import registry from 'plugins/kibana/settings/saved_object_registry';
import objectIndexHTML from 'plugins/kibana/settings/sections/objects/_objects.html';
import 'ui/directives/file_upload';
import uiRoutes from 'ui/routes';
import uiModules from 'ui/modules';
const MAX_SIZE = Math.pow(2, 31) - 1;
const MAX_SIZE = Math.pow(2, 31) - 1;
uiRoutes
.when('/settings/objects', {
@ -41,9 +41,9 @@ uiModules.get('apps/settings')
});
$q.all(services).then(function (data) {
$scope.services = _.sortBy(data, 'title');
$scope.services = sortBy(data, 'title');
let tab = $scope.services[0];
if ($state.tab) $scope.currentTab = tab = _.find($scope.services, {title: $state.tab});
if ($state.tab) $scope.currentTab = tab = find($scope.services, {title: $state.tab});
$scope.$watch('state.tab', function (tab) {
if (!tab) $scope.changeTab($scope.services[0]);
@ -83,23 +83,23 @@ uiModules.get('apps/settings')
};
$scope.bulkDelete = function () {
$scope.currentTab.service.delete(_.pluck($scope.selectedItems, 'id')).then(refreshData).then(function () {
$scope.currentTab.service.delete(pluck($scope.selectedItems, 'id')).then(refreshData).then(function () {
$scope.selectedItems.length = 0;
});
};
$scope.bulkExport = function () {
const objs = $scope.selectedItems.map(_.partialRight(_.extend, {type: $scope.currentTab.type}));
const objs = $scope.selectedItems.map(partialRight(extend, {type: $scope.currentTab.type}));
retrieveAndExportDocs(objs);
};
$scope.exportAll = () => {
Promise.map($scope.services, (service) =>
service.service.scanAll('').then((results) =>
results.hits.map((hit) => _.extend(hit, {type: service.type}))
)
).then((results) => retrieveAndExportDocs(_.flattenDeep(results)));
};
$scope.exportAll = () => Promise
.map($scope.services, service => service.service
.scanAll('')
.then(result => result.hits.map(hit => extend(hit, { type: service.type })))
)
.then(results => retrieveAndExportDocs(flattenDeep(results)))
.catch(error => notify.error(error));
function retrieveAndExportDocs(objs) {
if (!objs.length) return notify.error('No saved objects to export.');
@ -108,7 +108,7 @@ uiModules.get('apps/settings')
body: {docs: objs.map(transformToMget)}
})
.then(function (response) {
saveToFile(response.docs.map(_.partialRight(_.pick, '_id', '_type', '_source')));
saveToFile(response.docs.map(partialRight(pick, '_id', '_type', '_source')));
});
}
@ -131,7 +131,7 @@ uiModules.get('apps/settings')
}
return Promise.map(docs, function (doc) {
const service = _.find($scope.services, {type: doc._type}).service;
const service = find($scope.services, {type: doc._type}).service;
return service.get().then(function (obj) {
obj.id = doc._id;
return obj.applyESResp(doc).then(function () {

View file

@ -1,5 +1,5 @@
import _ from 'lodash';
import * as ingestProcessorApiKibanaToEsConverters from './ingest_processor_api_kibana_to_es_converters';
import * as ingestProcessorApiKibanaToEsConverters from '../processors/converters';
export default function ingestPipelineApiKibanaToEsConverter(pipelineApiDocument) {
return {

View file

@ -1,175 +0,0 @@
import _ from 'lodash';
export function append(processorApiDocument) {
return {
append: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.values
}
};
}
export function convert(processorApiDocument) {
const types = {
//<kibana type>: <ingest type>,
auto: 'auto',
number: 'float',
string: 'string',
boolean: 'boolean'
};
const processor = {
convert: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
type: types[processorApiDocument.type]
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.convert.target_field = processorApiDocument.target_field;
}
return processor;
}
export function date(processorApiDocument) {
const formats = [];
processorApiDocument.formats.forEach((format) => {
if (format.toUpperCase() === 'CUSTOM') {
if (processorApiDocument.custom_format) {
formats.push(processorApiDocument.custom_format);
}
} else {
formats.push(format);
}
});
return {
date: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field,
formats: formats,
timezone: processorApiDocument.timezone,
locale: processorApiDocument.locale
}
};
}
export function geoip(processorApiDocument) {
const processor = {
geoip: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.geoip.target_field = processorApiDocument.target_field;
}
if (!_.isEmpty(processorApiDocument.database_file)) {
processor.geoip.database_file = processorApiDocument.database_file;
}
if (!_.isEmpty(processorApiDocument.database_fields)) {
processor.geoip.properties = processorApiDocument.database_fields;
}
return processor;
}
export function grok(processorApiDocument) {
return {
grok: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern
}
};
}
export function gsub(processorApiDocument) {
return {
gsub: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern,
replacement: processorApiDocument.replacement
}
};
}
export function join(processorApiDocument) {
return {
join: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}
export function lowercase(processorApiDocument) {
return {
lowercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}
export function remove(processorApiDocument) {
return {
remove: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}
export function rename(processorApiDocument) {
return {
rename: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field
}
};
}
export function set(processorApiDocument) {
return {
set: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.value
}
};
}
export function split(processorApiDocument) {
return {
split: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}
export function trim(processorApiDocument) {
return {
trim: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}
export function uppercase(processorApiDocument) {
return {
uppercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,9 @@
export default function append(processorApiDocument) {
return {
append: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.target_field,
value: processorApiDocument.values
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const append = base.keys({
type_id: Joi.string().only('append').required(),
target_field: Joi.string().allow(''),
values: Joi.array().items(Joi.string().allow(''))
});

View file

@ -0,0 +1,5 @@
import Joi from 'joi';
export const base = Joi.object({
processor_id: Joi.string().required()
});

View file

@ -0,0 +1,24 @@
import _ from 'lodash';
export default function convert(processorApiDocument) {
const types = {
//<kibana type>: <ingest type>,
auto: 'auto',
number: 'float',
string: 'string',
boolean: 'boolean'
};
const processor = {
convert: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
type: types[processorApiDocument.type]
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.convert.target_field = processorApiDocument.target_field;
}
return processor;
}

View file

@ -0,0 +1,9 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const convert = base.keys({
type_id: Joi.string().only('convert').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
type: Joi.string()
});

View file

@ -0,0 +1,14 @@
export append from '../processors/append/kibana_to_es_converter';
export convert from '../processors/convert/kibana_to_es_converter';
export date from '../processors/date/kibana_to_es_converter';
export geoip from '../processors/geoip/kibana_to_es_converter';
export grok from '../processors/grok/kibana_to_es_converter';
export gsub from '../processors/gsub/kibana_to_es_converter';
export join from '../processors/join/kibana_to_es_converter';
export lowercase from '../processors/lowercase/kibana_to_es_converter';
export remove from '../processors/remove/kibana_to_es_converter';
export rename from '../processors/rename/kibana_to_es_converter';
export set from '../processors/set/kibana_to_es_converter';
export split from '../processors/split/kibana_to_es_converter';
export trim from '../processors/trim/kibana_to_es_converter';
export uppercase from '../processors/uppercase/kibana_to_es_converter';

View file

@ -0,0 +1,23 @@
export default function date(processorApiDocument) {
const formats = [];
processorApiDocument.formats.forEach((format) => {
if (format.toUpperCase() === 'CUSTOM') {
if (processorApiDocument.custom_format) {
formats.push(processorApiDocument.custom_format);
}
} else {
formats.push(format);
}
});
return {
date: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field,
formats: formats,
timezone: processorApiDocument.timezone,
locale: processorApiDocument.locale
}
};
}

View file

@ -0,0 +1,12 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const date = base.keys({
type_id: Joi.string().only('date').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
formats: Joi.array().items(Joi.string().allow('')),
timezone: Joi.string().allow(''),
locale: Joi.string().allow(''),
custom_format: Joi.string().allow('')
});

View file

@ -0,0 +1,21 @@
import _ from 'lodash';
export default function geoip(processorApiDocument) {
const processor = {
geoip: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
if (!_.isEmpty(processorApiDocument.target_field)) {
processor.geoip.target_field = processorApiDocument.target_field;
}
if (!_.isEmpty(processorApiDocument.database_file)) {
processor.geoip.database_file = processorApiDocument.database_file;
}
if (!_.isEmpty(processorApiDocument.database_fields)) {
processor.geoip.properties = processorApiDocument.database_fields;
}
return processor;
}

View file

@ -0,0 +1,10 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const geoip = base.keys({
type_id: Joi.string().only('geoip').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow(''),
database_file: Joi.string().allow(''),
database_fields: Joi.array().items(Joi.string().allow('')),
});

View file

@ -0,0 +1,9 @@
export default function grok(processorApiDocument) {
return {
grok: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const grok = base.keys({
type_id: Joi.string().only('grok').required(),
source_field: Joi.string().allow(''),
pattern: Joi.string().allow('')
});

View file

@ -0,0 +1,10 @@
export default function gsub(processorApiDocument) {
return {
gsub: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
pattern: processorApiDocument.pattern,
replacement: processorApiDocument.replacement
}
};
}

View file

@ -0,0 +1,9 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const gsub = base.keys({
type_id: Joi.string().only('gsub').required(),
source_field: Joi.string().allow(''),
pattern: Joi.string().allow(''),
replacement: Joi.string().allow('')
});

View file

@ -0,0 +1,9 @@
export default function join(processorApiDocument) {
return {
join: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
separator: processorApiDocument.separator
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const join = base.keys({
type_id: Joi.string().only('join').required(),
source_field: Joi.string().allow(''),
separator: Joi.string().allow('')
});

View file

@ -0,0 +1,8 @@
export default function lowercase(processorApiDocument) {
return {
lowercase: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,7 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const lowercase = base.keys({
type_id: Joi.string().only('lowercase').required(),
source_field: Joi.string().allow('')
});

View file

@ -0,0 +1,8 @@
export default function remove(processorApiDocument) {
return {
remove: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field
}
};
}

View file

@ -0,0 +1,7 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const remove = base.keys({
type_id: Joi.string().only('remove').required(),
source_field: Joi.string().allow('')
});

View file

@ -0,0 +1,9 @@
export default function rename(processorApiDocument) {
return {
rename: {
tag: processorApiDocument.processor_id,
field: processorApiDocument.source_field,
target_field: processorApiDocument.target_field
}
};
}

View file

@ -0,0 +1,8 @@
import Joi from 'joi';
import { base } from '../base/schema';
export const rename = base.keys({
type_id: Joi.string().only('rename').required(),
source_field: Joi.string().allow(''),
target_field: Joi.string().allow('')
});

Some files were not shown because too many files have changed in this diff Show more