[add data] adds grok processor

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui.js

@@ -3,5 +3,6 @@ import './processor_ui_append';
 import './processor_ui_convert';
 import './processor_ui_date';
 import './processor_ui_geoip';
+import './processor_ui_grok';
 import './processor_ui_gsub';
 import './processor_ui_set';

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui_grok.js

@@ -0,0 +1,40 @@
+import uiModules from 'ui/modules';
+import _ from 'lodash';
+import keysDeep from '../lib/keys_deep';
+import template from '../views/processor_ui_grok.html';
+
+const app = uiModules.get('kibana');
+
+//scope.processor, scope.pipeline are attached by the process_container.
+app.directive('processorUiGrok', function () {
+  return {
+    restrict: 'E',
+    template: template,
+    controller : function ($scope) {
+      const processor = $scope.processor;
+      const pipeline = $scope.pipeline;
+
+      function consumeNewInputObject() {
+        $scope.fields = keysDeep(processor.inputObject);
+        refreshFieldData();
+      }
+
+      function refreshFieldData() {
+        $scope.fieldData = _.get(processor.inputObject, processor.sourceField);
+      }
+
+      function processorUiChanged() {
+        pipeline.setDirty();
+      }
+
+      $scope.$watch('processor.inputObject', consumeNewInputObject);
+
+      $scope.$watch('processor.sourceField', () => {
+        refreshFieldData();
+        processorUiChanged();
+      });
+
+      $scope.$watch('processor.pattern', processorUiChanged);
+    }
+  };
+});

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/processor_types.js

@@ -1,3 +1,6 @@
+import _ from 'lodash';
+import keysDeep from './keys_deep';
+
 class Processor {
   constructor(processorId, typeId, title) {
     if (!typeId || !title) {
@@ -124,6 +127,33 @@ export class GeoIp extends Processor {
   }
 };
 
+export class Grok extends Processor {
+  constructor(processorId) {
+    super(processorId, 'grok', 'Grok');
+    this.sourceField = '';
+    this.pattern = '';
+  }
+
+  get description() {
+    const inputKeys = keysDeep(this.inputObject);
+    const outputKeys = keysDeep(this.outputObject);
+    const addedKeys = _.difference(outputKeys, inputKeys);
+    const added = addedKeys.sort().map(field => `[${field}]`).join(', ');
+    const source = this.sourceField || '?';
+
+    return `[${source}] -> ${added}`;
+  }
+
+  get model() {
+    return {
+      processorId: this.processorId,
+      typeId: this.typeId,
+      sourceField: this.sourceField || '',
+      pattern: this.pattern || ''
+    };
+  }
+};
+
 export class Gsub extends Processor {
   constructor(processorId) {
     super(processorId, 'gsub', 'Gsub');

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/views/processor_ui_grok.html

@@ -0,0 +1,16 @@
+<div class="form-group">
+  <label>Field:</label>
+  <select
+    class="form-control"
+    ng-options="field as field for field in fields"
+    ng-model="processor.sourceField">
+  </select>
+</div>
+<div class="form-group">
+  <label>Field Data:</label>
+  <pre>{{ fieldData }}</pre>
+</div>
+<div class="form-group">
+  <label>Pattern:</label>
+  <input type="text" class="form-control" ng-model="processor.pattern">
+</div>

src/plugins/kibana/server/lib/converters/ingest_processor_api_kibana_to_es_converters.js

@@ -67,6 +67,16 @@ export function geoip(processorApiDocument) {
   };
 }
 
+export function grok(processorApiDocument) {
+  return {
+    grok: {
+      tag: processorApiDocument.processor_id,
+      field: processorApiDocument.source_field,
+      pattern: processorApiDocument.pattern
+    }
+  };
+}
+
 export function gsub(processorApiDocument) {
   return {
     gsub: {

src/plugins/kibana/server/lib/schemas/resources/ingest_processor_schemas.js

@@ -33,6 +33,12 @@ export const geoip = base.keys({
   target_field: Joi.string().allow('')
 });
 
+export const grok = base.keys({
+  type_id: Joi.string().only('grok').required(),
+  source_field: Joi.string().allow(''),
+  pattern: Joi.string().allow('')
+});
+
 export const gsub = base.keys({
   type_id: Joi.string().only('gsub').required(),
   source_field: Joi.string().allow(''),

test/unit/api/ingest/processors/_date.js

@@ -28,7 +28,7 @@ define(function (require) {
             input: { dob: '07/05/1979' },
             processors: [{
               processor_id: 'processor1',
-              type_id: 'convert',
+              type_id: 'date',
               source_field: 42,
               target_field: 'dob',
               formats: 'Custom',

test/unit/api/ingest/processors/_grok.js

@@ -0,0 +1,72 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+  const testPipeline = {
+    processors: [{
+      processor_id: 'processor1',
+      type_id: 'grok',
+      source_field: 'foo',
+      pattern: '%{GREEDYDATA:bar} - %{GREEDYDATA:baz}'
+    }],
+    input: { foo: 'value1 - value2' }
+  };
+
+  return function (bdd, scenarioManager, request) {
+    bdd.describe('simulate - grok processor', () => {
+
+      bdd.it('should return 400 for an invalid payload', () => {
+        return Promise.all([
+          // Grok processor requires source_field property
+          request.post('/kibana/ingest/simulate')
+          .send({
+            input: {},
+            processors: [{
+              processor_id: 'processor1',
+              type_id: 'grok',
+              source_field: 123,
+              pattern: '%{GREEDYDATA:bar} - %{GREEDYDATA:baz}'
+            }],
+          })
+          .expect(400)
+        ]);
+      });
+
+      bdd.it('should return 200 for a valid simulate request', () => {
+        return request.post('/kibana/ingest/simulate')
+          .send(testPipeline)
+          .expect(200);
+      });
+
+      bdd.it('should return a simulated output with the correct result for the given processor', function () {
+        const expected = {
+          foo: 'value1 - value2',
+          bar: 'value1',
+          baz: 'value2'
+        };
+        return request.post('/kibana/ingest/simulate')
+          .send(testPipeline)
+          .expect(200)
+          .then(function (response) {
+            expect(response.body[0].output).to.eql(expected);
+          });
+      });
+
+      bdd.it('should enforce snake case', () => {
+        return request.post('/kibana/ingest/simulate')
+        .send({
+          processors: [{
+            processorId: 'processor1',
+            typeId: 'grok',
+            sourceField: 'foo',
+            pattern: '%{GREEDYDATA:bar} - %{GREEDYDATA:baz}'
+          }],
+          input: { foo: 'value1 - value2' }
+        })
+        .expect(400);
+      });
+
+    });
+  };
+});

test/unit/api/ingest/processors/index.js

@@ -3,6 +3,7 @@ define(function (require) {
   var convert = require('./_convert');
   var date = require('./_date');
   var geoip = require('./_geoip');
+  var grok = require('./_grok');
   var gsub = require('./_gsub');
   var set = require('./_set');
 
@@ -11,6 +12,7 @@ define(function (require) {
     convert(bdd, scenarioManager, request);
     date(bdd, scenarioManager, request);
     geoip(bdd, scenarioManager, request);
+    grok(bdd, scenarioManager, request);
     gsub(bdd, scenarioManager, request);
     set(bdd, scenarioManager, request);
   };