[add data] adds the date processor

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui.js

@@ -1,5 +1,6 @@
 import './processor_ui_container';
 import './processor_ui_append';
 import './processor_ui_convert';
+import './processor_ui_date';
 import './processor_ui_gsub';
 import './processor_ui_set';

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/directives/processor_ui_date.js

@@ -0,0 +1,62 @@
+import uiModules from 'ui/modules';
+import template from '../views/processor_ui_date.html';
+import _ from 'lodash';
+import keysDeep from '../lib/keys_deep';
+import selectableArray from '../lib/selectable_array';
+import '../styles/_processor_ui_date.less';
+
+const app = uiModules.get('kibana');
+
+//scope.processor, scope.pipeline are attached by the process_container.
+app.directive('processorUiDate', function () {
+  return {
+    restrict: 'E',
+    template: template,
+    controller : function ($scope, debounce) {
+      const processor = $scope.processor;
+      const pipeline = $scope.pipeline;
+
+      function consumeNewInputObject() {
+        $scope.fields = keysDeep(processor.inputObject);
+        refreshFieldData();
+      }
+
+      function refreshFieldData() {
+        $scope.fieldData = _.get(processor.inputObject, processor.sourceField);
+      }
+
+      function processorUiChanged() {
+        pipeline.setDirty();
+      }
+
+      function updateFormats() {
+        const selectedFormats = $scope.formats.map((o) => {
+          if (!o.selected) return;
+          return o.title;
+        });
+        processor.formats = _.compact(selectedFormats);
+
+        $scope.customFormatSelected = !_.isUndefined(_.find(processor.formats, (o) => {
+          return o === 'Custom';
+        }));
+        processorUiChanged();
+      }
+      updateFormats = debounce(updateFormats, 200);
+
+      $scope.formats = selectableArray(['ISO8601', 'UNIX', 'UNIX_MS', 'TAI64N', 'Custom'], processor.formats);
+      $scope.updateFormats = updateFormats;
+
+      $scope.$watch('processor.inputObject', consumeNewInputObject);
+
+      $scope.$watch('processor.sourceField', () => {
+        refreshFieldData();
+        processorUiChanged();
+      });
+
+      $scope.$watch('processor.customFormat', updateFormats);
+      $scope.$watch('processor.targetField', processorUiChanged);
+      $scope.$watch('processor.timezone', processorUiChanged);
+      $scope.$watch('processor.locale', processorUiChanged);
+    }
+  };
+});

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/processor_types.js

@@ -70,6 +70,37 @@ export class Convert extends Processor {
   }
 };
 
+export class Date extends Processor {
+  constructor(processorId) {
+    super(processorId, 'date', 'Date');
+    this.sourceField = '';
+    this.targetField = '@timestamp';
+    this.formats = [];
+    this.timezone = 'Etc/UTC';
+    this.locale = 'ENGLISH';
+    this.customFormat = '';
+  }
+
+  get description() {
+    const source = this.sourceField || '?';
+    const target = this.targetField || '?';
+    return `[${source}] -> [${target}]`;
+  }
+
+  get model() {
+    return {
+      processorId: this.processorId,
+      typeId: this.typeId,
+      sourceField: this.sourceField || '',
+      targetField: this.targetField || '',
+      formats: this.formats || [],
+      timezone: this.timezone || '',
+      locale: this.locale || '',
+      customFormat: this.customFormat || ''
+    };
+  }
+};
+
 export class Gsub extends Processor {
   constructor(processorId) {
     super(processorId, 'gsub', 'Gsub');

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/lib/selectable_array.js

@@ -0,0 +1,14 @@
+import _ from 'lodash';
+
+export default function selectableArray(array, selectedValues) {
+  return array.map((item) => {
+    const selected = _.find(selectedValues, (selectedValue) => {
+      return selectedValue.toUpperCase() === item.toUpperCase();
+    });
+
+    return {
+      title: item,
+      selected: !_.isUndefined(selected)
+    };
+  });
+};

src/plugins/kibana/public/settings/sections/indices/add_data_steps/pipeline_setup/views/processor_ui_date.html

@@ -0,0 +1,74 @@
+<div class="form-group">
+  <label>Field:</label>
+  <select
+    class="form-control"
+    ng-options="field as field for field in fields"
+    ng-model="processor.sourceField">
+  </select>
+</div>
+<div class="form-group">
+  <label>Field Data:</label>
+  <pre>{{ fieldData }}</pre>
+</div>
+<div class="form-group">
+  <label>Target Field:</label>
+  <input type="text" class="form-control" ng-model="processor.targetField">
+</div>
+<div class="form-group">
+  <label>Formats:</label>
+  <div ng-repeat="format in formats">
+    <input
+      type="checkbox"
+      id="format_{{processor.processorId}}_{{$index}}"
+      ng-model="format.selected"
+      ng-click="updateFormats()" />
+    <label for="format_{{processor.processorId}}_{{$index}}">
+      {{format.title}}
+      <a
+        aria-label="Custom Date Format Help"
+        tooltip="Custom Date Format Help"
+        tooltip-append-to-body="true"
+        href="http://www.joda.org/joda-time/key_format.html"
+        target="_blank"
+        ng-show="format.title === 'Custom'">
+        <i aria-hidden="true" class="fa fa-question-circle"></i>
+      </a>
+    </label>
+  </div>
+  <div
+    class="custom-date-format"
+    ng-show="customFormatSelected">
+    <input
+      type="text"
+      class="form-control"
+      ng-model="processor.customFormat">
+  </div>
+</div>
+<div class="form-group">
+  <label>
+    Timezone:
+    <a
+      aria-label="Timezone Help"
+      tooltip="Timezone Help"
+      tooltip-append-to-body="true"
+      href="http://joda-time.sourceforge.net/timezones.html"
+      target="_blank">
+      <i aria-hidden="true" class="fa fa-question-circle"></i>
+    </a>
+  </label>
+  <input type="text" class="form-control" ng-model="processor.timezone"></div>
+</div>
+<div class="form-group">
+  <label>
+    Locale:
+    <a
+      aria-label="Locale Help"
+      tooltip="Locale Help"
+      tooltip-append-to-body="true"
+      href="https://www.elastic.co/guide/index.html"
+      target="_blank">
+      <i aria-hidden="true" class="fa fa-question-circle"></i>
+    </a>
+  </label>
+  <input type="text" class="form-control" ng-model="processor.locale"></div>
+</div>

src/plugins/kibana/server/lib/converters/ingest_processor_api_kibana_to_es_converters.js

@@ -27,6 +27,30 @@ export function convert(processorApiDocument) {
   };
 }
 
+export function date(processorApiDocument) {
+  const formats = [];
+  processorApiDocument.formats.forEach((format) => {
+    if (format.toUpperCase() === 'CUSTOM') {
+      if (processorApiDocument.custom_format) {
+        formats.push(processorApiDocument.custom_format);
+      }
+    } else {
+      formats.push(format);
+    }
+  });
+
+  return {
+    date: {
+      tag: processorApiDocument.processor_id,
+      match_field: processorApiDocument.source_field,
+      target_field: processorApiDocument.target_field,
+      match_formats: formats,
+      timezone: processorApiDocument.timezone,
+      locale: processorApiDocument.locale
+    }
+  };
+}
+
 export function gsub(processorApiDocument) {
   return {
     gsub: {

src/plugins/kibana/server/lib/schemas/resources/ingest_processor_schemas.js

@@ -17,6 +17,16 @@ export const convert = base.keys({
   type: Joi.string()
 });
 
+export const date = base.keys({
+  type_id: Joi.string().only('date').required(),
+  source_field: Joi.string().allow(''),
+  target_field: Joi.string().allow(''),
+  formats: Joi.array().items(Joi.string().allow('')),
+  timezone: Joi.string().allow(''),
+  locale: Joi.string().allow(''),
+  custom_format: Joi.string().allow('')
+});
+
 export const gsub = base.keys({
   type_id: Joi.string().only('gsub').required(),
   source_field: Joi.string().allow(''),

test/unit/api/ingest/processors/_date.js

@@ -0,0 +1,79 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+  const testPipeline = {
+    processors: [{
+      processor_id: 'processor1',
+      type_id: 'date',
+      source_field: 'dob',
+      target_field: 'dob',
+      formats: ['Custom'],
+      timezone: 'Etc/UTC',
+      locale: 'ENGLISH',
+      custom_format: 'MM/dd/yyyy'
+    }],
+    input: { dob: '07/05/1979' }
+  };
+
+  return function (bdd, scenarioManager, request) {
+    bdd.describe('simulate - date processor', () => {
+
+      bdd.it('should return 400 for an invalid payload', () => {
+        return Promise.all([
+          // Date processor requires source_field property
+          request.post('/kibana/ingest/simulate')
+          .send({
+            input: { dob: '07/05/1979' },
+            processors: [{
+              processor_id: 'processor1',
+              type_id: 'convert',
+              source_field: 42,
+              target_field: 'dob',
+              formats: 'Custom',
+              timezone: 'Etc/UTC',
+              locale: 'ENGLISH',
+              custom_format: 'MM/dd/yyyy'
+            }]
+          })
+          .expect(400)
+        ]);
+      });
+
+      bdd.it('should return 200 for a valid simulate request', () => {
+        return request.post('/kibana/ingest/simulate')
+          .send(testPipeline)
+          .expect(200);
+      });
+
+      bdd.it('should return a simulated output with the correct result for the given processor', function () {
+        return request.post('/kibana/ingest/simulate')
+          .send(testPipeline)
+          .expect(200)
+          .then(function (response) {
+            expect(response.body[0].output.dob).to.be('1979-07-05T00:00:00.000Z');
+          });
+      });
+
+      bdd.it('should enforce snake case', () => {
+        return request.post('/kibana/ingest/simulate')
+        .send({
+          processors: [{
+            processorId: 'processor1',
+            typeId: 'date',
+            sourceField: 'dob',
+            targetField: 'dob',
+            formats: ['Custom'],
+            timezone: 'Etc/UTC',
+            locale: 'ENGLISH',
+            customFormat: 'MM/dd/yyyy'
+          }],
+          input: { dob: '07/05/1979' }
+        })
+        .expect(400);
+      });
+
+    });
+  };
+});

test/unit/api/ingest/processors/index.js

@@ -1,15 +1,16 @@
 define(function (require) {
-  var convert = require('./_convert');
-  var set = require('./_set');
-  var gsub = require('./_gsub');
   var append = require('./_append');
+  var convert = require('./_convert');
+  var date = require('./_date');
+  var gsub = require('./_gsub');
+  var set = require('./_set');
 
   return function processors(bdd, scenarioManager, request) {
-    convert(bdd, scenarioManager, request);
-    set(bdd, scenarioManager, request);
-    gsub(bdd, scenarioManager, request);
     append(bdd, scenarioManager, request);
+    convert(bdd, scenarioManager, request);
+    date(bdd, scenarioManager, request);
+    gsub(bdd, scenarioManager, request);
+    set(bdd, scenarioManager, request);
   };
 
 });
-