Merge pull request #5213 from Bargs/indexPatternApi

Create an Ingest API

package.json

@@ -180,7 +180,8 @@
     "portscanner": "1.0.0",
     "simple-git": "1.8.0",
     "sinon": "1.17.2",
-    "source-map": "0.4.4"
+    "source-map": "0.4.4",
+    "supertest-as-promised": "2.0.2"
   },
   "engines": {
     "node": "4.2.4",

src/plugins/kibana/index.js

@@ -1,3 +1,5 @@
+const ingest = require('./server/routes/api/ingest');
+
 module.exports = function (kibana) {
   return new kibana.Plugin({
 
@@ -43,6 +45,10 @@ module.exports = function (kibana) {
           };
         }
       }
+    },
+
+    init: function (server, options) {
+      ingest(server);
     }
   });
 

src/plugins/kibana/server/lib/__tests__/case_conversion.js

@@ -0,0 +1,35 @@
+const expect = require('expect.js');
+const { keysToSnakeCaseShallow, keysToCamelCaseShallow } = require('../case_conversion');
+const _ = require('lodash');
+
+describe('keysToSnakeCaseShallow', function () {
+
+  it('should convert all of an object\'s keys to snake case', function () {
+    const result = keysToSnakeCaseShallow({
+      camelCase: 'camel_case',
+      'kebab-case': 'kebab_case',
+      snake_case: 'snake_case'
+    });
+
+    _.forEach(result, function (value, key) {
+      expect(key).to.be(value);
+    });
+  });
+
+});
+
+describe('keysToCamelCaseShallow', function () {
+
+  it('should convert all of an object\'s keys to camel case', function () {
+    const result = keysToCamelCaseShallow({
+      camelCase: 'camelCase',
+      'kebab-case': 'kebabCase',
+      snake_case: 'snakeCase'
+    });
+
+    _.forEach(result, function (value, key) {
+      expect(key).to.be(value);
+    });
+  });
+
+});

src/plugins/kibana/server/lib/__tests__/convert_pattern_and_template_name.js

@@ -0,0 +1,31 @@
+const {templateToPattern, patternToTemplate} = require('../convert_pattern_and_template_name');
+const expect = require('expect.js');
+
+describe('convertPatternAndTemplateName', function () {
+
+  describe('templateToPattern', function () {
+
+    it('should convert an index template\'s name to its matching index pattern\'s title', function () {
+      expect(templateToPattern('kibana-logstash-*')).to.be('logstash-*');
+    });
+
+    it('should throw an error if the template name isn\'t a valid kibana namespaced name', function () {
+      expect(templateToPattern).withArgs('logstash-*').to.throwException('not a valid kibana namespaced template name');
+      expect(templateToPattern).withArgs('').to.throwException(/not a valid kibana namespaced template name/);
+    });
+
+  });
+
+  describe('patternToTemplate', function () {
+
+    it('should convert an index pattern\'s title to its matching index template\'s name', function () {
+      expect(patternToTemplate('logstash-*')).to.be('kibana-logstash-*');
+    });
+
+    it('should throw an error if the pattern is empty', function () {
+      expect(patternToTemplate).withArgs('').to.throwException(/pattern must not be empty/);
+    });
+
+  });
+
+});

src/plugins/kibana/server/lib/__tests__/create_mappings_from_pattern_fields.js

@@ -0,0 +1,82 @@
+const createMappingsFromPatternFields = require('../create_mappings_from_pattern_fields');
+const expect = require('expect.js');
+const _ = require('lodash');
+
+let testFields;
+
+describe('createMappingsFromPatternFields', function () {
+
+  beforeEach(function () {
+    testFields = [
+      {
+        'name': 'ip',
+        'type': 'ip'
+      },
+      {
+        'name': 'agent',
+        'type': 'string'
+      },
+      {
+        'name': 'bytes',
+        'type': 'number'
+      }
+    ];
+  });
+
+  it('should throw an error if the argument is empty', function () {
+    expect(createMappingsFromPatternFields).to.throwException(/argument must not be empty/);
+  });
+
+  it('should not modify the original argument', function () {
+    const testFieldClone = _.cloneDeep(testFields);
+    const mappings = createMappingsFromPatternFields(testFields);
+
+    expect(mappings.ip).to.not.be(testFields[0]);
+    expect(_.isEqual(testFields, testFieldClone)).to.be.ok();
+  });
+
+  it('should set the same default mapping for all non-strings', function () {
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    _.forEach(mappings, function (mapping) {
+      if (mapping.type !== 'string') {
+        expect(_.isEqual(mapping, {
+          type: mapping.type,
+          index: 'not_analyzed',
+          doc_values: true
+        })).to.be.ok();
+      }
+    });
+  });
+
+  it('should give strings a multi-field mapping', function () {
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    _.forEach(mappings, function (mapping) {
+      if (mapping.type === 'string') {
+        expect(mapping).to.have.property('fields');
+      }
+    });
+  });
+
+  it('should handle nested fields', function () {
+    testFields.push({name: 'geo.coordinates', type: 'geo_point'});
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    expect(mappings).to.have.property('geo');
+    expect(mappings.geo).to.have.property('properties');
+    expect(mappings.geo.properties).to.have.property('coordinates');
+    expect(_.isEqual(mappings.geo.properties.coordinates, {
+      type: 'geo_point',
+      index: 'not_analyzed',
+      doc_values: true
+    })).to.be.ok();
+  });
+
+  it('should map all number fields as an ES double', function () {
+    let mappings = createMappingsFromPatternFields(testFields);
+
+    expect(mappings).to.have.property('bytes');
+    expect(mappings.bytes).to.have.property('type', 'double');
+  });
+});

src/plugins/kibana/server/lib/__tests__/handle_es_error.js

@@ -0,0 +1,41 @@
+var expect = require('expect.js');
+var Boom = require('boom');
+var esErrors = require('elasticsearch').errors;
+var handleESError = require('../handle_es_error');
+
+describe('handleESError', function () {
+
+  it('should transform elasticsearch errors into boom errors with the same status code', function () {
+    var conflict = handleESError(new esErrors.Conflict());
+    expect(conflict.isBoom).to.be(true);
+    expect(conflict.output.statusCode).to.be(409);
+
+    var forbidden = handleESError(new esErrors[403]);
+    expect(forbidden.isBoom).to.be(true);
+    expect(forbidden.output.statusCode).to.be(403);
+
+    var notFound = handleESError(new esErrors.NotFound());
+    expect(notFound.isBoom).to.be(true);
+    expect(notFound.output.statusCode).to.be(404);
+
+    var badRequest = handleESError(new esErrors.BadRequest());
+    expect(badRequest.isBoom).to.be(true);
+    expect(badRequest.output.statusCode).to.be(400);
+  });
+
+  it('should return an unknown error without transforming it', function () {
+    var unknown = new Error('mystery error');
+    expect(handleESError(unknown)).to.be(unknown);
+  });
+
+  it('should return a boom 503 server timeout error for ES connection errors', function () {
+    expect(handleESError(new esErrors.ConnectionFault()).output.statusCode).to.be(503);
+    expect(handleESError(new esErrors.ServiceUnavailable()).output.statusCode).to.be(503);
+    expect(handleESError(new esErrors.NoConnections()).output.statusCode).to.be(503);
+    expect(handleESError(new esErrors.RequestTimeout()).output.statusCode).to.be(503);
+  });
+
+  it('should throw an error if called with a non-error argument', function () {
+    expect(handleESError).withArgs('notAnError').to.throwException();
+  });
+});

src/plugins/kibana/server/lib/__tests__/init_default_field_props.js

@@ -0,0 +1,74 @@
+const initDefaultFieldProps = require('../init_default_field_props');
+const expect = require('expect.js');
+const _ = require('lodash');
+let fields;
+
+const testData = [
+  {
+    'name': 'ip',
+    'type': 'ip'
+  }, {
+    'name': '@timestamp',
+    'type': 'date'
+  }, {
+    'name': 'agent',
+    'type': 'string'
+  }, {
+    'name': 'bytes',
+    'type': 'number'
+  },
+  {
+    'name': 'geo.coordinates',
+    'type': 'geo_point'
+  }
+];
+
+describe('initDefaultFieldProps', function () {
+
+  beforeEach(function () {
+    fields = _.cloneDeep(testData);
+  });
+
+  it('should throw an error if no argument is passed or the argument is not an array', function () {
+    expect(initDefaultFieldProps).to.throwException(/requires an array argument/);
+    expect(initDefaultFieldProps).withArgs({}).to.throwException(/requires an array argument/);
+  });
+
+  it('should set the same defaults for everything but strings', function () {
+    const results = initDefaultFieldProps(fields);
+    _.forEach(results, function (field) {
+      if (field.type !== 'string') {
+        expect(field).to.have.property('indexed', true);
+        expect(field).to.have.property('analyzed', false);
+        expect(field).to.have.property('doc_values', true);
+        expect(field).to.have.property('scripted', false);
+        expect(field).to.have.property('count', 0);
+      }
+    });
+  });
+
+  it('should make string fields analyzed', function () {
+    const results = initDefaultFieldProps(fields);
+    _.forEach(results, function (field) {
+      if (field.type === 'string' && !_.contains(field.name, 'raw')) {
+        expect(field).to.have.property('indexed', true);
+        expect(field).to.have.property('analyzed', true);
+        expect(field).to.have.property('doc_values', false);
+        expect(field).to.have.property('scripted', false);
+        expect(field).to.have.property('count', 0);
+      }
+    });
+  });
+
+  it('should create an extra raw non-analyzed field for strings', function () {
+    const results = initDefaultFieldProps(fields);
+    const rawField = _.find(results, function (field) {
+      return _.contains(field.name, 'raw');
+    });
+    expect(rawField).to.have.property('indexed', true);
+    expect(rawField).to.have.property('analyzed', false);
+    expect(rawField).to.have.property('doc_values', true);
+    expect(rawField).to.have.property('scripted', false);
+    expect(rawField).to.have.property('count', 0);
+  });
+});

src/plugins/kibana/server/lib/case_conversion.js

@@ -0,0 +1,15 @@
+const _ = require('lodash');
+
+module.exports = {
+  keysToSnakeCaseShallow: function (object) {
+    return _.mapKeys(object, (value, key) => {
+      return _.snakeCase(key);
+    });
+  },
+
+  keysToCamelCaseShallow: function (object) {
+    return _.mapKeys(object, (value, key) => {
+      return _.camelCase(key);
+    });
+  }
+};

src/plugins/kibana/server/lib/convert_pattern_and_template_name.js

@@ -0,0 +1,22 @@
+// To avoid index template naming collisions the index pattern creation API
+// namespaces template names by prepending 'kibana-' to the matching pattern's title.
+// e.g. a pattern with title `logstash-*` will have a matching template named `kibana-logstash-*`.
+// This module provides utility functions for easily converting between template and pattern names.
+
+module.exports = {
+  templateToPattern: (templateName) => {
+    if (templateName.indexOf('kibana-') === -1) {
+      throw new Error('not a valid kibana namespaced template name');
+    }
+
+    return templateName.slice(templateName.indexOf('-') + 1);
+  },
+
+  patternToTemplate: (patternName) => {
+    if (patternName === '') {
+      throw new Error('pattern must not be empty');
+    }
+
+    return `kibana-${patternName.toLowerCase()}`;
+  }
+};

src/plugins/kibana/server/lib/create_mappings_from_pattern_fields.js

@@ -0,0 +1,38 @@
+const _ = require('lodash');
+
+// Creates an ES field mapping from a single field object in a kibana index pattern
+module.exports = function createMappingsFromPatternFields(fields) {
+  if (_.isEmpty(fields)) {
+    throw new Error('argument must not be empty');
+  }
+
+  const mappings = {};
+
+  _.forEach(fields, function (field) {
+    let mapping;
+
+    if (field.type === 'string') {
+      mapping = {
+        type: 'string',
+        index: 'analyzed',
+        omit_norms: true,
+        fielddata: {format: 'disabled'},
+        fields: {
+          raw: {type: 'string', index: 'not_analyzed', doc_values: true, ignore_above: 256}
+        }
+      };
+    }
+    else {
+      const fieldType = field.type === 'number' ? 'double' : field.type;
+      mapping = {
+        type: fieldType,
+        index: 'not_analyzed',
+        doc_values: true
+      };
+    }
+
+    _.set(mappings, field.name.split('.').join('.properties.'), mapping);
+  });
+
+  return mappings;
+};

src/plugins/kibana/server/lib/handle_es_error.js

@@ -0,0 +1,26 @@
+const Boom = require('boom');
+const esErrors = require('elasticsearch').errors;
+const _ = require('lodash');
+
+module.exports = function handleESError(error) {
+  if (!(error instanceof Error)) {
+    throw new Error('Expected an instance of Error');
+  }
+
+  if (error instanceof esErrors.ConnectionFault ||
+    error instanceof esErrors.ServiceUnavailable ||
+    error instanceof esErrors.NoConnections ||
+    error instanceof esErrors.RequestTimeout) {
+    return Boom.serverTimeout(error);
+  } else if (error instanceof esErrors.Conflict || _.contains(error.message, 'index_template_already_exists')) {
+    return Boom.conflict(error);
+  } else if (error instanceof esErrors[403]) {
+    return Boom.forbidden(error);
+  } else if (error instanceof esErrors.NotFound) {
+    return Boom.notFound(error);
+  } else if (error instanceof esErrors.BadRequest || error instanceof TypeError) {
+    return Boom.badRequest(error);
+  } else {
+    return error;
+  }
+};

src/plugins/kibana/server/lib/init_default_field_props.js

@@ -0,0 +1,45 @@
+const _ = require('lodash');
+
+module.exports = function initDefaultFieldProps(fields) {
+  if (fields === undefined || !_.isArray(fields)) {
+    throw new Error('requires an array argument');
+  }
+
+  const results = [];
+
+  _.forEach(fields, function (field) {
+    const newField = _.cloneDeep(field);
+    results.push(newField);
+
+    if (newField.type === 'string') {
+      _.defaults(newField, {
+        indexed: true,
+        analyzed: true,
+        doc_values: false,
+        scripted: false,
+        count: 0
+      });
+
+      results.push({
+        name: newField.name + '.raw',
+        type: 'string',
+        indexed: true,
+        analyzed: false,
+        doc_values: true,
+        scripted: false,
+        count: 0
+      });
+    }
+    else {
+      _.defaults(newField, {
+        indexed: true,
+        analyzed: false,
+        doc_values: true,
+        scripted: false,
+        count: 0
+      });
+    }
+  });
+
+  return results;
+};

src/plugins/kibana/server/lib/schemas/resources/index_pattern_schema.js

@@ -0,0 +1,23 @@
+const Joi = require('joi');
+
+module.exports = Joi.object({
+  id: Joi.string().required(),
+  title: Joi.string().required(),
+  time_field_name: Joi.string(),
+  interval_name: Joi.string(),
+  not_expandable: Joi.boolean(),
+  fields: Joi.array().items(
+    Joi.object({
+      name: Joi.string().required(),
+      type: Joi.string().required(),
+      count: Joi.number().integer(),
+      scripted: Joi.boolean(),
+      doc_values: Joi.boolean(),
+      analyzed: Joi.boolean(),
+      indexed: Joi.boolean(),
+      script: Joi.string(),
+      lang: Joi.string()
+    })
+  ).required().min(1),
+  field_format_map: Joi.object()
+});

src/plugins/kibana/server/routes/api/ingest/index.js

@@ -0,0 +1,4 @@
+export default function (server) {
+  require('./register_post')(server);
+  require('./register_delete')(server);
+}

src/plugins/kibana/server/routes/api/ingest/register_delete.js

@@ -0,0 +1,31 @@
+const Promise = require('bluebird');
+const handleESError = require('../../../lib/handle_es_error');
+const {templateToPattern, patternToTemplate} = require('../../../lib/convert_pattern_and_template_name');
+
+module.exports = function registerDelete(server) {
+  server.route({
+    path: '/api/kibana/ingest/{id}',
+    method: 'DELETE',
+    handler: function (req, reply) {
+      const callWithRequest = server.plugins.elasticsearch.callWithRequest;
+      const deletePatternParams = {
+        index: '.kibana',
+        type: 'index-pattern',
+        id: req.params.id
+      };
+
+      Promise.all([
+        callWithRequest(req, 'delete', deletePatternParams),
+        callWithRequest(req, 'indices.deleteTemplate', {name: patternToTemplate(req.params.id), ignore: [404]})
+      ])
+      .then(
+        function (pattern) {
+          reply({success: true});
+        },
+        function (error) {
+          reply(handleESError(error));
+        }
+      );
+    }
+  });
+};

src/plugins/kibana/server/routes/api/ingest/register_post.js

@@ -0,0 +1,108 @@
+const Boom = require('boom');
+const _ = require('lodash');
+const {templateToPattern, patternToTemplate} = require('../../../lib/convert_pattern_and_template_name');
+const indexPatternSchema = require('../../../lib/schemas/resources/index_pattern_schema');
+const handleESError = require('../../../lib/handle_es_error');
+const { keysToCamelCaseShallow } = require('../../../lib/case_conversion');
+const createMappingsFromPatternFields = require('../../../lib/create_mappings_from_pattern_fields');
+const initDefaultFieldProps = require('../../../lib/init_default_field_props');
+
+module.exports = function registerPost(server) {
+  server.route({
+    path: '/api/kibana/ingest',
+    method: 'POST',
+    config: {
+      validate: {
+        payload: indexPatternSchema
+      }
+    },
+    handler: function (req, reply) {
+      const callWithRequest = server.plugins.elasticsearch.callWithRequest;
+      const requestDocument = _.cloneDeep(req.payload);
+      const indexPatternId = requestDocument.id;
+      const indexPattern = keysToCamelCaseShallow(requestDocument);
+      delete indexPattern.id;
+
+      const mappings = createMappingsFromPatternFields(indexPattern.fields);
+      indexPattern.fields = initDefaultFieldProps(indexPattern.fields);
+
+      indexPattern.fields = JSON.stringify(indexPattern.fields);
+      indexPattern.fieldFormatMap = JSON.stringify(indexPattern.fieldFormatMap);
+
+      return callWithRequest(req, 'indices.exists', {index: indexPatternId})
+      .then((matchingIndices) => {
+        if (matchingIndices) {
+          throw Boom.conflict('Cannot create an index pattern via this API if existing indices already match the pattern');
+        }
+
+        const patternCreateParams = {
+          index: '.kibana',
+          type: 'index-pattern',
+          id: indexPatternId,
+          body: indexPattern
+        };
+
+        return callWithRequest(req, 'create', patternCreateParams)
+        .then((patternResponse) => {
+          const templateParams = {
+            order: 0,
+            create: true,
+            name: patternToTemplate(indexPatternId),
+            body: {
+              template: indexPatternId,
+              mappings: {
+                _default_: {
+                  dynamic_templates: [{
+                    string_fields: {
+                      match: '*',
+                      match_mapping_type: 'string',
+                      mapping: {
+                        type: 'string',
+                        index: 'analyzed',
+                        omit_norms: true,
+                        fielddata: {format: 'disabled'},
+                        fields: {
+                          raw: {type: 'string', index: 'not_analyzed', doc_values: true, ignore_above: 256}
+                        }
+                      }
+                    }
+                  }],
+                  properties: mappings
+                }
+              }
+            }
+          };
+
+          return callWithRequest(req, 'indices.putTemplate', templateParams)
+          .catch((templateError) => {
+            const deleteParams = {
+              index: '.kibana',
+              type: 'index-pattern',
+              id: indexPatternId
+            };
+
+            return callWithRequest(req, 'delete', deleteParams)
+            .then(() => {
+              throw templateError;
+            }, (patternDeletionError) => {
+              throw new Error(
+                `index-pattern ${indexPatternId} created successfully but index template
+                creation failed. Failed to rollback index-pattern creation, must delete manually.
+                ${patternDeletionError.toString()}
+                ${templateError.toString()}`
+              );
+            });
+          });
+        });
+      })
+      .then(
+        function () {
+          reply().code(204);
+        },
+        function (error) {
+          reply(handleESError(error));
+        }
+      );
+    }
+  });
+};

tasks/config/intern.js

@@ -8,6 +8,12 @@ module.exports = function (grunt) {
       config: 'test/intern',
       reporters: ['Console']
     },
-    dev: {}
+    dev: {},
+    api: {
+      options: {
+        runType: 'client',
+        config: 'test/apiIntern'
+      }
+    }
   };
 };

tasks/config/run.js

@@ -24,6 +24,23 @@ module.exports = function (grunt) {
       ]
     },
 
+    apiTestServer: {
+      options: {
+        wait: false,
+        ready: /Server running/,
+        quiet: false,
+        failOnError: false
+      },
+      cmd: binScript,
+      args: [
+        '--server.port=' + uiConfig.servers.kibana.port,
+        '--server.xsrf.disableProtection=true',
+        '--optimize.enabled=false',
+        '--elasticsearch.url=' + format(uiConfig.servers.elasticsearch),
+        '--logging.json=false'
+      ]
+    },
+
     testUIServer: {
       options: {
         wait: false,
@@ -31,7 +48,7 @@ module.exports = function (grunt) {
         quiet: false,
         failOnError: false
       },
-      cmd: /^win/.test(platform) ? '.\\bin\\kibana.bat' : './bin/kibana',
+      cmd: binScript,
       args: [
         '--server.port=' + uiConfig.servers.kibana.port,
         '--env.name=development',

tasks/test.js

@@ -7,7 +7,8 @@ module.exports = function (grunt) {
   grunt.registerTask('test:quick', [
     'test:server',
     'test:ui',
-    'test:browser'
+    'test:browser',
+    'test:api'
   ]);
 
   grunt.registerTask('test:dev', [
@@ -37,6 +38,23 @@ module.exports = function (grunt) {
     'intern:dev'
   ]);
 
+  grunt.registerTask('test:api', [
+    'esvm:ui',
+    'run:apiTestServer',
+    'intern:api',
+    'esvm_shutdown:ui',
+    'stop:apiTestServer'
+  ]);
+
+  grunt.registerTask('test:api:server', [
+    'esvm:ui',
+    'run:apiTestServer:keepalive'
+  ]);
+
+  grunt.registerTask('test:api:runner', [
+    'intern:api'
+  ]);
+
   grunt.registerTask('test', function (subTask) {
     if (subTask) grunt.fail.fatal(`invalid task "test:${subTask}"`);
 

test/apiIntern.js

@@ -0,0 +1,12 @@
+define({
+  suites: [
+    'test/unit/api/ingest/index'
+  ],
+  excludeInstrumentation: /(fixtures|node_modules)\//,
+  loaderOptions: {
+    paths: {
+      'bluebird': './node_modules/bluebird/js/browser/bluebird.js',
+      'moment': './node_modules/moment/moment.js'
+    }
+  }
+});

test/unit/api/ingest/_del.js

@@ -0,0 +1,54 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var createTestData = require('intern/dojo/node!../../../unit/api/ingest/data');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+
+  return function (bdd, scenarioManager, request) {
+
+    bdd.describe('DELETE ingest', function deleteIngestConfig() {
+
+      bdd.beforeEach(function () {
+        return scenarioManager.reload('emptyKibana')
+        .then(function () {
+          return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204);
+        });
+      });
+
+      bdd.afterEach(function () {
+        return request.del('/kibana/ingest/logstash-*')
+        .then(function () {
+          return scenarioManager.client.indices.deleteTemplate({name: 'kibana-logstash-*'})
+          .catch(function (err) {
+            if (err.status !== 404) {
+              throw err;
+            }
+          });
+        });
+      });
+
+      bdd.it('should return 200 for successful deletion of pattern and template', function () {
+        return request.del('/kibana/ingest/logstash-*')
+          .expect(200)
+          .then(function () {
+            return request.get('/kibana/ingest/logstash-*').expect(404);
+          })
+          .then(function () {
+            return scenarioManager.client.indices.getTemplate({name: 'kibana-logstash-*'})
+              .catch(function (error) {
+                expect(error.status).to.be(404);
+              });
+          });
+      });
+
+      bdd.it('should return 404 for a non-existent id', function () {
+        return request.del('/kibana/ingest/doesnotexist')
+          .expect(404);
+      });
+
+    });
+  };
+});

test/unit/api/ingest/_post.js

@@ -0,0 +1,169 @@
+define(function (require) {
+  var Promise = require('bluebird');
+  var createTestData = require('intern/dojo/node!../../../unit/api/ingest/data');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+
+  return function (bdd, scenarioManager, request) {
+    bdd.describe('POST ingest', function postIngest() {
+
+      bdd.beforeEach(function () {
+        return scenarioManager.reload('emptyKibana');
+      });
+
+      bdd.afterEach(function () {
+        return request.del('/kibana/ingest/logstash-*');
+      });
+
+      bdd.it('should return 400 for an invalid payload', function invalidPayload() {
+        return Promise.all([
+          request.post('/kibana/ingest').expect(400),
+
+          request.post('/kibana/ingest')
+            .send({})
+            .expect(400),
+
+          request.post('/kibana/ingest')
+            .send(_.set(createTestData(), 'title', false))
+            .expect(400),
+
+          request.post('/kibana/ingest')
+            .send(_.set(createTestData(), 'fields', {}))
+            .expect(400),
+
+          request.post('/kibana/ingest')
+            .send(_.set(createTestData(), 'fields', []))
+            .expect(400),
+
+          // Fields must have a name and type
+          request.post('/kibana/ingest')
+            .send(_.set(createTestData(), 'fields', [{count: 0}]))
+            .expect(400)
+        ]);
+      });
+
+      bdd.it('should return 204 when an ingest config is successfully created', function createIngestConfig() {
+        return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204);
+      });
+
+      bdd.it('should create an index template if a fields array is included', function createTemplate() {
+        return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204)
+          .then(function () {
+            return scenarioManager.client.indices.getTemplate({name: 'kibana-logstash-*'});
+          });
+      });
+
+      bdd.it('should provide defaults for field properties', function createTemplate() {
+        return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204)
+          .then(function () {
+            return scenarioManager.client.get({
+              index: '.kibana',
+              type: 'index-pattern',
+              id: 'logstash-*'
+            })
+            .then(function (res) {
+              var fields = JSON.parse(res._source.fields);
+              // @timestamp was created with only name and type, all other fields should be set as defaults by API
+              expect(res._source.title).to.be('logstash-*');
+              expect(fields[1].name).to.be('@timestamp');
+              expect(fields[1].type).to.be('date');
+              expect(fields[1].count).to.be(0);
+              expect(fields[1].scripted).to.be(false);
+              expect(fields[1].indexed).to.be(true);
+              expect(fields[1].analyzed).to.be(false);
+              expect(fields[1].doc_values).to.be(true);
+            });
+          });
+      });
+
+      bdd.it('should create index template with _default_ mappings based on the info in the ingest config',
+      function createTemplate() {
+        return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204)
+          .then(function () {
+            return scenarioManager.client.indices.getTemplate({name: 'kibana-logstash-*'})
+            .then(function (template) {
+              var mappings = template['kibana-logstash-*'].mappings._default_.properties;
+              expect(mappings).to.be.ok();
+              expect(_.isEqual(mappings.ip, {index: 'not_analyzed', type: 'ip', doc_values: true})).to.be.ok();
+              expect(_.isEqual(mappings['@timestamp'], {index: 'not_analyzed', type: 'date', doc_values: true})).to.be.ok();
+              expect(_.isEqual(mappings.bytes, {index: 'not_analyzed', type: 'double', doc_values: true})).to.be.ok();
+
+              // object fields are mapped as such, with individual mappings for each of their properties
+              expect(_.isEqual(mappings.geo, {
+                properties: {
+                  coordinates: {
+                    index: 'not_analyzed',
+                    type: 'geo_point',
+                    doc_values: true
+                  }
+                }
+              })).to.be.ok();
+
+              // strings should be mapped as multi fields
+              expect(mappings.agent).to.have.property('fields');
+            });
+          });
+      });
+
+      bdd.it('should return 409 conflict when a pattern with the given ID already exists', function patternConflict() {
+        return request.post('/kibana/ingest')
+          .send(createTestData())
+          .expect(204)
+          .then(function () {
+            return request.post('/kibana/ingest')
+              .send(createTestData())
+              .expect(409);
+          });
+      });
+
+
+      bdd.it('should return 409 conflict when an index template with the given ID already exists', function templateConflict() {
+        return scenarioManager.client.indices.putTemplate({
+          name: 'kibana-logstash-*', body: {
+            template: 'logstash-*'
+          }
+        }).then(function () {
+          return request.post('/kibana/ingest')
+            .send(createTestData())
+            .expect(409);
+        })
+        .then(function () {
+          return scenarioManager.client.indices.deleteTemplate({
+            name: 'kibana-logstash-*'
+          });
+        });
+      });
+
+      bdd.it('should return 409 conflict when the pattern matches existing indices',
+        function existingIndicesConflict() {
+          var pattern = createTestData();
+          pattern.id = pattern.title = '.kib*';
+
+          return request.post('/kibana/ingest')
+            .send(pattern)
+            .expect(409);
+        });
+
+      bdd.it('should enforce snake_case in the request body', function () {
+        var pattern = createTestData();
+        pattern = _.mapKeys(pattern, function (value, key) {
+          return _.camelCase(key);
+        });
+
+        return request.post('/kibana/ingest')
+          .send(pattern)
+          .expect(400);
+      });
+
+    });
+
+  };
+});

test/unit/api/ingest/data.js

@@ -0,0 +1,26 @@
+module.exports = function createTestData() {
+  return {
+    'id': 'logstash-*',
+    'title': 'logstash-*',
+    'time_field_name': '@timestamp',
+    'fields': [
+      {
+        'name': 'ip',
+        'type': 'ip'
+      }, {
+        'name': '@timestamp',
+        'type': 'date'
+      }, {
+        'name': 'agent',
+        'type': 'string'
+      }, {
+        'name': 'bytes',
+        'type': 'number'
+      },
+      {
+        'name': 'geo.coordinates',
+        'type': 'geo_point'
+      }
+    ]
+  };
+};

test/unit/api/ingest/index.js

@@ -0,0 +1,27 @@
+define(function (require) {
+  var bdd = require('intern!bdd');
+  var serverConfig = require('intern/dojo/node!../../../serverConfig');
+  var ScenarioManager = require('intern/dojo/node!../../../fixtures/scenarioManager');
+  var request = require('intern/dojo/node!supertest-as-promised');
+  var url = require('intern/dojo/node!url');
+  var _ = require('intern/dojo/node!lodash');
+  var expect = require('intern/dojo/node!expect.js');
+  var post = require('./_post');
+  var del = require('./_del');
+
+  bdd.describe('ingest API', function () {
+    var scenarioManager = new ScenarioManager(url.format(serverConfig.servers.elasticsearch));
+    request = request(url.format(serverConfig.servers.kibana) + '/api');
+
+    bdd.before(function () {
+      return scenarioManager.load('emptyKibana');
+    });
+
+    bdd.after(function () {
+      return scenarioManager.unload('emptyKibana');
+    });
+
+    post(bdd, scenarioManager, request);
+    del(bdd, scenarioManager, request);
+  });
+});