Support Elasticsearch date_nanos datatype (#36111)

* Add date_nanos to date type in kibana field types
* date_nanos by default is formatted by "Date Nanos" format
* Format computed date_nanos field to strict_date_time to prevent rounding
* Hide Discover - "View surrounding documents" btn for date_nanos (will be subject of another PR)
* Append number of nano seconds to formatted timeField
* Add new key dateNanosFormat to UI setting defaults

src/legacy/core_plugins/kibana/common/field_formats/types/__tests__/date_nanos.js

@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from '@kbn/expect';
+import moment from 'moment-timezone';
+import { createDateNanosFormat, analysePatternForFract, formatWithNanos } from '../date_nanos';
+import { FieldFormat } from '../../../../../../ui/field_formats/field_format';
+
+const DateFormat = createDateNanosFormat(FieldFormat);
+
+describe('Date Nanos Format', function () {
+  let convert;
+  let mockConfig;
+
+  beforeEach(function () {
+    mockConfig = {};
+    mockConfig.dateNanosFormat = 'MMMM Do YYYY, HH:mm:ss.SSSSSSSSS';
+    mockConfig['dateFormat:tz'] = 'Browser';
+    const getConfig = (key) => mockConfig[key];
+    const date = new DateFormat({}, getConfig);
+
+    convert = date.convert.bind(date);
+  });
+
+
+  it('should inject fractional seconds into formatted timestamp', function () {
+    [{
+      input: '2019-05-20T14:04:56.357001234Z',
+      pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS',
+      expected: 'May 20, 2019 @ 14:04:56.357001234',
+    }, {
+      input: '2019-05-05T14:04:56.357111234Z',
+      pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS',
+      expected: 'May 5, 2019 @ 14:04:56.357111234',
+    }, {
+      input: '2019-05-05T14:04:56.357Z',
+      pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS',
+      expected: 'May 5, 2019 @ 14:04:56.357000000',
+    }, {
+      input: '2019-05-05T14:04:56Z',
+      pattern: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS',
+      expected: 'May 5, 2019 @ 14:04:56.000000000',
+    }, {
+      input: '2019-05-05T14:04:56.201900001Z',
+      pattern: 'MMM D, YYYY @ HH:mm:ss SSSS',
+      expected: 'May 5, 2019 @ 14:04:56 2019',
+    }, {
+      input: '2019-05-05T14:04:56.201900001Z',
+      pattern: 'SSSSSSSSS',
+      expected: '201900001',
+    }].forEach(fixture => {
+      const fracPattern = analysePatternForFract(fixture.pattern);
+      const momentDate = moment(fixture.input).utc();
+      const value = formatWithNanos(momentDate, fixture.input, fracPattern);
+      expect(value).to.be(fixture.expected);
+    });
+  });
+
+  it('decoding an undefined or null date should return an empty string', function () {
+    expect(convert(null)).to.be('-');
+    expect(convert(undefined)).to.be('-');
+  });
+
+  it('should clear the memoization cache after changing the date', function () {
+    function setDefaultTimezone() {
+      moment.tz.setDefault(mockConfig['dateFormat:tz']);
+    }
+
+    const dateTime = '2019-05-05T14:04:56.201900001Z';
+
+    mockConfig['dateFormat:tz'] = 'America/Chicago';
+    setDefaultTimezone();
+    const chicagoTime = convert(dateTime);
+
+    mockConfig['dateFormat:tz'] = 'America/Phoenix';
+    setDefaultTimezone();
+    const phoenixTime = convert(dateTime);
+
+    expect(chicagoTime).not.to.equal(phoenixTime);
+  });
+
+  it('should return the value itself when it cannot successfully be formatted', function () {
+    const dateMath = 'now+1M/d';
+    expect(convert(dateMath)).to.be(dateMath);
+  });
+});

src/legacy/core_plugins/kibana/common/field_formats/types/date_nanos.js

@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import moment from 'moment';
+import _ from 'lodash';
+
+/**
+ * Analyse the given moment.js format pattern for the fractional sec part (S,SS,SSS...)
+ * returning length, match, pattern and an escaped pattern, that excludes the fractional
+ * part when formatting with moment.js -> e.g. [SSS]
+ */
+export function analysePatternForFract(pattern) {
+
+  const fracSecMatch = pattern.match('S+'); //extract fractional seconds sub-pattern
+  return {
+    length: fracSecMatch[0] ? fracSecMatch[0].length : 0,
+    patternNanos: fracSecMatch[0],
+    pattern,
+    patternEscaped: fracSecMatch[0] ? pattern.replace(fracSecMatch[0], `[${fracSecMatch[0]}]`) : '',
+  };
+}
+
+/**
+ * Format a given moment.js date object
+ * Since momentjs would loose the exact value for fractional seconds with a higher resolution than
+ * milliseconds, the fractional pattern is replaced by the fractional value of the raw timestamp
+ */
+export function formatWithNanos(dateMomentObj, valRaw, fracPatternObj) {
+
+  if (fracPatternObj.length <= 3) {
+    //S,SS,SSS is formatted correctly by moment.js
+    return dateMomentObj.format(fracPatternObj.pattern);
+
+  } else {
+    //Beyond SSS the precise value of the raw datetime string is used
+    const valFormatted = dateMomentObj.format(fracPatternObj.patternEscaped);
+    //Extract fractional value of ES formatted timestamp, zero pad if necessary:
+    //2020-05-18T20:45:05.957Z -> 957000000
+    //2020-05-18T20:45:05.957000123Z -> 957000123
+    //we do not need to take care of the year 10000 bug since max year of date_nanos is 2262
+    const valNanos = valRaw
+      .substr(20, valRaw.length - 21) //remove timezone(Z)
+      .padEnd(9, '0') //pad shorter fractionals
+      .substr(0, fracPatternObj.patternNanos.length);
+    return valFormatted.replace(fracPatternObj.patternNanos, valNanos);
+  }
+}
+
+export function createDateNanosFormat(FieldFormat) {
+  return class DateNanosFormat extends FieldFormat {
+    constructor(params, getConfig) {
+      super(params);
+
+      this.getConfig = getConfig;
+    }
+
+    getParamDefaults() {
+      return {
+        pattern: this.getConfig('dateNanosFormat'),
+        timezone: this.getConfig('dateFormat:tz'),
+      };
+    }
+
+    _convert(val) {
+      // don't give away our ref to converter so
+      // we can hot-swap when config changes
+      const pattern = this.param('pattern');
+      const timezone = this.param('timezone');
+      const fractPattern = analysePatternForFract(pattern);
+
+      const timezoneChanged = this._timeZone !== timezone;
+      const datePatternChanged = this._memoizedPattern !== pattern;
+      if (timezoneChanged || datePatternChanged) {
+        this._timeZone = timezone;
+        this._memoizedPattern = pattern;
+
+        this._memoizedConverter = _.memoize(function converter(val) {
+          if (val === null || val === undefined) {
+            return '-';
+          }
+
+          const date = moment(val);
+
+          if (date.isValid()) {
+            return formatWithNanos(date, val, fractPattern);
+          } else {
+            return val;
+          }
+        });
+      }
+
+      return this._memoizedConverter(val);
+    }
+
+    static id = 'date_nanos';
+    static title = 'Date Nanos';
+    static fieldType = 'date';
+  };
+}

src/legacy/core_plugins/kibana/public/discover/doc_table/components/table_row/details.html

@@ -21,7 +21,7 @@
             class="euiLink"
             data-test-subj="docTableRowAction"
             ng-href="{{ getContextAppHref() }}"
-            ng-if="indexPattern.isTimeBased()"
+            ng-if="indexPattern.isTimeBased() && !indexPattern.isTimeNanosBased()"
             i18n-id="kbn.docTable.tableRow.viewSurroundingDocumentsLinkText"
             i18n-default-message="View surrounding documents"
           ></a>

src/legacy/core_plugins/kibana/public/field_formats/__tests__/_conformance.js

@@ -29,6 +29,7 @@ const config = chrome.getUiSettingsClient();
 const formatIds = [
   'bytes',
   'date',
+  'date_nanos',
   'duration',
   'ip',
   'number',

src/legacy/core_plugins/kibana/public/field_formats/register.js

@@ -21,6 +21,7 @@ import { fieldFormats } from 'ui/registry/field_formats';
 import { createUrlFormat } from '../../common/field_formats/types/url';
 import { createBytesFormat } from '../../common/field_formats/types/bytes';
 import { createDateFormat } from '../../common/field_formats/types/date';
+import { createDateNanosFormat } from '../../common/field_formats/types/date_nanos';
 import { createRelativeDateFormat } from '../../common/field_formats/types/relative_date';
 import { createDurationFormat } from '../../common/field_formats/types/duration';
 import { createIpFormat } from '../../common/field_formats/types/ip';
@@ -36,6 +37,7 @@ import { createStaticLookupFormat } from '../../common/field_formats/types/stati
 fieldFormats.register(createUrlFormat);
 fieldFormats.register(createBytesFormat);
 fieldFormats.register(createDateFormat);
+fieldFormats.register(createDateNanosFormat);
 fieldFormats.register(createRelativeDateFormat);
 fieldFormats.register(createDurationFormat);
 fieldFormats.register(createIpFormat);

src/legacy/core_plugins/kibana/ui_setting_defaults.js

@@ -208,6 +208,23 @@ export function getUiSettingDefaults() {
       type: 'select',
       options: weekdays
     },
+    'dateNanosFormat': {
+      name: i18n.translate('kbn.advancedSettings.dateNanosFormatTitle', {
+        defaultMessage: 'Date with nanoseconds format',
+      }),
+      value: 'MMM D, YYYY @ HH:mm:ss.SSSSSSSSS',
+      description: i18n.translate('kbn.advancedSettings.dateNanosFormatText', {
+        defaultMessage: 'Used for the {dateNanosLink} datatype of Elasticsearch',
+        values: {
+          dateNanosLink:
+            '<a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/date_nanos.html" target="_blank" rel="noopener noreferrer">' +
+            i18n.translate('kbn.advancedSettings.dateNanosLinkTitle', {
+              defaultMessage: 'date_nanos',
+            }) +
+            '</a>',
+        },
+      }),
+    },
     'defaultIndex': {
       name: i18n.translate('kbn.advancedSettings.defaultIndexTitle', {
         defaultMessage: 'Default index',
@@ -608,6 +625,7 @@ export function getUiSettingDefaults() {
 `{
   "ip": { "id": "ip", "params": {} },
   "date": { "id": "date", "params": {} },
+  "date_nanos": { "id": "date_nanos", "params": {}, "es": true },
   "number": { "id": "number", "params": {} },
   "boolean": { "id": "boolean", "params": {} },
   "_source": { "id": "_source", "params": {} },

src/legacy/ui/public/field_editor/field_editor.js

@@ -140,7 +140,7 @@ export class FieldEditorComponent extends PureComponent {
     const fieldTypes = get(FIELD_TYPES_BY_LANG, field.lang, DEFAULT_FIELD_TYPES);
     field.type = fieldTypes.includes(field.type) ? field.type : fieldTypes[0];
 
-    const DefaultFieldFormat = fieldFormats.getDefaultType(field.type);
+    const DefaultFieldFormat = fieldFormats.getDefaultType(field.type, field.esTypes);
     const fieldTypeFormats = [
       getDefaultFormat(DefaultFieldFormat),
       ...fieldFormats.byFieldType[field.type],

src/legacy/ui/public/index_patterns/_field.js

@@ -58,7 +58,7 @@ export function Field(indexPattern, spec) {
 
   let format = spec.format;
   if (!format || !(format instanceof FieldFormat)) {
-    format = indexPattern.fieldFormatMap[spec.name] || fieldFormats.getDefaultInstance(spec.type);
+    format = indexPattern.fieldFormatMap[spec.name] || fieldFormats.getDefaultInstance(spec.type, spec.esTypes);
   }
 
   const indexed = !!spec.indexed;

src/legacy/ui/public/index_patterns/_format_hit.js

@@ -73,7 +73,7 @@ export function formatHit(indexPattern, defaultFormat) {
     }
 
     const val = fieldName === '_source' ? hit._source : indexPattern.flattenHit(hit)[fieldName];
-    return partials[fieldName] = convert(hit, val, fieldName);
+    return convert(hit, val, fieldName);
   };
 
   return formatHit;

src/legacy/ui/public/index_patterns/_get_computed_fields.js

@@ -28,12 +28,10 @@ export function getComputedFields() {
   // Use a docvalue for each date field to ensure standardized formats when working with date fields
   // indexPattern.flattenHit will override "_source" values when the same field is also defined in "fields"
   docvalueFields = _.reject(self.fields.byType.date, 'scripted')
-    .map((dateField) => {
-      return {
-        field: dateField.name,
-        format: 'date_time',
-      };
-    });
+    .map((dateField) => ({
+      field: dateField.name,
+      format: dateField.esTypes && dateField.esTypes.indexOf('date_nanos') !== -1 ? 'strict_date_time' : 'date_time',
+    }));
 
   _.each(self.getScriptedFields(), function (field) {
     scriptFields[field.name] = {

src/legacy/ui/public/index_patterns/_index_pattern.js

@@ -368,6 +368,11 @@ export function IndexPatternProvider(Private, config, Promise, kbnUrl) {
       return !!this.timeFieldName && (!this.fields || !!this.getTimeField());
     }
 
+    isTimeNanosBased() {
+      const timeField = this.getTimeField();
+      return timeField && timeField.esTypes && timeField.esTypes.indexOf('date_nanos') !== -1;
+    }
+
     isUnsupportedTimePattern() {
       return !!this.intervalName;
     }

src/legacy/ui/public/registry/field_formats.js

@@ -51,10 +51,12 @@ class FieldFormatRegistry extends IndexedArray {
    * using the format:defaultTypeMap config map
    *
    * @param  {String} fieldType - the field type
-   * @return {String}
+   * @param  {String[]} esTypes - Array of ES data types
+   * @return {Object}
    */
-  getDefaultConfig = (fieldType) => {
-    return this._defaultMap[fieldType] || this._defaultMap._default_;
+  getDefaultConfig = (fieldType, esTypes) => {
+    const type = this.getDefaultTypeName(fieldType, esTypes);
+    return this._defaultMap[type] || this._defaultMap._default_;
   };
 
   /**
@@ -66,16 +68,43 @@ class FieldFormatRegistry extends IndexedArray {
   getType = (formatId) => {
     return this.byId[formatId];
   };
-
   /**
    * Get the default FieldFormat type (class) for
    * a field type, using the format:defaultTypeMap.
+   * used by the field editor
    *
    * @param  {String} fieldType
+   * @param  {String} esTypes - Array of ES data types
    * @return {Function}
    */
-  getDefaultType = (fieldType) => {
-    return this.byId[this.getDefaultConfig(fieldType).id];
+  getDefaultType = (fieldType, esTypes) => {
+    const config = this.getDefaultConfig(fieldType, esTypes);
+    return this.byId[config.id];
+  };
+
+  /**
+   * Get the name of the default type for ES types like date_nanos
+   * using the format:defaultTypeMap config map
+   *
+   * @param  {String[]} esTypes - Array of ES data types
+   * @return {String|undefined}
+   */
+  getTypeNameByEsTypes = (esTypes) => {
+    if(!Array.isArray(esTypes)) {
+      return;
+    }
+    return esTypes.find(type => this._defaultMap[type] && this._defaultMap[type].es);
+  };
+  /**
+   * Get the default FieldFormat type name for
+   * a field type, using the format:defaultTypeMap.
+   *
+   * @param  {String} fieldType
+   * @param  {String[]} esTypes
+   * @return {string}
+   */
+  getDefaultTypeName = (fieldType, esTypes) => {
+    return this.getTypeNameByEsTypes(esTypes) || fieldType;
   };
 
   /**
@@ -96,13 +125,41 @@ class FieldFormatRegistry extends IndexedArray {
    * Get the default fieldFormat instance for a field format.
    *
    * @param  {String} fieldType
+   * @param  {String[]} esTypes
    * @return {FieldFormat}
    */
-  getDefaultInstance = _.memoize(function (fieldType) {
-    const conf = this.getDefaultConfig(fieldType);
+  getDefaultInstancePlain(fieldType, esTypes) {
+    const conf = this.getDefaultConfig(fieldType, esTypes);
+
     const FieldFormat = this.byId[conf.id];
     return new FieldFormat(conf.params, this.getConfig);
-  });
+  }
+  /**
+   * Returns a cache key built by the given variables for caching in memoized
+   * Where esType contains fieldType, fieldType is returned
+   * -> kibana types have a higher priority in that case
+   * -> would lead to failing tests that match e.g. date format with/without esTypes
+   * https://lodash.com/docs#memoize
+   *
+   * @param  {String} fieldType
+   * @param  {String[]} esTypes
+   * @return {string}
+   */
+  getDefaultInstanceCacheResolver(fieldType, esTypes) {
+    return Array.isArray(esTypes) && esTypes.indexOf(fieldType) === -1
+      ? [fieldType, ...esTypes].join('-')
+      : fieldType;
+  }
+
+  /**
+   * Get the default fieldFormat instance for a field format.
+   * It's a memoized function that builds and reads a cache
+   *
+   * @param  {String} fieldType
+   * @param  {String[]} esTypes
+   * @return {FieldFormat}
+   */
+  getDefaultInstance = _.memoize(this.getDefaultInstancePlain, this.getDefaultInstanceCacheResolver);
 
 
   parseDefaultTypeMap(value) {

src/legacy/utils/kbn_field_types.js

@@ -53,7 +53,7 @@ export const KBN_FIELD_TYPES = [
     name: 'date',
     sortable: true,
     filterable: true,
-    esTypes: ['date'],
+    esTypes: ['date', 'date_nanos'],
   }),
   new KbnFieldType({
     name: 'ip',

test/functional/apps/discover/_date_nanos.js

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function ({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const PageObjects = getPageObjects(['common', 'timePicker', 'discover']);
+  const kibanaServer = getService('kibanaServer');
+  const fromTime = '2015-09-19 06:31:44.000';
+  const toTime = '2015-09-23 18:31:44.000';
+
+  describe('date_nanos', function () {
+
+    before(async function () {
+      await esArchiver.loadIfNeeded('date_nanos');
+      await kibanaServer.uiSettings.replace({ 'defaultIndex': 'date-nanos' });
+      await PageObjects.common.navigateToApp('discover');
+      await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
+    });
+
+    after(function unloadMakelogs() {
+      return esArchiver.unload('date_nanos');
+    });
+
+    it('should show a timestamp with nanoseconds in the first result row', async function () {
+      const time = await PageObjects.timePicker.getTimeConfig();
+      expect(time.start).to.be('Sep 19, 2015 @ 06:31:44.000');
+      expect(time.end).to.be('Sep 23, 2015 @ 18:31:44.000');
+      const rowData = await PageObjects.discover.getDocTableIndex(1);
+      expect(rowData.startsWith('Sep 22, 2015 @ 23:50:13.253123345')).to.be.ok();
+    });
+  });
+
+}

test/functional/apps/discover/index.js

@@ -40,5 +40,6 @@ export default function ({ getService, loadTestFile }) {
     loadTestFile(require.resolve('./_source_filters'));
     loadTestFile(require.resolve('./_large_string'));
     loadTestFile(require.resolve('./_inspector'));
+    loadTestFile(require.resolve('./_date_nanos'));
   });
 }

test/functional/fixtures/es_archiver/date_nanos/mappings.json

@@ -0,0 +1,19 @@
+{
+  "type": "index",
+  "value": {
+    "index": "date-nanos",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date_nanos"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "0",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}