Add multi field info to the IndexPattern (#33681)

Adds two fields to the IndexPattern Field:

* parent - the name of the field this field is a child of
* subType - The type of child this field is. Currently the only valid value is multi but we could expand this to include aliases, object children, and nested children.

The thinking behind implementing these two new properties instead of a simple isMultiField flag is that it should be generic enough to describe other sorts of parent -> child relationships between fields.

packages/kbn-es-query/src/__fixtures__/index_pattern_response.json

@@ -144,7 +144,9 @@
       "scripted": false,
       "searchable": true,
       "aggregatable": true,
-      "readFromDocValues": true
+      "readFromDocValues": true,
+      "parent": "machine.os",
+      "subType": "multi"
     },
     {
       "name": "geo.src",

src/fixtures/logstash_fields.js

@@ -25,7 +25,7 @@ function stubbedLogstashFields() {
   return [
     //                                  |aggregatable
     //                                  |      |searchable
-    // name               esType        |      |      |metadata
+    // name               esType        |      |      |metadata       | parent      | subType
     ['bytes',             'long',       true,  true,  { count: 10 } ],
     ['ssl',               'boolean',    true,  true,  { count: 20 } ],
     ['@timestamp',        'date',       true,  true,  { count: 30 } ],
@@ -41,7 +41,7 @@ function stubbedLogstashFields() {
     ['geo.coordinates',   'geo_point',  true,  true ],
     ['extension',         'keyword',    true,  true ],
     ['machine.os',        'text',       true,  true ],
-    ['machine.os.raw',    'keyword',    true,  true ],
+    ['machine.os.raw',    'keyword',    true,  true,   {},            'machine.os', 'multi' ],
     ['geo.src',           'keyword',    true,  true ],
     ['_id',               '_id',        true,  true ],
     ['_type',             '_type',      true,  true ],
@@ -59,7 +59,9 @@ function stubbedLogstashFields() {
       esType,
       aggregatable,
       searchable,
-      metadata = {}
+      metadata = {},
+      parent = undefined,
+      subType = undefined,
     ] = row;
 
     const {
@@ -83,6 +85,8 @@ function stubbedLogstashFields() {
       script,
       lang,
       scripted,
+      parent,
+      subType,
     };
   });
 }

src/legacy/server/index_patterns/service/index_patterns_service.d.ts

@@ -29,4 +29,6 @@ export interface FieldDescriptor {
   readFromDocValues: boolean;
   searchable: boolean;
   type: string;
+  parent?: string;
+  subType?: string;
 }

src/legacy/server/index_patterns/service/lib/field_capabilities/__fixtures__/es_field_caps_response.json

@@ -193,6 +193,34 @@
           "index1"
         ]
       }
+    },
+    "multi_parent": {
+      "text": {
+        "type": "text",
+        "searchable": true,
+        "aggregatable": false
+      }
+    },
+    "multi_parent.child": {
+      "keyword": {
+        "type": "keyword",
+        "searchable": true,
+        "aggregatable": true
+      }
+    },
+    "object_parent": {
+      "object": {
+        "type": "object",
+        "searchable": false,
+        "aggregatable": false
+      }
+    },
+    "object_parent.child": {
+      "keyword": {
+        "type": "keyword",
+        "searchable": true,
+        "aggregatable": true
+      }
     }
   }
 }

src/legacy/server/index_patterns/service/lib/field_capabilities/field_caps_response.js

@@ -79,7 +79,7 @@ import { shouldReadFieldFromDocValues } from './should_read_field_from_doc_value
  */
 export function readFieldCapsResponse(fieldCapsResponse) {
   const capsByNameThenType = fieldCapsResponse.fields;
-  return Object.keys(capsByNameThenType).map(fieldName => {
+  const kibanaFormattedCaps = Object.keys(capsByNameThenType).map(fieldName => {
     const capsByType = capsByNameThenType[fieldName];
     const types = Object.keys(capsByType);
 
@@ -120,7 +120,26 @@ export function readFieldCapsResponse(fieldCapsResponse) {
       aggregatable: isAggregatable,
       readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
     };
-  }).filter(field => {
+  });
+
+  // Get all types of sub fields. These could be multi fields or children of nested/object types
+  const subFields = kibanaFormattedCaps.filter(field => {
+    return field.name.includes('.');
+  });
+
+  // Discern which sub fields are multi fields. If the parent field is not an object or nested field
+  // the child must be a multi field.
+  subFields.forEach(field => {
+    const parentFieldName = field.name.split('.').slice(0, -1).join('.');
+    const parentFieldCaps = kibanaFormattedCaps.find(caps => caps.name === parentFieldName);
+
+    if (parentFieldCaps && !['object', 'nested'].includes(parentFieldCaps.type)) {
+      field.parent = parentFieldName;
+      field.subType = 'multi';
+    }
+  });
+
+  return kibanaFormattedCaps.filter(field => {
     return !['object', 'nested'].includes(field.type);
   });
 }

src/legacy/server/index_patterns/service/lib/field_capabilities/field_caps_response.test.js

@@ -18,7 +18,7 @@
  */
 
 /* eslint import/no-duplicates: 0 */
-import { cloneDeep } from 'lodash';
+import { cloneDeep, omit } from 'lodash';
 import sinon from 'sinon';
 
 import * as shouldReadFieldFromDocValuesNS from './should_read_field_from_doc_values';
@@ -37,21 +37,20 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
     describe('conflicts', () => {
       it('returns a field for each in response, no filtering', () => {
         const fields = readFieldCapsResponse(esResponse);
-        expect(fields).toHaveLength(19);
+        expect(fields).toHaveLength(22);
       });
 
-      it('includes only name, type, searchable, aggregatable, readFromDocValues, and maybe conflictDescriptions of each field', () => {
+      it('includes only name, type, searchable, aggregatable, readFromDocValues, and maybe conflictDescriptions, parent, ' +
+        'and subType of each field', () => {
         const responseClone = cloneDeep(esResponse);
         // try to trick it into including an extra field
         responseClone.fields['@timestamp'].date.extraCapability = true;
         const fields = readFieldCapsResponse(responseClone);
 
         fields.forEach(field => {
-          if (field.conflictDescriptions) {
-            delete field.conflictDescriptions;
-          }
+          const fieldWithoutOptionalKeys = omit(field, 'conflictDescriptions', 'parent', 'subType');
 
-          expect(Object.keys(field)).toEqual([
+          expect(Object.keys(fieldWithoutOptionalKeys)).toEqual([
             'name',
             'type',
             'searchable',
@@ -65,7 +64,8 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
         sandbox.spy(shouldReadFieldFromDocValuesNS, 'shouldReadFieldFromDocValues');
         const fields = readFieldCapsResponse(esResponse);
         const conflictCount = fields.filter(f => f.type === 'conflict').length;
-        sinon.assert.callCount(shouldReadFieldFromDocValues, fields.length - conflictCount);
+        // +1 is for the object field which gets filtered out of the final return value from readFieldCapsResponse
+        sinon.assert.callCount(shouldReadFieldFromDocValues, fields.length - conflictCount + 1);
       });
 
       it('converts es types to kibana types', () => {
@@ -121,6 +121,23 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
         expect(mixSearchable.searchable).toBe(true);
         expect(mixSearchableOther.searchable).toBe(true);
       });
+
+      it('returns multi fields with parent and subType keys describing the relationship', () => {
+        const fields = readFieldCapsResponse(esResponse);
+        const child = fields.find(f => f.name === 'multi_parent.child');
+        expect(child).toHaveProperty('parent', 'multi_parent');
+        expect(child).toHaveProperty('subType', 'multi');
+      });
+
+      it('should not confuse object children for multi field children', () => {
+        // We detect multi fields by finding fields that have a dot in their name and then looking
+        // to see if their parents are *not* object or nested fields. In the future we may want to
+        // add parent and subType info for object and nested fields but for now we don't need it.
+        const fields = readFieldCapsResponse(esResponse);
+        const child = fields.find(f => f.name === 'object_parent.child');
+        expect(child).not.toHaveProperty('parent');
+        expect(child).not.toHaveProperty('subType');
+      });
     });
   });
 });

src/legacy/ui/public/index_patterns/_field.d.ts

@@ -23,4 +23,6 @@ export interface Field {
   aggregatable: boolean;
   filterable: boolean;
   searchable: boolean;
+  parent?: string;
+  subType?: string;
 }

src/legacy/ui/public/index_patterns/_field.js

@@ -98,6 +98,10 @@ export function Field(indexPattern, spec) {
   // conflict info
   obj.writ('conflictDescriptions');
 
+  // multi info
+  obj.fact('parent');
+  obj.fact('subType');
+
   return obj.create();
 }
 

test/api_integration/apis/index_patterns/fields_for_wildcard_route/index.js

@@ -21,5 +21,6 @@ export default function ({ loadTestFile }) {
   describe('index_patterns/_fields_for_wildcard route', () => {
     loadTestFile(require.resolve('./params'));
     loadTestFile(require.resolve('./conflicts'));
+    loadTestFile(require.resolve('./response'));
   });
 }

test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js

@@ -58,7 +58,9 @@ export default function ({ getService }) {
               searchable: true,
               aggregatable: true,
               name: 'baz.keyword',
-              readFromDocValues: true
+              readFromDocValues: true,
+              parent: 'baz',
+              subType: 'multi',
             },
             {
               type: 'number',
@@ -86,7 +88,7 @@ export default function ({ getService }) {
         .expect(200, {
           fields: [
             {
-              aggregatable: false,
+              aggregatable: true,
               name: '_id',
               readFromDocValues: false,
               searchable: true,
@@ -118,7 +120,9 @@ export default function ({ getService }) {
               searchable: true,
               aggregatable: true,
               name: 'baz.keyword',
-              readFromDocValues: true
+              readFromDocValues: true,
+              parent: 'baz',
+              subType: 'multi',
             },
             {
               aggregatable: false,

x-pack/plugins/kuery_autocomplete/public/autocomplete_providers/__fixtures__/index_pattern_response.json

@@ -144,7 +144,9 @@
       "scripted": false,
       "searchable": true,
       "aggregatable": true,
-      "readFromDocValues": true
+      "readFromDocValues": true,
+      "parent": "machine.os",
+      "subType": "multi"
     },
     {
       "name": "geo.src",