[DataViews] when a mapped field exists sharing a name as a runtime field, specify the mapped field in queries - not the RT field (#138471)

* runtime fields vs mapped data

* optimizations per feedback

src/plugins/data_views/common/data_views/data_view.test.ts

@@ -6,19 +6,15 @@
  * Side Public License, v 1.
  */
 
-import { map, last } from 'lodash';
-
-import { DataView } from './data_view';
-
-import { CharacterNotAllowedInField } from '@kbn/kibana-utils-plugin/common';
-
-import { DataViewField } from '../fields';
-
-import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks';
 import { FieldFormat } from '@kbn/field-formats-plugin/common';
-import { RuntimeField, RuntimeTypeExceptComposite } from '../types';
-import { stubLogstashFields } from '../field.stub';
+import { fieldFormatsMock } from '@kbn/field-formats-plugin/common/mocks';
+import { CharacterNotAllowedInField } from '@kbn/kibana-utils-plugin/common';
+import { last, map } from 'lodash';
 import { stubbedSavedObjectIndexPattern } from '../data_view.stub';
+import { stubLogstashFields } from '../field.stub';
+import { DataViewField } from '../fields';
+import { RuntimeField, RuntimeTypeExceptComposite } from '../types';
+import { DataView } from './data_view';
 
 class MockFieldFormatter {}
 
@@ -41,7 +37,7 @@ const runtimeField = {
   type: 'string',
 };
 
-fieldFormatsMock.getInstance = jest.fn().mockImplementation(() => new MockFieldFormatter()) as any;
+fieldFormatsMock.getInstance = jest.fn().mockImplementation(() => new MockFieldFormatter());
 
 // helper function to create index patterns
 function create(id: string, spec?: object) {
@@ -309,6 +305,29 @@ describe('IndexPattern', () => {
       expect(indexPattern.toSpec()!.fields!['@tags'].runtimeField).toBeUndefined();
     });
 
+    test('ignore runtime field mapping if a mapped field exists with the same name', () => {
+      expect(indexPattern.getRuntimeMappings()).toEqual({
+        runtime_field: { script: { source: "emit('hello world')" }, type: 'keyword' },
+      });
+
+      // add a runtime field called "theme"
+      indexPattern.addRuntimeField('theme', runtimeWithAttrs);
+
+      // add a new mapped field also called "theme"
+      indexPattern.fields.add({
+        name: 'theme',
+        type: 'keyword',
+        aggregatable: true,
+        searchable: true,
+        readFromDocValues: false,
+        isMapped: true,
+      });
+
+      expect(indexPattern.getRuntimeMappings()).toEqual({
+        runtime_field: { script: { source: "emit('hello world')" }, type: 'keyword' },
+      });
+    });
+
     test('add and remove runtime field as new field', () => {
       indexPattern.addRuntimeField('new_field', runtimeWithAttrs);
       expect(indexPattern.toSpec().runtimeFieldMap).toEqual({
@@ -354,9 +373,9 @@ describe('IndexPattern', () => {
       expect(indexPattern.toSpec()!.fields!.new_field).toBeUndefined();
     });
 
-    test('should not allow runtime field with * in name', async () => {
+    test('should not allow runtime field with * in name', () => {
       try {
-        await indexPattern.addRuntimeField('test*123', runtime);
+        indexPattern.addRuntimeField('test*123', runtime);
       } catch (e) {
         expect(e).toBeInstanceOf(CharacterNotAllowedInField);
       }

src/plugins/data_views/common/data_views/data_view.ts

@@ -496,6 +496,7 @@ export class DataView implements DataViewBase {
 
   /**
    * Get all runtime field definitions.
+   * NOTE: this does not strip out runtime fields that match mapped field names
    * @returns map of runtime field definitions by field name
    */
 
@@ -582,8 +583,19 @@ export class DataView implements DataViewBase {
    * Return the "runtime_mappings" section of the ES search query.
    */
   getRuntimeMappings(): estypes.MappingRuntimeFields {
-    // @ts-expect-error The ES client does not yet include the "composite" runtime type
-    return _.cloneDeep(this.runtimeFieldMap);
+    const mappedFields = this.getMappedFieldNames();
+    const records = Object.keys(this.runtimeFieldMap).reduce<Record<string, RuntimeFieldSpec>>(
+      (acc, fieldName) => {
+        // do not include fields that are mapped
+        if (!mappedFields.includes(fieldName)) {
+          acc[fieldName] = this.runtimeFieldMap[fieldName];
+        }
+
+        return acc;
+      },
+      {}
+    );
+    return records as estypes.MappingRuntimeFields;
   }
 
   /**
@@ -667,6 +679,15 @@ export class DataView implements DataViewBase {
     delete this.fieldFormatMap[fieldName];
   };
 
+  private getMappedFieldNames() {
+    return this.fields.getAll().reduce<string[]>((acc, dataViewField) => {
+      if (dataViewField.isMapped) {
+        acc.push(dataViewField.name);
+      }
+      return acc;
+    }, []);
+  }
+
   /**
    * Add composite runtime field and all subfields.
    * @param name field name