[7.12] [SECURITY SOLUTION] Fix unmapped field timeline (#99130) (#99213)

* [SECURITY SOLUTION] Fix unmapped field timeline (#99130)

* add unmapped include_unmapped

* bringing back unmapped field timeline

* add unit test

* fix unit test

x-pack/plugins/security_solution/common/search_strategy/timeline/events/all/index.ts

@@ -35,7 +35,7 @@ export interface TimelineEventsAllStrategyResponse extends IEsSearchResponse {
 }
 
 export interface TimelineEventsAllRequestOptions extends TimelineRequestOptionsPaginated {
-  fields: string[];
+  fields: string[] | Array<{ field: string; include_unmapped: boolean }>;
   fieldRequested: string[];
   language: 'eql' | 'kuery' | 'lucene';
 }

x-pack/plugins/security_solution/public/common/components/event_details/columns.test.tsx

@@ -55,14 +55,14 @@ describe('getColumns', () => {
       checkboxColumn = getColumns(defaultProps)[0] as Column;
     });
 
-    test('should be disabled when the field does not exist', () => {
+    test('should be enabled when the field does not exist', () => {
       const testField = 'nonExistingField';
       const wrapper = mount(
         <TestProviders>{checkboxColumn.render(testField, testData)}</TestProviders>
       ) as ReactWrapper;
       expect(
         wrapper.find(`[data-test-subj="toggle-field-${testField}"]`).first().prop('disabled')
-      ).toBe(true);
+      ).toBe(false);
     });
 
     test('should be enabled when the field does exist', () => {

x-pack/plugins/security_solution/public/common/components/event_details/columns.tsx

@@ -23,7 +23,6 @@ import React from 'react';
 import styled from 'styled-components';
 import { onFocusReFocusDraggable } from '../accessibility/helpers';
 import { BrowserFields } from '../../containers/source';
-import { ToStringArray } from '../../../graphql/types';
 import { ColumnHeaderOptions } from '../../../timelines/store/timeline/model';
 import { DragEffects } from '../drag_and_drop/draggable_wrapper';
 import { DroppableWrapper } from '../drag_and_drop/droppable_wrapper';
@@ -92,10 +91,6 @@ export const getColumns = ({
     width: '30px',
     render: (field: string, data: EventFieldsData) => {
       const label = data.isObjectArray ? i18n.NESTED_COLUMN(field) : i18n.VIEW_COLUMN(field);
-      const fieldFromBrowserField = getFieldFromBrowserField(
-        [data.category, 'fields', field],
-        browserFields
-      );
       return (
         <EuiToolTip content={label}>
           <EuiCheckbox
@@ -111,9 +106,7 @@ export const getColumns = ({
                 width: DEFAULT_COLUMN_MIN_WIDTH,
               })
             }
-            disabled={
-              (data.isObjectArray && data.type !== 'geo_point') || fieldFromBrowserField == null
-            }
+            disabled={data.isObjectArray && data.type !== 'geo_point'}
           />
         </EuiToolTip>
       );
@@ -193,40 +186,56 @@ export const getColumns = ({
     name: i18n.VALUE,
     sortable: true,
     truncateText: false,
-    render: (values: ToStringArray | null | undefined, data: EventFieldsData) => (
-      <FullWidthFlexGroup
-        direction="column"
-        alignItems="flexStart"
-        component="span"
-        gutterSize="none"
-      >
-        {values != null &&
-          values.map((value, i) => (
-            <FullWidthFlexItem
-              grow={false}
-              component="span"
-              key={`event-details-value-flex-item-${contextId}-${eventId}-${data.field}-${i}-${value}`}
-            >
-              <div data-colindex={3} onFocus={onFocusReFocusDraggable} role="button" tabIndex={0}>
-                {data.field === MESSAGE_FIELD_NAME ? (
-                  <OverflowField value={value} />
-                ) : (
-                  <FormattedFieldValue
-                    contextId={`event-details-value-formatted-field-value-${contextId}-${eventId}-${data.field}-${i}-${value}`}
-                    eventId={eventId}
-                    fieldFormat={data.format}
-                    fieldName={data.field}
-                    fieldType={data.type}
-                    isObjectArray={data.isObjectArray}
-                    value={value}
-                    linkValue={getLinkValue(data.field)}
-                  />
-                )}
-              </div>
-            </FullWidthFlexItem>
-          ))}
-      </FullWidthFlexGroup>
-    ),
+    render: (values: string[] | null | undefined, data: EventFieldsData) => {
+      const fieldFromBrowserField = getFieldFromBrowserField(
+        [data.category, 'fields', data.field],
+        browserFields
+      );
+      return (
+        <FullWidthFlexGroup
+          direction="column"
+          alignItems="flexStart"
+          component="span"
+          gutterSize="none"
+        >
+          {values != null &&
+            values.map((value, i) => {
+              if (fieldFromBrowserField == null) {
+                return <EuiText size="s">{value}</EuiText>;
+              }
+              return (
+                <FullWidthFlexItem
+                  grow={false}
+                  component="span"
+                  key={`event-details-value-flex-item-${contextId}-${eventId}-${data.field}-${i}-${value}`}
+                >
+                  <div
+                    data-colindex={3}
+                    onFocus={onFocusReFocusDraggable}
+                    role="button"
+                    tabIndex={0}
+                  >
+                    {data.field === MESSAGE_FIELD_NAME ? (
+                      <OverflowField value={value} />
+                    ) : (
+                      <FormattedFieldValue
+                        contextId={`event-details-value-formatted-field-value-${contextId}-${eventId}-${data.field}-${i}-${value}`}
+                        eventId={eventId}
+                        fieldFormat={data.format}
+                        fieldName={data.field}
+                        fieldType={data.type}
+                        isObjectArray={data.isObjectArray}
+                        value={value}
+                        linkValue={getLinkValue(data.field)}
+                      />
+                    )}
+                  </div>
+                </FullWidthFlexItem>
+              );
+            })}
+        </FullWidthFlexGroup>
+      );
+    },
   },
   {
     field: 'valuesConcatenated',

x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts

@@ -13,6 +13,7 @@ import {
 } from '../../../../../../common/search_strategy';
 import { toStringArray } from '../../../../helpers/to_array';
 import { getDataSafety, getDataFromFieldsHits } from '../details/helpers';
+import { TIMELINE_EVENTS_FIELDS } from './constants';
 
 const getTimestamp = (hit: EventHit): string => {
   if (hit.fields && hit.fields['@timestamp']) {
@@ -23,6 +24,12 @@ const getTimestamp = (hit: EventHit): string => {
   return '';
 };
 
+export const buildFieldsRequest = (fields: string[]) =>
+  uniq([...fields.filter((f) => !f.startsWith('_')), ...TIMELINE_EVENTS_FIELDS]).map((field) => ({
+    field,
+    include_unmapped: true,
+  }));
+
 export const formatTimelineData = async (
   dataFields: readonly string[],
   ecsFields: readonly string[],

x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { cloneDeep, uniq } from 'lodash/fp';
+import { cloneDeep } from 'lodash/fp';
 
 import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants';
 import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common';
@@ -20,7 +20,7 @@ import { inspectStringifyObject } from '../../../../../utils/build_query';
 import { SecuritySolutionTimelineFactory } from '../../types';
 import { buildTimelineEventsAllQuery } from './query.events_all.dsl';
 import { TIMELINE_EVENTS_FIELDS } from './constants';
-import { formatTimelineData } from './helpers';
+import { buildFieldsRequest, formatTimelineData } from './helpers';
 
 export const timelineEventsAll: SecuritySolutionTimelineFactory<TimelineEventsQueries.all> = {
   buildDsl: (options: TimelineEventsAllRequestOptions) => {
@@ -28,7 +28,7 @@ export const timelineEventsAll: SecuritySolutionTimelineFactory<TimelineEventsQu
       throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`);
     }
     const { fieldRequested, ...queryOptions } = cloneDeep(options);
-    queryOptions.fields = uniq([...fieldRequested, ...TIMELINE_EVENTS_FIELDS]);
+    queryOptions.fields = buildFieldsRequest(fieldRequested);
     return buildTimelineEventsAllQuery(queryOptions);
   },
   parse: async (
@@ -36,7 +36,7 @@ export const timelineEventsAll: SecuritySolutionTimelineFactory<TimelineEventsQu
     response: IEsSearchResponse<unknown>
   ): Promise<TimelineEventsAllStrategyResponse> => {
     const { fieldRequested, ...queryOptions } = cloneDeep(options);
-    queryOptions.fields = uniq([...fieldRequested, ...TIMELINE_EVENTS_FIELDS]);
+    queryOptions.fields = buildFieldsRequest(fieldRequested);
     const { activePage, querySize } = options.pagination;
     const totalCount = response.rawResponse.hits.total || 0;
     const hits = response.rawResponse.hits.hits;

x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.test.ts

@@ -40,7 +40,10 @@ describe('buildTimelineDetailsQuery', () => {
             },
           ],
           "fields": Array [
-            "*",
+            Object {
+              "field": "*",
+              "include_unmapped": true,
+            },
           ],
           "query": Object {
             "terms": Object {

x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/query.events_details.dsl.ts

@@ -22,7 +22,7 @@ export const buildTimelineDetailsQuery = (
         _id: [id],
       },
     },
-    fields: ['*'],
+    fields: [{ field: '*', include_unmapped: true }],
     _source: true,
   },
   size: 1,