[7.12] Discover: Add handling for source column (#91815) (#91981)

* Discover: Add handling for source column (#91815)

* enable by default

* [Discover] Fix context view for date_nanos format with custom timestamps (#54089)

* Switch from _source to fields when fetching anchor records with date_nanos timestamps

* Add testdata

* Add functional test

* fix search_after

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Matthias Wilhelm <matthias.wilhelm@elastic.co>

src/plugins/discover/public/application/angular/context.js

@@ -65,6 +65,7 @@ function ContextAppRouteController($routeParams, $scope, $route) {
     storeInSessionStorage: getServices().uiSettings.get('state:storeInSessionStorage'),
     history: getServices().history(),
     toasts: getServices().core.notifications.toasts,
+    uiSettings: getServices().core.uiSettings,
   });
   this.state = { ...appState.getState() };
   this.anchorId = $routeParams.id;

src/plugins/discover/public/application/angular/context/api/context.ts

@@ -64,9 +64,9 @@ function fetchContextProvider(indexPatterns: IndexPatternsContract, useNewFields
     const searchSource = await createSearchSource(indexPattern, filters);
     const sortDirToApply = type === 'successors' ? sortDir : reverseSortDir(sortDir);
 
-    const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor._source[timeField]) : '';
+    const nanos = indexPattern.isTimeNanosBased() ? extractNanos(anchor.fields[timeField][0]) : '';
     const timeValueMillis =
-      nanos !== '' ? convertIsoToMillis(anchor._source[timeField]) : anchor.sort[0];
+      nanos !== '' ? convertIsoToMillis(anchor.fields[timeField][0]) : anchor.sort[0];
 
     const intervals = generateIntervals(LOOKUP_OFFSETS, timeValueMillis, type, sortDir);
     let documents: EsHitRecordList = [];

src/plugins/discover/public/application/angular/context_state.test.ts

@@ -6,10 +6,12 @@
  * Side Public License, v 1.
  */
 
+import { IUiSettingsClient } from 'kibana/public';
 import { getState } from './context_state';
 import { createBrowserHistory, History } from 'history';
 import { FilterManager, Filter } from '../../../../data/public';
 import { coreMock } from '../../../../../core/public/mocks';
+import { SEARCH_FIELDS_FROM_SOURCE } from '../../../common';
 const setupMock = coreMock.createSetup();
 
 describe('Test Discover Context State', () => {
@@ -23,6 +25,10 @@ describe('Test Discover Context State', () => {
       defaultStepSize: '4',
       timeFieldName: 'time',
       history,
+      uiSettings: {
+        get: <T>(key: string) =>
+          ((key === SEARCH_FIELDS_FROM_SOURCE ? true : ['_source']) as unknown) as T,
+      } as IUiSettingsClient,
     });
     state.startSync();
   });

src/plugins/discover/public/application/angular/context_state.ts

@@ -8,7 +8,7 @@
 
 import _ from 'lodash';
 import { History } from 'history';
-import { NotificationsStart } from 'kibana/public';
+import { NotificationsStart, IUiSettingsClient } from 'kibana/public';
 import {
   createStateContainer,
   createKbnUrlStateStorage,
@@ -17,6 +17,7 @@ import {
   withNotifyOnErrors,
 } from '../../../../kibana_utils/public';
 import { esFilters, FilterManager, Filter, Query } from '../../../../data/public';
+import { handleSourceColumnState } from './helpers';
 
 export interface AppState {
   /**
@@ -73,6 +74,11 @@ interface GetStateParams {
    * kbnUrlStateStorage will use it notifying about inner errors
    */
   toasts?: NotificationsStart['toasts'];
+
+  /**
+   * core ui settings service
+   */
+  uiSettings: IUiSettingsClient;
 }
 
 interface GetStateReturn {
@@ -123,6 +129,7 @@ export function getState({
   storeInSessionStorage = false,
   history,
   toasts,
+  uiSettings,
 }: GetStateParams): GetStateReturn {
   const stateStorage = createKbnUrlStateStorage({
     useHash: storeInSessionStorage,
@@ -134,7 +141,12 @@ export function getState({
   const globalStateContainer = createStateContainer<GlobalState>(globalStateInitial);
 
   const appStateFromUrl = stateStorage.get(APP_STATE_URL_KEY) as AppState;
-  const appStateInitial = createInitialAppState(defaultStepSize, timeFieldName, appStateFromUrl);
+  const appStateInitial = createInitialAppState(
+    defaultStepSize,
+    timeFieldName,
+    appStateFromUrl,
+    uiSettings
+  );
   const appStateContainer = createStateContainer<AppState>(appStateInitial);
 
   const { start, stop } = syncStates([
@@ -257,7 +269,8 @@ function getFilters(state: AppState | GlobalState): Filter[] {
 function createInitialAppState(
   defaultSize: string,
   timeFieldName: string,
-  urlState: AppState
+  urlState: AppState,
+  uiSettings: IUiSettingsClient
 ): AppState {
   const defaultState = {
     columns: ['_source'],
@@ -270,8 +283,11 @@ function createInitialAppState(
     return defaultState;
   }
 
-  return {
-    ...defaultState,
-    ...urlState,
-  };
+  return handleSourceColumnState(
+    {
+      ...defaultState,
+      ...urlState,
+    },
+    uiSettings
+  );
 }

src/plugins/discover/public/application/angular/discover.js

@@ -110,7 +110,7 @@ app.config(($routeProvider) => {
         const history = getHistory();
         const savedSearchId = $route.current.params.id;
         return data.indexPatterns.ensureDefaultIndexPattern(history).then(() => {
-          const { appStateContainer } = getState({ history });
+          const { appStateContainer } = getState({ history, uiSettings: config });
           const { index } = appStateContainer.getState();
           return Promise.props({
             ip: loadIndexPattern(index, data.indexPatterns, config),
@@ -195,6 +195,7 @@ function discoverController($route, $scope, Promise) {
     storeInSessionStorage: config.get('state:storeInSessionStorage'),
     history,
     toasts: core.notifications.toasts,
+    uiSettings: config,
   });
 
   const {

src/plugins/discover/public/application/angular/discover_state.test.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+import { IUiSettingsClient } from 'kibana/public';
 import {
   getState,
   GetStateReturn,
@@ -14,11 +15,17 @@ import {
 import { createBrowserHistory, History } from 'history';
 import { dataPluginMock } from '../../../../data/public/mocks';
 import { SavedSearch } from '../../saved_searches';
+import { SEARCH_FIELDS_FROM_SOURCE } from '../../../common';
 
 let history: History;
 let state: GetStateReturn;
 const getCurrentUrl = () => history.createHref(history.location);
 
+const uiSettingsMock = {
+  get: <T>(key: string) =>
+    ((key === SEARCH_FIELDS_FROM_SOURCE ? true : ['_source']) as unknown) as T,
+} as IUiSettingsClient;
+
 describe('Test discover state', () => {
   beforeEach(async () => {
     history = createBrowserHistory();
@@ -26,6 +33,7 @@ describe('Test discover state', () => {
     state = getState({
       getStateDefaults: () => ({ index: 'test' }),
       history,
+      uiSettings: uiSettingsMock,
     });
     await state.replaceUrlAppState({});
     await state.startSync();
@@ -81,6 +89,7 @@ describe('Test discover state with legacy migration', () => {
     state = getState({
       getStateDefaults: () => ({ index: 'test' }),
       history,
+      uiSettings: uiSettingsMock,
     });
     expect(state.appStateContainer.getState()).toMatchInlineSnapshot(`
       Object {
@@ -106,6 +115,7 @@ describe('createSearchSessionRestorationDataProvider', () => {
     data: mockDataPlugin,
     appStateContainer: getState({
       history: createBrowserHistory(),
+      uiSettings: uiSettingsMock,
     }).appStateContainer,
     getSavedSearch: () => mockSavedSearch,
   });

src/plugins/discover/public/application/angular/discover_state.ts

@@ -9,7 +9,7 @@
 import { isEqual } from 'lodash';
 import { i18n } from '@kbn/i18n';
 import { History } from 'history';
-import { NotificationsStart } from 'kibana/public';
+import { NotificationsStart, IUiSettingsClient } from 'kibana/public';
 import {
   createKbnUrlStateStorage,
   createStateContainer,
@@ -30,6 +30,7 @@ import { migrateLegacyQuery } from '../helpers/migrate_legacy_query';
 import { DiscoverGridSettings } from '../components/discover_grid/types';
 import { DISCOVER_APP_URL_GENERATOR, DiscoverUrlGeneratorState } from '../../url_generator';
 import { SavedSearch } from '../../saved_searches';
+import { handleSourceColumnState } from './helpers';
 
 export interface AppState {
   /**
@@ -90,6 +91,11 @@ interface GetStateParams {
    * kbnUrlStateStorage will use it notifying about inner errors
    */
   toasts?: NotificationsStart['toasts'];
+
+  /**
+   * core ui settings service
+   */
+  uiSettings: IUiSettingsClient;
 }
 
 export interface GetStateReturn {
@@ -149,6 +155,7 @@ export function getState({
   storeInSessionStorage = false,
   history,
   toasts,
+  uiSettings,
 }: GetStateParams): GetStateReturn {
   const defaultAppState = getStateDefaults ? getStateDefaults() : {};
   const stateStorage = createKbnUrlStateStorage({
@@ -163,10 +170,14 @@ export function getState({
     appStateFromUrl.query = migrateLegacyQuery(appStateFromUrl.query);
   }
 
-  let initialAppState = {
-    ...defaultAppState,
-    ...appStateFromUrl,
-  };
+  let initialAppState = handleSourceColumnState(
+    {
+      ...defaultAppState,
+      ...appStateFromUrl,
+    },
+    uiSettings
+  );
+  // todo filter source depending on fields fetchinbg flag (if no columns remain and source fetching is enabled, use default columns)
   let previousAppState: AppState;
   const appStateContainer = createStateContainer<AppState>(initialAppState);
 

src/plugins/discover/public/application/angular/helpers/index.ts

@@ -8,3 +8,4 @@
 
 export { buildPointSeriesData } from './point_series';
 export { formatRow } from './row_formatter';
+export { handleSourceColumnState } from './state_helpers';

src/plugins/discover/public/application/angular/helpers/state_helpers.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { IUiSettingsClient } from 'src/core/public';
+import { SEARCH_FIELDS_FROM_SOURCE, DEFAULT_COLUMNS_SETTING } from '../../../../common';
+
+/**
+ * Makes sure the current state is not referencing the source column when using the fields api
+ * @param state
+ * @param uiSettings
+ */
+export function handleSourceColumnState<TState extends { columns?: string[] }>(
+  state: TState,
+  uiSettings: IUiSettingsClient
+): TState {
+  if (!state.columns) {
+    return state;
+  }
+  const useNewFieldsApi = !uiSettings.get(SEARCH_FIELDS_FROM_SOURCE);
+  const defaultColumns = uiSettings.get(DEFAULT_COLUMNS_SETTING);
+  if (useNewFieldsApi) {
+    // if fields API is used, filter out the source column
+    return {
+      ...state,
+      columns: state.columns.filter((column) => column !== '_source'),
+    };
+  } else if (state.columns.length === 0) {
+    // if _source fetching is used and there are no column, switch back to default columns
+    // this can happen if the fields API was previously used
+    return {
+      ...state,
+      columns: [...defaultColumns],
+    };
+  }
+
+  return state;
+}

src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx

@@ -315,7 +315,8 @@ export const DiscoverGrid = ({
           <DiscoverGridFlyout
             indexPattern={indexPattern}
             hit={expandedDoc}
-            columns={columns}
+            // if default columns are used, dont make them part of the URL - the context state handling will take care to restore them
+            columns={defaultColumns ? [] : columns}
             onFilter={onFilter}
             onRemoveColumn={onRemoveColumn}
             onAddColumn={onAddColumn}

src/plugins/discover/server/ui_settings.ts

@@ -191,7 +191,7 @@ export const uiSettings: Record<string, UiSettingsParams> = {
   [SEARCH_FIELDS_FROM_SOURCE]: {
     name: 'Read fields from _source',
     description: `When enabled will load documents directly from \`_source\`. This is soon going to be deprecated. When disabled, will retrieve fields via the new Fields API in the high-level search service.`,
-    value: true,
+    value: false,
     category: ['discover'],
     schema: schema.boolean(),
   },

test/functional/apps/context/index.js

@@ -32,5 +32,6 @@ export default function ({ getService, getPageObjects, loadTestFile }) {
     loadTestFile(require.resolve('./_filters'));
     loadTestFile(require.resolve('./_size'));
     loadTestFile(require.resolve('./_date_nanos'));
+    loadTestFile(require.resolve('./_date_nanos_custom_timestamp'));
   });
 }

test/functional/apps/discover/_shared_links.ts

@@ -77,7 +77,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
             '/app/discover?_t=1453775307251#' +
             '/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time' +
             ":(from:'2015-09-19T06:31:44.000Z',to:'2015-09" +
-            "-23T18:31:44.000Z'))&_a=(columns:!(_source),filters:!(),index:'logstash-" +
+            "-23T18:31:44.000Z'))&_a=(columns:!(),filters:!(),index:'logstash-" +
             "*',interval:auto,query:(language:kuery,query:'')" +
             ",sort:!(!('@timestamp',desc)))";
           const actualUrl = await PageObjects.share.getSharedUrl();

test/functional/fixtures/es_archiver/date_nanos_custom/data.json

@@ -0,0 +1,56 @@
+{
+  "type": "doc",
+  "value": {
+    "id": "index-pattern:date_nanos_custom_timestamp",
+    "index": ".kibana",
+    "source": {
+      "index-pattern": {
+        "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"test\",\"type\":\"string\",\"esTypes\":[\"text\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"test.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"test\"}}},{\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date_nanos\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]",
+        "timeFieldName": "timestamp",
+        "title": "date_nanos_custom_timestamp"
+      },
+      "references": [
+      ],
+      "type": "index-pattern",
+      "updated_at": "2020-01-09T21:43:20.283Z"
+    }
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "1",
+    "index": "date_nanos_custom_timestamp",
+    "source": {
+      "test": "1",
+      "timestamp": "2019-10-21 00:30:04.828740"
+    }
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "2",
+    "index": "date_nanos_custom_timestamp",
+    "source": {
+      "test": "1",
+      "timestamp": "2019-10-21 08:30:04.828733"
+    }
+  }
+}
+
+{
+  "type": "doc",
+  "value": {
+    "id": "3",
+    "index": "date_nanos_custom_timestamp",
+    "source": {
+      "test": "1",
+      "timestamp": "2019-10-21 00:30:04.828723"
+    }
+  }
+}
+
+

test/functional/fixtures/es_archiver/date_nanos_custom/mappings.json

@@ -0,0 +1,31 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": "date_nanos_custom_timestamp",
+    "mappings": {
+      "properties": {
+        "test": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "type": "text"
+        },
+        "timestamp": {
+          "format": "yyyy-MM-dd HH:mm:ss.SSSSSS",
+          "type": "date_nanos"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}