[ML] DataFrame Analytics: Regression results view (#49667) (#49898)

* enable analytics table view link for regression jobs

* add results table component

* can filter for training/testing

* add search functionality to table

* move shared types to analytics types file

* anchor isTraining,predicted,actual columns to left.

* ensure search string persists in search bar input

* show jobStatus badge in results view

* add jobType, status badge to outlier exploration page

* update exploration tests

x-pack/legacy/plugins/ml/public/data_frame_analytics/_index.scss

@@ -1,4 +1,5 @@
 @import 'pages/analytics_exploration/components/exploration/index';
+@import 'pages/analytics_exploration/components/regression_exploration/index';
 @import 'pages/analytics_management/components/analytics_list/index';
 @import 'pages/analytics_management/components/create_analytics_form/index';
 @import 'pages/analytics_management/components/create_analytics_flyout/index';

x-pack/legacy/plugins/ml/public/data_frame_analytics/common/analytics.ts

@@ -29,6 +29,15 @@ interface RegressionAnalysis {
   };
 }
 
+export const SEARCH_SIZE = 1000;
+
+export enum INDEX_STATUS {
+  UNUSED,
+  LOADING,
+  LOADED,
+  ERROR,
+}
+
 export interface Eval {
   meanSquaredError: number | '';
   rSquared: number | '';
@@ -91,6 +100,16 @@ export const getPredictionFieldName = (analysis: AnalysisConfig) => {
   return predictionFieldName;
 };
 
+export const getPredictedFieldName = (resultsField: string, analysis: AnalysisConfig) => {
+  // default is 'ml'
+  const predictionFieldName = getPredictionFieldName(analysis);
+  const defaultPredictionField = `${getDependentVar(analysis)}_prediction`;
+  const predictedField = `${resultsField}.${
+    predictionFieldName ? predictionFieldName : defaultPredictionField
+  }`;
+  return predictedField;
+};
+
 export const isOutlierAnalysis = (arg: any): arg is OutlierAnalysis => {
   const keys = Object.keys(arg);
   return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION;

x-pack/legacy/plugins/ml/public/data_frame_analytics/common/fields.ts

@@ -5,6 +5,7 @@
  */
 
 import { getNestedProperty } from '../../util/object_utils';
+import { DataFrameAnalyticsConfig, getPredictedFieldName, getDependentVar } from './analytics';
 
 export type EsId = string;
 export type EsDocSource = Record<string, any>;
@@ -16,6 +17,7 @@ export interface EsDoc extends Record<string, any> {
 }
 
 export const MAX_COLUMNS = 20;
+export const DEFAULT_REGRESSION_COLUMNS = 8;
 
 const ML__ID_COPY = 'ml__id_copy';
 
@@ -68,6 +70,104 @@ export const sortColumns = (obj: EsDocSource, resultsField: string) => (a: strin
   return a.localeCompare(b);
 };
 
+export const sortRegressionResultsFields = (
+  a: string,
+  b: string,
+  jobConfig: DataFrameAnalyticsConfig
+) => {
+  const dependentVariable = getDependentVar(jobConfig.analysis);
+  const resultsField = jobConfig.dest.results_field;
+  const predictedField = getPredictedFieldName(resultsField, jobConfig.analysis);
+  if (a === `${resultsField}.is_training`) {
+    return -1;
+  }
+  if (b === `${resultsField}.is_training`) {
+    return 1;
+  }
+  if (a === predictedField) {
+    return -1;
+  }
+  if (b === predictedField) {
+    return 1;
+  }
+  if (a === dependentVariable) {
+    return -1;
+  }
+  if (b === dependentVariable) {
+    return 1;
+  }
+  return a.localeCompare(b);
+};
+
+// Used to sort columns:
+// Anchor on the left ml.is_training, <predictedField>, <actual>
+export const sortRegressionResultsColumns = (
+  obj: EsDocSource,
+  jobConfig: DataFrameAnalyticsConfig
+) => (a: string, b: string) => {
+  const dependentVariable = getDependentVar(jobConfig.analysis);
+  const resultsField = jobConfig.dest.results_field;
+  const predictedField = getPredictedFieldName(resultsField, jobConfig.analysis);
+
+  const typeofA = typeof obj[a];
+  const typeofB = typeof obj[b];
+
+  if (a === `${resultsField}.is_training`) {
+    return -1;
+  }
+
+  if (b === `${resultsField}.is_training`) {
+    return 1;
+  }
+
+  if (a === predictedField) {
+    return -1;
+  }
+
+  if (b === predictedField) {
+    return 1;
+  }
+
+  if (a === dependentVariable) {
+    return -1;
+  }
+
+  if (b === dependentVariable) {
+    return 1;
+  }
+
+  if (typeofA !== 'string' && typeofB === 'string') {
+    return 1;
+  }
+  if (typeofA === 'string' && typeofB !== 'string') {
+    return -1;
+  }
+  if (typeofA === 'string' && typeofB === 'string') {
+    return a.localeCompare(b);
+  }
+
+  const tokensA = a.split('.');
+  const prefixA = tokensA[0];
+  const tokensB = b.split('.');
+  const prefixB = tokensB[0];
+
+  if (prefixA === resultsField && tokensA.length > 1 && prefixB !== resultsField) {
+    tokensA.shift();
+    tokensA.shift();
+    if (tokensA.join('.') === b) return 1;
+    return tokensA.join('.').localeCompare(b);
+  }
+
+  if (prefixB === resultsField && tokensB.length > 1 && prefixA !== resultsField) {
+    tokensB.shift();
+    tokensB.shift();
+    if (tokensB.join('.') === a) return -1;
+    return a.localeCompare(tokensB.join('.'));
+  }
+
+  return a.localeCompare(b);
+};
+
 export function getFlattenedFields(obj: EsDocSource, resultsField: string): EsFieldName[] {
   const flatDocFields: EsFieldName[] = [];
   const newDocFields = Object.keys(obj);
@@ -84,6 +184,46 @@ export function getFlattenedFields(obj: EsDocSource, resultsField: string): EsFi
   return flatDocFields.filter(f => f !== ML__ID_COPY);
 }
 
+export const getDefaultRegressionFields = (
+  docs: EsDoc[],
+  jobConfig: DataFrameAnalyticsConfig
+): EsFieldName[] => {
+  const resultsField = jobConfig.dest.results_field;
+  if (docs.length === 0) {
+    return [];
+  }
+
+  const newDocFields = getFlattenedFields(docs[0]._source, resultsField);
+  return newDocFields
+    .filter(k => {
+      if (k === `${resultsField}.is_training`) {
+        return true;
+      }
+      // predicted value of dependent variable
+      if (k === getPredictedFieldName(resultsField, jobConfig.analysis)) {
+        return true;
+      }
+      // actual value of dependent variable
+      if (k === getDependentVar(jobConfig.analysis)) {
+        return true;
+      }
+      if (k.split('.')[0] === resultsField) {
+        return false;
+      }
+
+      let value = false;
+      docs.forEach(row => {
+        const source = row._source;
+        if (source[k] !== null) {
+          value = true;
+        }
+      });
+      return value;
+    })
+    .sort((a, b) => sortRegressionResultsFields(a, b, jobConfig))
+    .slice(0, DEFAULT_REGRESSION_COLUMNS);
+};
+
 export const getDefaultSelectableFields = (docs: EsDoc[], resultsField: string): EsFieldName[] => {
   if (docs.length === 0) {
     return [];

x-pack/legacy/plugins/ml/public/data_frame_analytics/common/index.ts

@@ -21,12 +21,18 @@ export {
   getValuesFromResponse,
   loadEvalData,
   Eval,
+  getPredictedFieldName,
+  INDEX_STATUS,
+  SEARCH_SIZE,
 } from './analytics';
 
 export {
   getDefaultSelectableFields,
+  getDefaultRegressionFields,
   getFlattenedFields,
   sortColumns,
+  sortRegressionResultsColumns,
+  sortRegressionResultsFields,
   toggleSelectedField,
   EsId,
   EsDoc,

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.test.tsx

@@ -6,6 +6,7 @@
 
 import { shallow } from 'enzyme';
 import React from 'react';
+import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
 
 jest.mock('../../../../../contexts/ui/use_ui_chrome_context');
 jest.mock('ui/new_platform');
@@ -20,7 +21,9 @@ jest.mock('react', () => {
 
 describe('Data Frame Analytics: <Exploration />', () => {
   test('Minimal initialization', () => {
-    const wrapper = shallow(<Exploration jobId="the-job-id" />);
+    const wrapper = shallow(
+      <Exploration jobId="the-job-id" jobStatus={DATA_FRAME_TASK_STATE.STOPPED} />
+    );
     // Without the jobConfig being loaded, the component will just return empty.
     expect(wrapper.text()).toMatch('');
     // TODO Once React 16.9 is available we can write tests covering asynchronous hooks.

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.tsx

@@ -50,10 +50,13 @@ import {
   EsFieldName,
   EsDoc,
   MAX_COLUMNS,
+  INDEX_STATUS,
 } from '../../../../common';
 
 import { getOutlierScoreFieldName } from './common';
-import { INDEX_STATUS, useExploreData } from './use_explore_data';
+import { useExploreData } from './use_explore_data';
+import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
+import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns';
 
 const customColorScaleFactory = (n: number) => (t: number) => {
   if (t < 1 / n) {
@@ -78,7 +81,7 @@ const ExplorationTitle: React.SFC<{ jobId: string }> = ({ jobId }) => (
   <EuiTitle size="xs">
     <span>
       {i18n.translate('xpack.ml.dataframe.analytics.exploration.jobIdTitle', {
-        defaultMessage: 'Job ID {jobId}',
+        defaultMessage: 'Outlier detection job ID {jobId}',
         values: { jobId },
       })}
     </span>
@@ -87,9 +90,10 @@ const ExplorationTitle: React.SFC<{ jobId: string }> = ({ jobId }) => (
 
 interface Props {
   jobId: string;
+  jobStatus: DATA_FRAME_TASK_STATE;
 }
 
-export const Exploration: FC<Props> = React.memo(({ jobId }) => {
+export const Exploration: FC<Props> = React.memo(({ jobId, jobStatus }) => {
   const [jobConfig, setJobConfig] = useState<DataFrameAnalyticsConfig | undefined>(undefined);
 
   const [pageIndex, setPageIndex] = useState(0);
@@ -378,7 +382,14 @@ export const Exploration: FC<Props> = React.memo(({ jobId }) => {
   if (status === INDEX_STATUS.LOADED && tableItems.length === 0) {
     return (
       <EuiPanel grow={false}>
-        <ExplorationTitle jobId={jobConfig.id} />
+        <EuiFlexGroup gutterSize="s">
+          <EuiFlexItem grow={false}>
+            <ExplorationTitle jobId={jobConfig.id} />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <span>{getTaskStateBadge(jobStatus)}</span>
+          </EuiFlexItem>
+        </EuiFlexGroup>
         <EuiCallOut
           title={i18n.translate('xpack.ml.dataframe.analytics.exploration.noDataCalloutTitle', {
             defaultMessage: 'Empty index query result.',
@@ -400,7 +411,14 @@ export const Exploration: FC<Props> = React.memo(({ jobId }) => {
     <EuiPanel grow={false}>
       <EuiFlexGroup alignItems="center" justifyContent="spaceBetween" responsive={false}>
         <EuiFlexItem grow={false}>
-          <ExplorationTitle jobId={jobConfig.id} />
+          <EuiFlexGroup gutterSize="s">
+            <EuiFlexItem grow={false}>
+              <ExplorationTitle jobId={jobConfig.id} />
+            </EuiFlexItem>
+            <EuiFlexItem grow={false}>
+              <span>{getTaskStateBadge(jobStatus)}</span>
+            </EuiFlexItem>
+          </EuiFlexGroup>
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiFlexGroup alignItems="center" gutterSize="xs" responsive={false}>

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/use_explore_data.ts

@@ -18,19 +18,12 @@ import {
   getFlattenedFields,
   DataFrameAnalyticsConfig,
   EsFieldName,
+  INDEX_STATUS,
+  SEARCH_SIZE,
 } from '../../../../common';
 
 import { getOutlierScoreFieldName } from './common';
 
-const SEARCH_SIZE = 1000;
-
-export enum INDEX_STATUS {
-  UNUSED,
-  LOADING,
-  LOADED,
-  ERROR,
-}
-
 type TableItem = Record<string, any>;
 
 interface LoadExploreDataArg {

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_index.scss

@@ -0,0 +1 @@
+@import 'regression_exploration';

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_regression_exploration.scss

@@ -0,0 +1,3 @@
+.mlRegressionExploration__evaluateLoadingSpinner {
+  display: inline-block;
+}

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx

@@ -7,13 +7,22 @@
 import React, { FC, Fragment, useEffect, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer, EuiStat, EuiTitle } from '@elastic/eui';
+import { idx } from '@kbn/elastic-idx';
 import { ErrorCallout } from './error_callout';
-import { getValuesFromResponse, loadEvalData, Eval } from '../../../../common';
+import {
+  getValuesFromResponse,
+  getDependentVar,
+  getPredictionFieldName,
+  loadEvalData,
+  Eval,
+  DataFrameAnalyticsConfig,
+} from '../../../../common';
+import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns';
+import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
 
 interface Props {
-  jobId: string;
-  index: string;
-  dependentVariable: string;
+  jobConfig: DataFrameAnalyticsConfig;
+  jobStatus: DATA_FRAME_TASK_STATE;
 }
 
 const meanSquaredErrorText = i18n.translate(
@@ -30,23 +39,28 @@ const rSquaredText = i18n.translate(
 );
 const defaultEval: Eval = { meanSquaredError: '', rSquared: '', error: null };
 
-export const EvaluatePanel: FC<Props> = ({ jobId, index, dependentVariable }) => {
+export const EvaluatePanel: FC<Props> = ({ jobConfig, jobStatus }) => {
   const [trainingEval, setTrainingEval] = useState<Eval>(defaultEval);
   const [generalizationEval, setGeneralizationEval] = useState<Eval>(defaultEval);
   const [isLoadingTraining, setIsLoadingTraining] = useState<boolean>(false);
   const [isLoadingGeneralization, setIsLoadingGeneralization] = useState<boolean>(false);
 
+  const index = idx(jobConfig, _ => _.dest.index) as string;
+  const dependentVariable = getDependentVar(jobConfig.analysis);
+  const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
+  // default is 'ml'
+  const resultsField = jobConfig.dest.results_field;
+
   const loadData = async () => {
     setIsLoadingGeneralization(true);
     setIsLoadingTraining(true);
-    // TODO: resultsField and predictionFieldName will need to be properly passed to this function
-    // once the results view is in use.
+
     const genErrorEval = await loadEvalData({
       isTraining: false,
       index,
       dependentVariable,
-      resultsField: 'ml',
-      predictionFieldName: undefined,
+      resultsField,
+      predictionFieldName,
     });
 
     if (genErrorEval.success === true && genErrorEval.eval) {
@@ -65,14 +79,13 @@ export const EvaluatePanel: FC<Props> = ({ jobId, index, dependentVariable }) =>
         error: genErrorEval.error,
       });
     }
-    // TODO: resultsField and predictionFieldName will need to be properly passed to this function
-    // once the results view is in use.
+
     const trainingErrorEval = await loadEvalData({
       isTraining: true,
       index,
       dependentVariable,
-      resultsField: 'ml',
-      predictionFieldName: undefined,
+      resultsField,
+      predictionFieldName,
     });
 
     if (trainingErrorEval.success === true && trainingErrorEval.eval) {
@@ -99,14 +112,21 @@ export const EvaluatePanel: FC<Props> = ({ jobId, index, dependentVariable }) =>
 
   return (
     <EuiPanel>
-      <EuiTitle size="xs">
-        <span>
-          {i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.jobIdTitle', {
-            defaultMessage: 'Job ID {jobId}',
-            values: { jobId },
-          })}
-        </span>
-      </EuiTitle>
+      <EuiFlexGroup gutterSize="s">
+        <EuiFlexItem grow={false}>
+          <EuiTitle size="xs">
+            <span>
+              {i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.jobIdTitle', {
+                defaultMessage: 'Regression job ID {jobId}',
+                values: { jobId: jobConfig.id },
+              })}
+            </span>
+          </EuiTitle>
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <span>{getTaskStateBadge(jobStatus)}</span>
+        </EuiFlexItem>
+      </EuiFlexGroup>
       <EuiSpacer size="m" />
       <EuiFlexGroup justifyContent="spaceBetween">
         <EuiFlexItem>

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx

@@ -4,25 +4,61 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { FC, Fragment } from 'react';
-// import { i18n } from '@kbn/i18n';
-import { EuiSpacer } from '@elastic/eui';
-
+import React, { FC, Fragment, useState, useEffect } from 'react';
+import { EuiSpacer, EuiLoadingSpinner, EuiPanel } from '@elastic/eui';
+import { ml } from '../../../../../services/ml_api_service';
+import { DataFrameAnalyticsConfig } from '../../../../common';
 import { EvaluatePanel } from './evaluate_panel';
-// import { ResultsTable } from './results_table';
+import { ResultsTable } from './results_table';
+import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
+
+interface GetDataFrameAnalyticsResponse {
+  count: number;
+  data_frame_analytics: DataFrameAnalyticsConfig[];
+}
+
+const LoadingPanel: FC = () => (
+  <EuiPanel>
+    <EuiLoadingSpinner className="mlRegressionExploration__evaluateLoadingSpinner" size="xl" />
+  </EuiPanel>
+);
 
 interface Props {
   jobId: string;
-  destIndex: string;
-  dependentVariable: string;
+  jobStatus: DATA_FRAME_TASK_STATE;
 }
 
-export const RegressionExploration: FC<Props> = ({ jobId, destIndex, dependentVariable }) => {
+export const RegressionExploration: FC<Props> = ({ jobId, jobStatus }) => {
+  const [jobConfig, setJobConfig] = useState<DataFrameAnalyticsConfig | undefined>(undefined);
+  const [isLoadingJobConfig, setIsLoadingJobConfig] = useState<boolean>(false);
+
+  useEffect(() => {
+    (async function() {
+      setIsLoadingJobConfig(true);
+      const analyticsConfigs: GetDataFrameAnalyticsResponse = await ml.dataFrameAnalytics.getDataFrameAnalytics(
+        jobId
+      );
+      if (
+        Array.isArray(analyticsConfigs.data_frame_analytics) &&
+        analyticsConfigs.data_frame_analytics.length > 0
+      ) {
+        setJobConfig(analyticsConfigs.data_frame_analytics[0]);
+        setIsLoadingJobConfig(false);
+      }
+    })();
+  }, []);
+
   return (
     <Fragment>
-      <EvaluatePanel jobId={jobId} index={destIndex} dependentVariable={dependentVariable} />
+      {isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
+      {isLoadingJobConfig === false && jobConfig !== undefined && (
+        <EvaluatePanel jobConfig={jobConfig} jobStatus={jobStatus} />
+      )}
       <EuiSpacer />
-      {/* <ResultsTable jobId={jobId} /> */}
+      {isLoadingJobConfig === true && jobConfig === undefined && <LoadingPanel />}
+      {isLoadingJobConfig === false && jobConfig !== undefined && (
+        <ResultsTable jobConfig={jobConfig} jobStatus={jobStatus} />
+      )}
     </Fragment>
   );
 };

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx

@@ -0,0 +1,482 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { Fragment, FC, useEffect, useState } from 'react';
+import moment from 'moment-timezone';
+
+import { i18n } from '@kbn/i18n';
+import {
+  EuiBadge,
+  EuiButtonIcon,
+  EuiCallOut,
+  EuiCheckbox,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiPanel,
+  EuiPopover,
+  EuiPopoverTitle,
+  EuiProgress,
+  EuiSpacer,
+  EuiText,
+  EuiTitle,
+  EuiToolTip,
+  Query,
+} from '@elastic/eui';
+
+import { Query as QueryType } from '../../../analytics_management/components/analytics_list/common';
+
+import {
+  ColumnType,
+  MlInMemoryTableBasic,
+  OnTableChangeArg,
+  SortingPropType,
+  SORT_DIRECTION,
+} from '../../../../../components/ml_in_memory_table';
+
+import { formatHumanReadableDateTimeSeconds } from '../../../../../util/date_utils';
+import { SavedSearchQuery } from '../../../../../contexts/kibana';
+
+import {
+  sortRegressionResultsColumns,
+  sortRegressionResultsFields,
+  toggleSelectedField,
+  DataFrameAnalyticsConfig,
+  EsFieldName,
+  EsDoc,
+  MAX_COLUMNS,
+  getPredictedFieldName,
+  INDEX_STATUS,
+} from '../../../../common';
+import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns';
+import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common';
+
+import { useExploreData, defaultSearchQuery } from './use_explore_data';
+
+const PAGE_SIZE_OPTIONS = [5, 10, 25, 50];
+
+const ExplorationTitle: React.SFC<{ jobId: string }> = ({ jobId }) => (
+  <EuiTitle size="xs">
+    <span>
+      {i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.jobIdTitle', {
+        defaultMessage: 'Regression job ID {jobId}',
+        values: { jobId },
+      })}
+    </span>
+  </EuiTitle>
+);
+
+interface Props {
+  jobConfig: DataFrameAnalyticsConfig;
+  jobStatus: DATA_FRAME_TASK_STATE;
+}
+
+export const ResultsTable: FC<Props> = React.memo(({ jobConfig, jobStatus }) => {
+  const [pageIndex, setPageIndex] = useState(0);
+  const [pageSize, setPageSize] = useState(25);
+  const [clearTable, setClearTable] = useState(false);
+  const [selectedFields, setSelectedFields] = useState([] as EsFieldName[]);
+  const [isColumnsPopoverVisible, setColumnsPopoverVisible] = useState(false);
+  const [searchQuery, setSearchQuery] = useState<SavedSearchQuery>(defaultSearchQuery);
+  const [searchError, setSearchError] = useState<any>(undefined);
+  const [searchString, setSearchString] = useState<string | undefined>(undefined);
+
+  // EuiInMemoryTable has an issue with dynamic sortable columns
+  // and will trigger a full page Kibana error in such a case.
+  // The following is a workaround until this is solved upstream:
+  // - If the sortable/columns config changes,
+  //   the table will be unmounted/not rendered.
+  //   This is what setClearTable(true) in toggleColumn() does.
+  // - After that on next render it gets re-enabled. To make sure React
+  //   doesn't consolidate the state updates, setTimeout is used.
+  if (clearTable) {
+    setTimeout(() => setClearTable(false), 0);
+  }
+
+  function toggleColumnsPopover() {
+    setColumnsPopoverVisible(!isColumnsPopoverVisible);
+  }
+
+  function closeColumnsPopover() {
+    setColumnsPopoverVisible(false);
+  }
+
+  function toggleColumn(column: EsFieldName) {
+    if (tableItems.length > 0 && jobConfig !== undefined) {
+      setClearTable(true);
+      // spread to a new array otherwise the component wouldn't re-render
+      setSelectedFields([...toggleSelectedField(selectedFields, column)]);
+    }
+  }
+
+  const {
+    errorMessage,
+    loadExploreData,
+    sortField,
+    sortDirection,
+    status,
+    tableItems,
+  } = useExploreData(jobConfig, selectedFields, setSelectedFields);
+
+  let docFields: EsFieldName[] = [];
+  let docFieldsCount = 0;
+  if (tableItems.length > 0) {
+    docFields = Object.keys(tableItems[0]);
+    docFields.sort((a, b) => sortRegressionResultsFields(a, b, jobConfig));
+    docFieldsCount = docFields.length;
+  }
+
+  const columns: ColumnType[] = [];
+
+  if (jobConfig !== undefined && selectedFields.length > 0 && tableItems.length > 0) {
+    columns.push(
+      ...selectedFields.sort(sortRegressionResultsColumns(tableItems[0], jobConfig)).map(k => {
+        const column: ColumnType = {
+          field: k,
+          name: k,
+          sortable: true,
+          truncateText: true,
+        };
+
+        const render = (d: any, fullItem: EsDoc) => {
+          if (Array.isArray(d) && d.every(item => typeof item === 'string')) {
+            // If the cells data is an array of strings, return as a comma separated list.
+            // The list will get limited to 5 items with `…` at the end if there's more in the original array.
+            return `${d.slice(0, 5).join(', ')}${d.length > 5 ? ', …' : ''}`;
+          } else if (Array.isArray(d)) {
+            // If the cells data is an array of e.g. objects, display a 'array' badge with a
+            // tooltip that explains that this type of field is not supported in this table.
+            return (
+              <EuiToolTip
+                content={i18n.translate(
+                  'xpack.ml.dataframe.analytics.regressionExploration.indexArrayToolTipContent',
+                  {
+                    defaultMessage:
+                      'The full content of this array based column cannot be displayed.',
+                  }
+                )}
+              >
+                <EuiBadge>
+                  {i18n.translate(
+                    'xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent',
+                    {
+                      defaultMessage: 'array',
+                    }
+                  )}
+                </EuiBadge>
+              </EuiToolTip>
+            );
+          } else if (typeof d === 'object' && d !== null) {
+            // If the cells data is an object, display a 'object' badge with a
+            // tooltip that explains that this type of field is not supported in this table.
+            return (
+              <EuiToolTip
+                content={i18n.translate(
+                  'xpack.ml.dataframe.analytics.regressionExploration.indexObjectToolTipContent',
+                  {
+                    defaultMessage:
+                      'The full content of this object based column cannot be displayed.',
+                  }
+                )}
+              >
+                <EuiBadge>
+                  {i18n.translate(
+                    'xpack.ml.dataframe.analytics.regressionExploration.indexObjectBadgeContent',
+                    {
+                      defaultMessage: 'object',
+                    }
+                  )}
+                </EuiBadge>
+              </EuiToolTip>
+            );
+          }
+
+          return d;
+        };
+
+        let columnType;
+
+        if (tableItems.length > 0) {
+          columnType = typeof tableItems[0][k];
+        }
+
+        if (typeof columnType !== 'undefined') {
+          switch (columnType) {
+            case 'boolean':
+              column.dataType = 'boolean';
+              break;
+            case 'Date':
+              column.align = 'right';
+              column.render = (d: any) =>
+                formatHumanReadableDateTimeSeconds(moment(d).unix() * 1000);
+              break;
+            case 'number':
+              column.dataType = 'number';
+              column.render = render;
+              break;
+            default:
+              column.render = render;
+              break;
+          }
+        } else {
+          column.render = render;
+        }
+
+        return column;
+      })
+    );
+  }
+
+  useEffect(() => {
+    if (jobConfig !== undefined) {
+      const predictedFieldName = getPredictedFieldName(
+        jobConfig.dest.results_field,
+        jobConfig.analysis
+      );
+      const predictedFieldSelected = selectedFields.includes(predictedFieldName);
+
+      const field = predictedFieldSelected ? predictedFieldName : selectedFields[0];
+      const direction = predictedFieldSelected ? SORT_DIRECTION.DESC : SORT_DIRECTION.ASC;
+      loadExploreData({ field, direction, searchQuery });
+      return;
+    }
+  }, [JSON.stringify(searchQuery)]);
+
+  useEffect(() => {
+    // by default set the sorting to descending on the prediction field (`<dependent_varible or prediction_field_name>_prediction`).
+    // if that's not available sort ascending on the first column.
+    // also check if the current sorting field is still available.
+    if (jobConfig !== undefined && columns.length > 0 && !selectedFields.includes(sortField)) {
+      const predictedFieldName = getPredictedFieldName(
+        jobConfig.dest.results_field,
+        jobConfig.analysis
+      );
+      const predictedFieldSelected = selectedFields.includes(predictedFieldName);
+
+      const field = predictedFieldSelected ? predictedFieldName : selectedFields[0];
+      const direction = predictedFieldSelected ? SORT_DIRECTION.DESC : SORT_DIRECTION.ASC;
+      loadExploreData({ field, direction, searchQuery });
+      return;
+    }
+  }, [jobConfig, columns.length, sortField, sortDirection, tableItems.length]);
+
+  let sorting: SortingPropType = false;
+  let onTableChange;
+
+  if (columns.length > 0 && sortField !== '' && sortField !== undefined) {
+    sorting = {
+      sort: {
+        field: sortField,
+        direction: sortDirection,
+      },
+    };
+
+    onTableChange = ({
+      page = { index: 0, size: 10 },
+      sort = { field: sortField, direction: sortDirection },
+    }: OnTableChangeArg) => {
+      const { index, size } = page;
+      setPageIndex(index);
+      setPageSize(size);
+
+      if (sort.field !== sortField || sort.direction !== sortDirection) {
+        setClearTable(true);
+        loadExploreData({ ...sort, searchQuery });
+      }
+    };
+  }
+
+  const pagination = {
+    initialPageIndex: pageIndex,
+    initialPageSize: pageSize,
+    totalItemCount: tableItems.length,
+    pageSizeOptions: PAGE_SIZE_OPTIONS,
+    hidePerPageOptions: false,
+  };
+
+  const onQueryChange = ({ query, error }: { query: QueryType; error: any }) => {
+    if (error) {
+      setSearchError(error.message);
+    } else {
+      try {
+        const esQueryDsl = Query.toESQuery(query);
+        setSearchQuery(esQueryDsl);
+        setSearchString(query.text);
+        setSearchError(undefined);
+      } catch (e) {
+        setSearchError(e.toString());
+      }
+    }
+  };
+
+  const search = {
+    onChange: onQueryChange,
+    defaultQuery: searchString,
+    box: {
+      incremental: false,
+      placeholder: i18n.translate(
+        'xpack.ml.dataframe.analytics.regressionExploration.searchBoxPlaceholder',
+        {
+          defaultMessage: 'E.g. avg>0.5',
+        }
+      ),
+    },
+    filters: [
+      {
+        type: 'field_value_toggle_group',
+        field: `${jobConfig.dest.results_field}.is_training`,
+        items: [
+          {
+            value: false,
+            name: i18n.translate(
+              'xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel',
+              {
+                defaultMessage: 'Testing',
+              }
+            ),
+          },
+          {
+            value: true,
+            name: i18n.translate(
+              'xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel',
+              {
+                defaultMessage: 'Training',
+              }
+            ),
+          },
+        ],
+      },
+    ],
+  };
+
+  if (jobConfig === undefined) {
+    return null;
+  }
+
+  if (status === INDEX_STATUS.ERROR) {
+    return (
+      <EuiPanel grow={false}>
+        <EuiFlexGroup gutterSize="s">
+          <EuiFlexItem grow={false}>
+            <ExplorationTitle jobId={jobConfig.id} />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <span>{getTaskStateBadge(jobStatus)}</span>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+        <EuiCallOut
+          title={i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.indexError', {
+            defaultMessage: 'An error occurred loading the index data.',
+          })}
+          color="danger"
+          iconType="cross"
+        >
+          <p>{errorMessage}</p>
+        </EuiCallOut>
+      </EuiPanel>
+    );
+  }
+
+  return (
+    <EuiPanel grow={false}>
+      <EuiFlexGroup alignItems="center" justifyContent="spaceBetween" responsive={false}>
+        <EuiFlexItem grow={false}>
+          <EuiFlexGroup gutterSize="s">
+            <EuiFlexItem grow={false}>
+              <ExplorationTitle jobId={jobConfig.id} />
+            </EuiFlexItem>
+            <EuiFlexItem grow={false}>
+              <span>{getTaskStateBadge(jobStatus)}</span>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <EuiFlexGroup alignItems="center" gutterSize="xs" responsive={false}>
+            <EuiFlexItem style={{ textAlign: 'right' }}>
+              {docFieldsCount > MAX_COLUMNS && (
+                <EuiText size="s">
+                  {i18n.translate(
+                    'xpack.ml.dataframe.analytics.regressionExploration.fieldSelection',
+                    {
+                      defaultMessage:
+                        '{selectedFieldsLength, number} of {docFieldsCount, number} {docFieldsCount, plural, one {field} other {fields}} selected',
+                      values: { selectedFieldsLength: selectedFields.length, docFieldsCount },
+                    }
+                  )}
+                </EuiText>
+              )}
+            </EuiFlexItem>
+            <EuiFlexItem grow={false}>
+              <EuiText size="s">
+                <EuiPopover
+                  id="popover"
+                  button={
+                    <EuiButtonIcon
+                      iconType="gear"
+                      onClick={toggleColumnsPopover}
+                      aria-label={i18n.translate(
+                        'xpack.ml.dataframe.analytics.regressionExploration.selectColumnsAriaLabel',
+                        {
+                          defaultMessage: 'Select columns',
+                        }
+                      )}
+                    />
+                  }
+                  isOpen={isColumnsPopoverVisible}
+                  closePopover={closeColumnsPopover}
+                  ownFocus
+                >
+                  <EuiPopoverTitle>
+                    {i18n.translate(
+                      'xpack.ml.dataframe.analytics.regressionExploration.selectFieldsPopoverTitle',
+                      {
+                        defaultMessage: 'Select fields',
+                      }
+                    )}
+                  </EuiPopoverTitle>
+                  <div style={{ maxHeight: '400px', overflowY: 'scroll' }}>
+                    {docFields.map(d => (
+                      <EuiCheckbox
+                        key={d}
+                        id={d}
+                        label={d}
+                        checked={selectedFields.includes(d)}
+                        onChange={() => toggleColumn(d)}
+                        disabled={selectedFields.includes(d) && selectedFields.length === 1}
+                      />
+                    ))}
+                  </div>
+                </EuiPopover>
+              </EuiText>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+      {status === INDEX_STATUS.LOADING && <EuiProgress size="xs" color="accent" />}
+      {status !== INDEX_STATUS.LOADING && (
+        <EuiProgress size="xs" color="accent" max={1} value={0} />
+      )}
+      {clearTable === false && (columns.length > 0 || searchQuery !== defaultSearchQuery) && (
+        <Fragment>
+          <EuiSpacer />
+          <MlInMemoryTableBasic
+            allowNeutralSort={false}
+            columns={columns}
+            compressed
+            hasActions={false}
+            isSelectable={false}
+            items={tableItems}
+            onTableChange={onTableChange}
+            pagination={pagination}
+            responsive={false}
+            search={search}
+            error={searchError}
+            sorting={sorting}
+          />
+        </Fragment>
+      )}
+    </EuiPanel>
+  );
+});

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts

@@ -0,0 +1,163 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useEffect, useState } from 'react';
+
+import { SearchResponse } from 'elasticsearch';
+
+import { SortDirection, SORT_DIRECTION } from '../../../../../components/ml_in_memory_table';
+
+import { ml } from '../../../../../services/ml_api_service';
+import { getNestedProperty } from '../../../../../util/object_utils';
+import { SavedSearchQuery } from '../../../../../contexts/kibana';
+
+import {
+  getDefaultRegressionFields,
+  getFlattenedFields,
+  DataFrameAnalyticsConfig,
+  EsFieldName,
+  getPredictedFieldName,
+  INDEX_STATUS,
+  SEARCH_SIZE,
+} from '../../../../common';
+
+export const defaultSearchQuery = {
+  match_all: {},
+};
+
+type TableItem = Record<string, any>;
+
+interface LoadExploreDataArg {
+  field: string;
+  direction: SortDirection;
+  searchQuery: SavedSearchQuery;
+}
+export interface UseExploreDataReturnType {
+  errorMessage: string;
+  loadExploreData: (arg: LoadExploreDataArg) => void;
+  sortField: EsFieldName;
+  sortDirection: SortDirection;
+  status: INDEX_STATUS;
+  tableItems: TableItem[];
+}
+
+interface SearchQuery {
+  query: SavedSearchQuery;
+  sort?: any;
+}
+
+export const useExploreData = (
+  jobConfig: DataFrameAnalyticsConfig | undefined,
+  selectedFields: EsFieldName[],
+  setSelectedFields: React.Dispatch<React.SetStateAction<EsFieldName[]>>
+): UseExploreDataReturnType => {
+  const [errorMessage, setErrorMessage] = useState('');
+  const [status, setStatus] = useState(INDEX_STATUS.UNUSED);
+  const [tableItems, setTableItems] = useState<TableItem[]>([]);
+  const [sortField, setSortField] = useState<string>('');
+  const [sortDirection, setSortDirection] = useState<SortDirection>(SORT_DIRECTION.ASC);
+
+  const loadExploreData = async ({ field, direction, searchQuery }: LoadExploreDataArg) => {
+    if (jobConfig !== undefined) {
+      setErrorMessage('');
+      setStatus(INDEX_STATUS.LOADING);
+
+      try {
+        const resultsField = jobConfig.dest.results_field;
+        const body: SearchQuery = {
+          query: searchQuery,
+        };
+
+        if (field !== undefined) {
+          body.sort = [
+            {
+              [field]: {
+                order: direction,
+              },
+            },
+          ];
+        }
+
+        const resp: SearchResponse<any> = await ml.esSearch({
+          index: jobConfig.dest.index,
+          size: SEARCH_SIZE,
+          body,
+        });
+
+        setSortField(field);
+        setSortDirection(direction);
+
+        const docs = resp.hits.hits;
+
+        if (docs.length === 0) {
+          setTableItems([]);
+          setStatus(INDEX_STATUS.LOADED);
+          return;
+        }
+
+        if (selectedFields.length === 0) {
+          const newSelectedFields = getDefaultRegressionFields(docs, jobConfig);
+          setSelectedFields(newSelectedFields);
+        }
+
+        // Create a version of the doc's source with flattened field names.
+        // This avoids confusion later on if a field name has dots in its name
+        // or is a nested fields when displaying it via EuiInMemoryTable.
+        const flattenedFields = getFlattenedFields(docs[0]._source, resultsField);
+        const transformedTableItems = docs.map(doc => {
+          const item: TableItem = {};
+          flattenedFields.forEach(ff => {
+            item[ff] = getNestedProperty(doc._source, ff);
+            if (item[ff] === undefined) {
+              // If the attribute is undefined, it means it was not a nested property
+              // but had dots in its actual name. This selects the property by its
+              // full name and assigns it to `item[ff]`.
+              item[ff] = doc._source[`"${ff}"`];
+            }
+            if (item[ff] === undefined) {
+              const parts = ff.split('.');
+              if (parts[0] === resultsField && parts.length >= 2) {
+                parts.shift();
+                if (doc._source[resultsField] !== undefined) {
+                  item[ff] = doc._source[resultsField][parts.join('.')];
+                }
+              }
+            }
+          });
+          return item;
+        });
+
+        setTableItems(transformedTableItems);
+        setStatus(INDEX_STATUS.LOADED);
+      } catch (e) {
+        if (e.message !== undefined) {
+          setErrorMessage(e.message);
+        } else {
+          setErrorMessage(JSON.stringify(e));
+        }
+        setTableItems([]);
+        setStatus(INDEX_STATUS.ERROR);
+      }
+    }
+  };
+
+  useEffect(() => {
+    if (jobConfig !== undefined) {
+      loadExploreData({
+        field: getPredictedFieldName(jobConfig.dest.results_field, jobConfig.analysis),
+        direction: SORT_DIRECTION.DESC,
+        searchQuery: defaultSearchQuery,
+      });
+    }
+  }, [jobConfig && jobConfig.id]);
+
+  return { errorMessage, loadExploreData, sortField, sortDirection, status, tableItems };
+};

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/directive.tsx

@@ -53,8 +53,7 @@ module.directive('mlDataFrameAnalyticsExploration', ($injector: InjectorService)
             <Page
               jobId={globalState.ml.jobId}
               analysisType={globalState.ml.analysisType}
-              destIndex={globalState.ml.destIndex}
-              depVar={globalState.ml.depVar}
+              jobStatus={globalState.ml.jobStatus}
             />
           </KibanaContext.Provider>
         </I18nContext>,

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/page.tsx

@@ -26,13 +26,13 @@ import { Exploration } from './components/exploration';
 import { RegressionExploration } from './components/regression_exploration';
 
 import { ANALYSIS_CONFIG_TYPE } from '../../common/analytics';
+import { DATA_FRAME_TASK_STATE } from '../analytics_management/components/analytics_list/common';
 
 export const Page: FC<{
   jobId: string;
-  analysisType: string;
-  destIndex: string;
-  depVar: string;
-}> = ({ jobId, analysisType, destIndex, depVar }) => (
+  analysisType: ANALYSIS_CONFIG_TYPE;
+  jobStatus: DATA_FRAME_TASK_STATE;
+}> = ({ jobId, analysisType, jobStatus }) => (
   <Fragment>
     <NavigationMenu tabId="data_frame_analytics" />
     <EuiPage data-test-subj="mlPageDataFrameAnalyticsExploration">
@@ -66,9 +66,11 @@ export const Page: FC<{
         </EuiPageContentHeader>
         <EuiPageContentBody>
           <EuiSpacer size="l" />
-          {analysisType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION && <Exploration jobId={jobId} />}
+          {analysisType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION && (
+            <Exploration jobId={jobId} jobStatus={jobStatus} />
+          )}
           {analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION && (
-            <RegressionExploration jobId={jobId} destIndex={destIndex} dependentVariable={depVar} />
+            <RegressionExploration jobId={jobId} jobStatus={jobStatus} />
           )}
         </EuiPageContentBody>
       </EuiPageBody>

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/actions.tsx

@@ -13,7 +13,7 @@ import {
   createPermissionFailureMessage,
 } from '../../../../../privilege/check_privilege';
 
-import { isOutlierAnalysis, getAnalysisType, getDependentVar } from '../../../../common/analytics';
+import { getAnalysisType } from '../../../../common/analytics';
 
 import { getResultsUrl, isDataFrameAnalyticsRunning, DataFrameAnalyticsListRow } from './common';
 import { stopAnalytics } from '../../services/analytics_service';
@@ -25,14 +25,11 @@ export const AnalyticsViewAction = {
   isPrimary: true,
   render: (item: DataFrameAnalyticsListRow) => {
     const analysisType = getAnalysisType(item.config.analysis);
-    const destIndex = item.config.dest.index;
-    const dependentVariable = getDependentVar(item.config.analysis);
+    const jobStatus = item.stats.state;
 
-    const url = getResultsUrl(item.id, analysisType, destIndex, dependentVariable);
-    // Disable 'View' link for regression until results view is complete
+    const url = getResultsUrl(item.id, analysisType, jobStatus);
     return (
       <EuiButtonEmpty
-        disabled={!isOutlierAnalysis(item.config.analysis)}
         onClick={() => (window.location.href = url)}
         size="xs"
         color="text"

x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts

@@ -5,7 +5,6 @@
  */
 
 import { DataFrameAnalyticsId, DataFrameAnalyticsConfig } from '../../../../common';
-import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics';
 
 export enum DATA_FRAME_TASK_STATE {
   ANALYZING = 'analyzing',
@@ -118,17 +117,6 @@ export function isCompletedAnalyticsJob(stats: DataFrameAnalyticsStats) {
   return stats.state === DATA_FRAME_TASK_STATE.STOPPED && progress === 100;
 }
 
-export function getResultsUrl(
-  jobId: string,
-  analysisType: string,
-  destIndex: string = '',
-  dependentVariable: string = ''
-) {
-  const destIndexParam = `,destIndex:${destIndex}`;
-  const depVarParam = `,depVar:${dependentVariable}`;
-  const isRegression = analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION;
-
-  return `ml#/data_frame_analytics/exploration?_g=(ml:(jobId:${jobId},analysisType:${analysisType}${
-    isRegression && destIndex !== '' ? destIndexParam : ''
-  }${isRegression && dependentVariable !== '' ? depVarParam : ''}))`;
+export function getResultsUrl(jobId: string, analysisType: string, status: DATA_FRAME_TASK_STATE) {
+  return `ml#/data_frame_analytics/exploration?_g=(ml:(jobId:${jobId},analysisType:${analysisType},jobStatus:${status}))`;
 }