[ML] Data Frame Analytics: adds support for runtime fields (#95734)

* add runtime mapping editor in wizard * ensure depVar is updated correctly with RF changes * remove old RF from includes * ensure cloning works with RF as depVar * ensure indexPattern RF work * scatterplot supports RTF. depVar options have indexPattern RTF on first load * remove unnecessary types * ensure supported fields included by default * update types in editor * use isRuntimeMappings * fix translations. ensure runtimeMappings persist when going back to step 1 * ensure histograms support runtime fields * update types
2025-04-24 09:48:58 -04:00 · 2021-04-02 11:06:31 -04:00 · 2021-04-02 11:06:31 -04:00 · 8fef5fd9e1
commit 8fef5fd9e1
parent fe3ec69f9e
20 changed files with 766 additions and 166 deletions
--- a/x-pack/plugins/ml/common/types/data_frame_analytics.ts
+++ b/x-pack/plugins/ml/common/types/data_frame_analytics.ts
@ -6,6 +6,8 @@
 */

 import Boom from '@hapi/boom';
+import { RuntimeMappings } from './fields';
+
 import { EsErrorBody } from '../util/errors';
 import { ANALYSIS_CONFIG_TYPE } from '../constants/data_frame_analytics';
 import { DATA_FRAME_TASK_STATE } from '../constants/data_frame_analytics';
@ -74,6 +76,7 @@ export interface DataFrameAnalyticsConfig {
  source: {
    index: IndexName | IndexName[];
    query?: any;
+    runtime_mappings?: RuntimeMappings;
  };
  analysis: AnalysisConfig;
  analyzed_fields: {
--- a/x-pack/plugins/ml/common/types/fields.ts
+++ b/x-pack/plugins/ml/common/types/fields.ts
@ -109,8 +109,8 @@ export interface AggCardinality {
 export type RollupFields = Record<FieldId, [Record<'agg', ES_AGGREGATION>]>;

 // Replace this with import once #88995 is merged
-const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
-type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
+export const RUNTIME_FIELD_TYPES = ['keyword', 'long', 'double', 'date', 'ip', 'boolean'] as const;
+export type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];

 export interface RuntimeField {
  type: RuntimeType;
--- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts
@ -49,9 +49,8 @@ import { getNestedProperty } from '../../util/object_utils';
 import { mlFieldFormatService } from '../../services/field_format_service';

 import { DataGridItem, IndexPagination, RenderCellValue } from './types';
-import type { RuntimeField } from '../../../../../../../src/plugins/data/common/index_patterns';
-import { RuntimeMappings } from '../../../../common/types/fields';
-import { isPopulatedObject } from '../../../../common/util/object_utils';
+import { RuntimeMappings, RuntimeField } from '../../../../common/types/fields';
+import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils';

 export const INIT_MAX_COLUMNS = 10;
 export const COLUMN_CHART_DEFAULT_VISIBILITY_ROWS_THRESHOLED = 10000;
@ -94,34 +93,36 @@ export const getFieldsFromKibanaIndexPattern = (indexPattern: IndexPattern): str
 /**
 * Return a map of runtime_mappings for each of the index pattern field provided
 * to provide in ES search queries
- * @param indexPatternFields
 * @param indexPattern
- * @param clonedRuntimeMappings
+ * @param RuntimeMappings
 */
-export const getRuntimeFieldsMapping = (
-  indexPatternFields: string[] | undefined,
+export function getCombinedRuntimeMappings(
  indexPattern: IndexPattern | undefined,
-  clonedRuntimeMappings?: RuntimeMappings
-) => {
-  if (!Array.isArray(indexPatternFields) || indexPattern === undefined) return {};
-  const ipRuntimeMappings = indexPattern.getComputedFields().runtimeFields;
-  let combinedRuntimeMappings: RuntimeMappings = {};
+  runtimeMappings?: RuntimeMappings
+): RuntimeMappings | undefined {
+  let combinedRuntimeMappings = {};

-  if (isPopulatedObject(ipRuntimeMappings)) {
-    indexPatternFields.forEach((ipField) => {
-      if (ipRuntimeMappings.hasOwnProperty(ipField)) {
-        // @ts-expect-error
-        combinedRuntimeMappings[ipField] = ipRuntimeMappings[ipField];
+  // And runtime field mappings defined by index pattern
+  if (indexPattern) {
+    const computedFields = indexPattern?.getComputedFields();
+    if (computedFields?.runtimeFields !== undefined) {
+      const indexPatternRuntimeMappings = computedFields.runtimeFields;
+      if (isRuntimeMappings(indexPatternRuntimeMappings)) {
+        combinedRuntimeMappings = { ...combinedRuntimeMappings, ...indexPatternRuntimeMappings };
      }
-    });
+    }
  }
-  if (isPopulatedObject(clonedRuntimeMappings)) {
-    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...clonedRuntimeMappings };
+
+  // Use runtime field mappings defined inline from API
+  // and override fields with same name from index pattern
+  if (isRuntimeMappings(runtimeMappings)) {
+    combinedRuntimeMappings = { ...combinedRuntimeMappings, ...runtimeMappings };
  }
-  return Object.keys(combinedRuntimeMappings).length > 0
-    ? { runtime_mappings: combinedRuntimeMappings }
-    : {};
-};
+
+  if (isRuntimeMappings(combinedRuntimeMappings)) {
+    return combinedRuntimeMappings;
+  }
+}

 export interface FieldTypes {
  [key: string]: ES_FIELD_TYPES;
--- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts
+++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts
@ -10,7 +10,7 @@ export {
  getDataGridSchemaFromESFieldType,
  getDataGridSchemaFromKibanaFieldType,
  getFieldsFromKibanaIndexPattern,
-  getRuntimeFieldsMapping,
+  getCombinedRuntimeMappings,
  multiColumnSortFactory,
  showDataGridColumnChartErrorMessageToast,
  useRenderCellValue,
--- a/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx
+++ b/x-pack/plugins/ml/public/application/components/scatterplot_matrix/scatterplot_matrix.tsx
@ -24,9 +24,13 @@ import {

 import { i18n } from '@kbn/i18n';

+import { IndexPattern } from '../../../../../../../src/plugins/data/public';
 import { extractErrorMessage } from '../../../../common';
+import { isRuntimeMappings } from '../../../../common/util/runtime_field_utils';
 import { stringHash } from '../../../../common/util/string_utils';
+import { RuntimeMappings } from '../../../../common/types/fields';
 import type { ResultsSearchQuery } from '../../data_frame_analytics/common/analytics';
+import { getCombinedRuntimeMappings } from '../../components/data_grid';

 import { useMlApiContext } from '../../contexts/kibana';

@ -84,6 +88,8 @@ export interface ScatterplotMatrixProps {
  color?: string;
  legendType?: LegendType;
  searchQuery?: ResultsSearchQuery;
+  runtimeMappings?: RuntimeMappings;
+  indexPattern?: IndexPattern;
 }

 export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
@ -93,6 +99,8 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
  color,
  legendType,
  searchQuery,
+  runtimeMappings,
+  indexPattern,
 }) => {
  const { esSearch } = useMlApiContext();

@ -185,6 +193,9 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
            }
          : searchQuery;

+        const combinedRuntimeMappings =
+          indexPattern && getCombinedRuntimeMappings(indexPattern, runtimeMappings);
+
        const resp: estypes.SearchResponse = await esSearch({
          index,
          body: {
@ -193,6 +204,9 @@ export const ScatterplotMatrix: FC<ScatterplotMatrixProps> = ({
            query,
            from: 0,
            size: fetchSize,
+            ...(isRuntimeMappings(combinedRuntimeMappings)
+              ? { runtime_mappings: combinedRuntimeMappings }
+              : {}),
          },
        });

--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step.tsx
@ -13,12 +13,17 @@ import { ConfigurationStepDetails } from './configuration_step_details';
 import { ConfigurationStepForm } from './configuration_step_form';
 import { ANALYTICS_STEPS } from '../../page';

-export const ConfigurationStep: FC<CreateAnalyticsStepProps> = ({
+export interface ConfigurationStepProps extends CreateAnalyticsStepProps {
+  isClone: boolean;
+}
+
+export const ConfigurationStep: FC<ConfigurationStepProps> = ({
  actions,
  state,
  setCurrentStep,
  step,
  stepActivated,
+  isClone,
 }) => {
  const showForm = step === ANALYTICS_STEPS.CONFIGURATION;
  const showDetails = step !== ANALYTICS_STEPS.CONFIGURATION && stepActivated === true;
@ -30,7 +35,12 @@ export const ConfigurationStep: FC<CreateAnalyticsStepProps> = ({
  return (
    <EuiForm className="mlDataFrameAnalyticsCreateForm" data-test-subj={dataTestSubj}>
      {showForm && (
-        <ConfigurationStepForm actions={actions} state={state} setCurrentStep={setCurrentStep} />
+        <ConfigurationStepForm
+          actions={actions}
+          isClone={isClone}
+          state={state}
+          setCurrentStep={setCurrentStep}
+        />
      )}
      {showDetails && <ConfigurationStepDetails setCurrentStep={setCurrentStep} state={state} />}
    </EuiForm>
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx
@ -5,10 +5,9 @@
 * 2.0.
 */

-import React, { FC, Fragment, useEffect, useMemo, useRef, useState } from 'react';
+import React, { FC, Fragment, useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import {
  EuiBadge,
-  EuiCallOut,
  EuiComboBox,
  EuiComboBoxOptionOption,
  EuiFormRow,
@ -18,11 +17,11 @@ import {
  EuiText,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
-import { debounce } from 'lodash';
+import { debounce, cloneDeep } from 'lodash';

-import { FormattedMessage } from '@kbn/i18n/react';
 import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
 import { useMlContext } from '../../../../../contexts/ml';
+import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common';

 import {
  ANALYSIS_CONFIG_TYPE,
@ -31,13 +30,18 @@ import {
  FieldSelectionItem,
 } from '../../../../common/analytics';
 import { getScatterplotMatrixLegendType } from '../../../../common/get_scatterplot_matrix_legend_type';
-import { CreateAnalyticsStepProps } from '../../../analytics_management/hooks/use_create_analytics_form';
+import { RuntimeMappings as RuntimeMappingsType } from '../../../../../../../common/types/fields';
+import {
+  isRuntimeMappings,
+  isRuntimeField,
+} from '../../../../../../../common/util/runtime_field_utils';
+import { AnalyticsJobType } from '../../../analytics_management/hooks/use_create_analytics_form/state';
 import { Messages } from '../shared';
 import {
  DEFAULT_MODEL_MEMORY_LIMIT,
  State,
 } from '../../../analytics_management/hooks/use_create_analytics_form/state';
-import { shouldAddAsDepVarOption } from './form_options_validation';
+import { handleExplainErrorMessage, shouldAddAsDepVarOption } from './form_options_validation';
 import { getToastNotifications } from '../../../../../util/dependency_cache';

 import { ANALYTICS_STEPS } from '../../page';
@ -55,6 +59,18 @@ import { ExplorationQueryBarProps } from '../../../analytics_exploration/compone
 import { Query } from '../../../../../../../../../../src/plugins/data/common/query';

 import { ScatterplotMatrix } from '../../../../../components/scatterplot_matrix';
+import { RuntimeMappings } from '../runtime_mappings';
+import { ConfigurationStepProps } from './configuration_step';
+
+const runtimeMappingKey = 'runtime_mapping';
+const notIncludedReason = 'field not in includes list';
+const requiredFieldsErrorText = i18n.translate(
+  'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage',
+  {
+    defaultMessage:
+      'At least one field must be included in the analysis in addition to the dependent variable.',
+  }
+);

 function getIndexDataQuery(savedSearchQuery: SavedSearchQuery, jobConfigQuery: any) {
  // Return `undefined` if savedSearchQuery itself is `undefined`, meaning it hasn't been initialized yet.
@ -65,18 +81,23 @@ function getIndexDataQuery(savedSearchQuery: SavedSearchQuery, jobConfigQuery: a
  return savedSearchQuery !== null ? savedSearchQuery : jobConfigQuery;
 }

-const requiredFieldsErrorText = i18n.translate(
-  'xpack.ml.dataframe.analytics.createWizard.requiredFieldsErrorMessage',
-  {
-    defaultMessage:
-      'At least one field must be included in the analysis in addition to the dependent variable.',
-  }
-);
+function getRuntimeDepVarOptions(jobType: AnalyticsJobType, runtimeMappings: RuntimeMappingsType) {
+  const runtimeOptions: EuiComboBoxOptionOption[] = [];
+  Object.keys(runtimeMappings).forEach((id) => {
+    const field = runtimeMappings[id];
+    if (isRuntimeField(field) && shouldAddAsDepVarOption(id, field.type, jobType)) {
+      runtimeOptions.push({
+        label: id,
+        key: `runtime_mapping_${id}`,
+      });
+    }
+  });
+  return runtimeOptions;
+}

-const maxRuntimeFieldsDisplayCount = 5;
-
-export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
+export const ConfigurationStepForm: FC<ConfigurationStepProps> = ({
  actions,
+  isClone,
  state,
  setCurrentStep,
 }) => {
@ -100,7 +121,7 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
  >();

  const { setEstimatedModelMemoryLimit, setFormState } = actions;
-  const { estimatedModelMemoryLimit, form, isJobCreated, requestMessages } = state;
+  const { cloneJob, estimatedModelMemoryLimit, form, isJobCreated, requestMessages } = state;
  const firstUpdate = useRef<boolean>(true);
  const {
    dependentVariable,
@ -111,10 +132,22 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
    modelMemoryLimit,
    previousJobType,
    requiredFieldsError,
+    runtimeMappings,
+    previousRuntimeMapping,
+    runtimeMappingsUpdated,
    sourceIndex,
    trainingPercent,
    useEstimatedMml,
  } = form;
+
+  const isJobTypeWithDepVar =
+    jobType === ANALYSIS_CONFIG_TYPE.REGRESSION || jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION;
+  const dependentVariableEmpty = isJobTypeWithDepVar && dependentVariable === '';
+  const hasBasicRequiredFields = jobType !== undefined;
+  const hasRequiredAnalysisFields =
+    (isJobTypeWithDepVar && dependentVariable !== '') ||
+    jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION;
+
  const [query, setQuery] = useState<Query>({
    query: jobConfigQueryString ?? '',
    language: SEARCH_QUERY_LANGUAGE.KUERY,
@ -132,7 +165,8 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
  const indexData = useIndexData(
    currentIndexPattern,
    getIndexDataQuery(savedSearchQuery, jobConfigQuery),
-    toastNotifications
+    toastNotifications,
+    runtimeMappings
  );

  const indexPreviewProps = {
@ -141,11 +175,6 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
    toastNotifications,
  };

-  const isJobTypeWithDepVar =
-    jobType === ANALYSIS_CONFIG_TYPE.REGRESSION || jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION;
-
-  const dependentVariableEmpty = isJobTypeWithDepVar && dependentVariable === '';
-
  const isStepInvalid =
    dependentVariableEmpty ||
    jobType === undefined ||
@ -155,20 +184,23 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
    unsupportedFieldsError !== undefined ||
    fetchingExplainData;

-  const loadDepVarOptions = async (formState: State['form']) => {
+  const loadDepVarOptions = async (
+    formState: State['form'],
+    runtimeOptions: EuiComboBoxOptionOption[] = []
+  ) => {
    setLoadingDepVarOptions(true);
    setMaxDistinctValuesError(undefined);

    try {
      if (currentIndexPattern !== undefined) {
        const depVarOptions = [];
-        let depVarUpdate = dependentVariable;
+        let depVarUpdate = formState.dependentVariable;
        // Get fields and filter for supported types for job type
        const { fields } = newJobCapsService;

        let resetDependentVariable = true;
        for (const field of fields) {
-          if (shouldAddAsDepVarOption(field, jobType)) {
+          if (shouldAddAsDepVarOption(field.id, field.type, jobType)) {
            depVarOptions.push({
              label: field.id,
            });
@ -179,10 +211,21 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
          }
        }

+        if (
+          isRuntimeMappings(formState.runtimeMappings) &&
+          Object.keys(formState.runtimeMappings).includes(form.dependentVariable)
+        ) {
+          resetDependentVariable = false;
+          depVarOptions.push({
+            label: form.dependentVariable,
+            key: `runtime_mapping_${form.dependentVariable}`,
+          });
+        }
+
        if (resetDependentVariable) {
          depVarUpdate = '';
        }
-        setDependentVariableOptions(depVarOptions);
+        setDependentVariableOptions([...runtimeOptions, ...depVarOptions]);
        setLoadingDepVarOptions(false);
        setDependentVariableFetchFail(false);
        setFormState({ dependentVariable: depVarUpdate });
@ -209,8 +252,23 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
    if (jobTypeChanged) {
      setLoadingFieldOptions(true);
    }
+    // Ensure runtime field is in 'includes' table if it is set as dependent variable
+    const depVarIsRuntimeField =
+      isJobTypeWithDepVar &&
+      runtimeMappings &&
+      Object.keys(runtimeMappings).includes(dependentVariable) &&
+      includes.length > 0 &&
+      includes.includes(dependentVariable) === false;
+    let formToUse = form;

-    const { success, expectedMemory, fieldSelection, errorMessage } = await fetchExplainData(form);
+    if (depVarIsRuntimeField) {
+      formToUse = cloneDeep(form);
+      formToUse.includes = [...includes, dependentVariable];
+    }
+
+    const { success, expectedMemory, fieldSelection, errorMessage } = await fetchExplainData(
+      formToUse
+    );

    if (success) {
      if (shouldUpdateEstimatedMml) {
@ -226,53 +284,33 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
        setFieldOptionsFetchFail(false);
        setMaxDistinctValuesError(undefined);
        setUnsupportedFieldsError(undefined);
-        setIncludesTableItems(fieldSelection ? fieldSelection : []);
        setFormState({
          ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}),
          requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined,
+          includes: formToUse.includes,
        });
+        setIncludesTableItems(fieldSelection ? fieldSelection : []);
      } else {
        setFormState({
          ...(shouldUpdateModelMemoryLimit ? { modelMemoryLimit: expectedMemory } : {}),
          requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined,
+          includes: formToUse.includes,
        });
      }
      setFetchingExplainData(false);
    } else {
-      let maxDistinctValuesErrorMessage;
-      let unsupportedFieldsErrorMessage;
-      if (
-        jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION &&
-        (errorMessage.includes('must have at most') || errorMessage.includes('must have at least'))
-      ) {
-        maxDistinctValuesErrorMessage = errorMessage;
-      } else if (
-        errorMessage.includes('status_exception') &&
-        errorMessage.includes('unsupported type')
-      ) {
-        unsupportedFieldsErrorMessage = errorMessage;
-      } else if (
-        errorMessage.includes('status_exception') &&
-        errorMessage.includes('Unable to estimate memory usage as no documents')
-      ) {
-        toastNotifications.addWarning(
-          i18n.translate('xpack.ml.dataframe.analytics.create.allDocsMissingFieldsErrorMessage', {
-            defaultMessage: `Unable to estimate memory usage. There are mapped fields for source index [{index}] that do not exist in any indexed documents. You will have to switch to the JSON editor for explicit field selection and include only fields that exist in indexed documents.`,
-            values: {
-              index: sourceIndex,
-            },
-          })
-        );
-      } else {
-        toastNotifications.addDanger({
-          title: i18n.translate(
-            'xpack.ml.dataframe.analytics.create.unableToFetchExplainDataMessage',
-            {
-              defaultMessage: 'An error occurred fetching analysis fields data.',
-            }
-          ),
-          text: errorMessage,
-        });
+      const {
+        maxDistinctValuesErrorMessage,
+        unsupportedFieldsErrorMessage,
+        toastNotificationDanger,
+        toastNotificationWarning,
+      } = handleExplainErrorMessage(errorMessage, sourceIndex, jobType);
+
+      if (toastNotificationDanger) {
+        toastNotifications.addDanger(toastNotificationDanger);
+      }
+      if (toastNotificationWarning) {
+        toastNotifications.addWarning(toastNotificationWarning);
      }

      const fallbackModelMemoryLimit =
@ -304,17 +342,126 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({

  useEffect(() => {
    if (isJobTypeWithDepVar) {
-      loadDepVarOptions(form);
+      const indexPatternRuntimeFields = getCombinedRuntimeMappings(currentIndexPattern);
+      let runtimeOptions;
+
+      if (indexPatternRuntimeFields) {
+        runtimeOptions = getRuntimeDepVarOptions(jobType, indexPatternRuntimeFields);
+      }
+
+      loadDepVarOptions(form, runtimeOptions);
    }
  }, [jobType]);

+  const handleRuntimeUpdate = useCallback(async () => {
+    if (runtimeMappingsUpdated) {
+      // Update dependent variable options
+      let resetDepVar = false;
+      if (isJobTypeWithDepVar) {
+        const filteredOptions = dependentVariableOptions.filter((option) => {
+          if (option.label === dependentVariable && option.key?.includes(runtimeMappingKey)) {
+            resetDepVar = true;
+          }
+          return !option.key?.includes(runtimeMappingKey);
+        });
+        // Runtime mappings have been removed
+        if (runtimeMappings === undefined && runtimeMappingsUpdated === true) {
+          setDependentVariableOptions(filteredOptions);
+        } else if (runtimeMappings) {
+          // add to filteredOptions if it's the type supported
+          const runtimeOptions = getRuntimeDepVarOptions(jobType, runtimeMappings);
+          setDependentVariableOptions([...filteredOptions, ...runtimeOptions]);
+        }
+      }
+
+      // Update includes - remove previous runtime mappings then add supported runtime fields to includes
+      const updatedIncludes = includes.filter((field) => {
+        const isRemovedRuntimeField = previousRuntimeMapping && previousRuntimeMapping[field];
+        return !isRemovedRuntimeField;
+      });
+
+      if (resetDepVar) {
+        setFormState({
+          dependentVariable: '',
+          includes: updatedIncludes,
+        });
+        setIncludesTableItems(
+          includesTableItems.filter(({ name }) => {
+            const isRemovedRuntimeField = previousRuntimeMapping && previousRuntimeMapping[name];
+            return !isRemovedRuntimeField;
+          })
+        );
+      }
+
+      if (!resetDepVar && hasBasicRequiredFields && hasRequiredAnalysisFields) {
+        const formCopy = cloneDeep(form);
+        // When switching back to step ensure runtime field is in 'includes' table if it is set as dependent variable
+        const depVarIsRuntimeField =
+          isJobTypeWithDepVar &&
+          runtimeMappings &&
+          Object.keys(runtimeMappings).includes(dependentVariable) &&
+          formCopy.includes.length > 0 &&
+          formCopy.includes.includes(dependentVariable) === false;
+
+        formCopy.includes = depVarIsRuntimeField
+          ? [...updatedIncludes, dependentVariable]
+          : updatedIncludes;
+
+        const { success, fieldSelection, errorMessage } = await fetchExplainData(formCopy);
+        if (success) {
+          // update the field selection table
+          const hasRequiredFields = fieldSelection.some(
+            (field) => field.is_included === true && field.is_required === false
+          );
+          let updatedFieldSelection;
+          // Update field selection to select supported runtime fields by default. Add those fields to 'includes'.
+          if (isRuntimeMappings(runtimeMappings)) {
+            updatedFieldSelection = fieldSelection.map((field) => {
+              if (
+                runtimeMappings[field.name] !== undefined &&
+                field.is_included === false &&
+                field.reason?.includes(notIncludedReason)
+              ) {
+                updatedIncludes.push(field.name);
+                field.is_included = true;
+              }
+              return field;
+            });
+          }
+          setIncludesTableItems(updatedFieldSelection ? updatedFieldSelection : fieldSelection);
+          setMaxDistinctValuesError(undefined);
+          setUnsupportedFieldsError(undefined);
+          setFormState({
+            includes: updatedIncludes,
+            requiredFieldsError: !hasRequiredFields ? requiredFieldsErrorText : undefined,
+          });
+        } else {
+          const {
+            maxDistinctValuesErrorMessage,
+            unsupportedFieldsErrorMessage,
+            toastNotificationDanger,
+            toastNotificationWarning,
+          } = handleExplainErrorMessage(errorMessage, sourceIndex, jobType);
+
+          if (toastNotificationDanger) {
+            toastNotifications.addDanger(toastNotificationDanger);
+          }
+          if (toastNotificationWarning) {
+            toastNotifications.addWarning(toastNotificationWarning);
+          }
+
+          setMaxDistinctValuesError(maxDistinctValuesErrorMessage);
+          setUnsupportedFieldsError(unsupportedFieldsErrorMessage);
+        }
+      }
+    }
+  }, [JSON.stringify(runtimeMappings)]);
+
  useEffect(() => {
-    const hasBasicRequiredFields = jobType !== undefined;
-
-    const hasRequiredAnalysisFields =
-      (isJobTypeWithDepVar && dependentVariable !== '') ||
-      jobType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION;
+    handleRuntimeUpdate();
+  }, [JSON.stringify(runtimeMappings)]);

+  useEffect(() => {
    if (hasBasicRequiredFields && hasRequiredAnalysisFields) {
      debouncedGetExplainData();
    }
@ -324,15 +471,6 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
    };
  }, [jobType, dependentVariable, trainingPercent, JSON.stringify(includes), jobConfigQueryString]);

-  const unsupportedRuntimeFields = useMemo(
-    () =>
-      currentIndexPattern.fields
-        .getAll()
-        .filter((f) => f.runtimeField)
-        .map((f) => `'${f.displayName}'`),
-    [currentIndexPattern.fields]
-  );
-
  const scatterplotMatrixProps = useMemo(
    () => ({
      color: isJobTypeWithDepVar ? dependentVariable : undefined,
@ -342,6 +480,8 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
      index: currentIndexPattern.title,
      legendType: getScatterplotMatrixLegendType(jobType),
      searchQuery: jobConfigQuery,
+      runtimeMappings,
+      indexPattern: currentIndexPattern,
    }),
    [
      currentIndexPattern.title,
@ -388,6 +528,7 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
          />
        </EuiFormRow>
      )}
+      {((isClone && cloneJob) || !isClone) && <RuntimeMappings actions={actions} state={state} />}
      <EuiFormRow
        label={
          <Fragment>
@ -476,11 +617,11 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
              singleSelection={true}
              options={dependentVariableOptions}
              selectedOptions={dependentVariable ? [{ label: dependentVariable }] : []}
-              onChange={(selectedOptions) =>
+              onChange={(selectedOptions) => {
                setFormState({
                  dependentVariable: selectedOptions[0].label || '',
-                })
-              }
+                });
+              }}
              isClearable={false}
              isInvalid={dependentVariable === ''}
              data-test-subj={`mlAnalyticsCreateJobWizardDependentVariableSelect${
@ -500,35 +641,6 @@ export const ConfigurationStepForm: FC<CreateAnalyticsStepProps> = ({
      >
        <Fragment />
      </EuiFormRow>
-      {Array.isArray(unsupportedRuntimeFields) && unsupportedRuntimeFields.length > 0 && (
-        <>
-          <EuiCallOut size="s" color="warning">
-            <FormattedMessage
-              id="xpack.ml.dataframe.analytics.create.unsupportedRuntimeFieldsCallout"
-              defaultMessage="The runtime {runtimeFieldsCount, plural, one {field} other {fields}} {unsupportedRuntimeFields} {extraCountMsg} are not supported for analysis."
-              values={{
-                runtimeFieldsCount: unsupportedRuntimeFields.length,
-                extraCountMsg:
-                  unsupportedRuntimeFields.length - maxRuntimeFieldsDisplayCount > 0 ? (
-                    <FormattedMessage
-                      id="xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg"
-                      defaultMessage="and {count} more"
-                      values={{
-                        count: unsupportedRuntimeFields.length - maxRuntimeFieldsDisplayCount,
-                      }}
-                    />
-                  ) : (
-                    ''
-                  ),
-                unsupportedRuntimeFields: unsupportedRuntimeFields
-                  .slice(0, maxRuntimeFieldsDisplayCount)
-                  .join(', '),
-              }}
-            />
-          </EuiCallOut>
-          <EuiSpacer />
-        </>
-      )}

      <AnalysisFieldsTable
        dependentVariable={dependentVariable}
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/form_options_validation.ts
@ -5,8 +5,10 @@
 * 2.0.
 */

+import { i18n } from '@kbn/i18n';
 import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
-import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../common/types/fields';
+import { RuntimeType } from '../../../../../../../../../../src/plugins/data/common';
+import { EVENT_RATE_FIELD_ID } from '../../../../../../../common/types/fields';
 import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics';
 import { AnalyticsJobType } from '../../../analytics_management/hooks/use_create_analytics_form/state';
 import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields';
@ -14,16 +16,69 @@ import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../com
 export const CATEGORICAL_TYPES = new Set(['ip', 'keyword']);

 // Regression supports numeric fields. Classification supports categorical, numeric, and boolean.
-export const shouldAddAsDepVarOption = (field: Field, jobType: AnalyticsJobType) => {
-  if (field.id === EVENT_RATE_FIELD_ID) return false;
+export const shouldAddAsDepVarOption = (
+  fieldId: string,
+  fieldType: ES_FIELD_TYPES | RuntimeType,
+  jobType: AnalyticsJobType
+) => {
+  if (fieldId === EVENT_RATE_FIELD_ID) return false;

-  const isBasicNumerical = BASIC_NUMERICAL_TYPES.has(field.type);
+  const isBasicNumerical = BASIC_NUMERICAL_TYPES.has(fieldType as ES_FIELD_TYPES);

  const isSupportedByClassification =
-    isBasicNumerical || CATEGORICAL_TYPES.has(field.type) || field.type === ES_FIELD_TYPES.BOOLEAN;
+    isBasicNumerical || CATEGORICAL_TYPES.has(fieldType) || fieldType === ES_FIELD_TYPES.BOOLEAN;

  if (jobType === ANALYSIS_CONFIG_TYPE.REGRESSION) {
-    return isBasicNumerical || EXTENDED_NUMERICAL_TYPES.has(field.type);
+    return isBasicNumerical || EXTENDED_NUMERICAL_TYPES.has(fieldType as ES_FIELD_TYPES);
  }
  if (jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION) return isSupportedByClassification;
 };
+
+export const handleExplainErrorMessage = (
+  errorMessage: string,
+  sourceIndex: string,
+  jobType: AnalyticsJobType
+) => {
+  let maxDistinctValuesErrorMessage;
+  let unsupportedFieldsErrorMessage;
+  let toastNotificationWarning;
+  let toastNotificationDanger;
+  if (
+    jobType === ANALYSIS_CONFIG_TYPE.CLASSIFICATION &&
+    (errorMessage.includes('must have at most') || errorMessage.includes('must have at least'))
+  ) {
+    maxDistinctValuesErrorMessage = errorMessage;
+  } else if (
+    errorMessage.includes('status_exception') &&
+    errorMessage.includes('unsupported type')
+  ) {
+    unsupportedFieldsErrorMessage = errorMessage;
+  } else if (
+    errorMessage.includes('status_exception') &&
+    errorMessage.includes('Unable to estimate memory usage as no documents')
+  ) {
+    toastNotificationWarning = i18n.translate(
+      'xpack.ml.dataframe.analytics.create.allDocsMissingFieldsErrorMessage',
+      {
+        defaultMessage: `Unable to estimate memory usage. There are mapped fields for source index [{index}] that do not exist in any indexed documents. You will have to switch to the JSON editor for explicit field selection and include only fields that exist in indexed documents.`,
+        values: {
+          index: sourceIndex,
+        },
+      }
+    );
+  } else {
+    toastNotificationDanger = {
+      title: i18n.translate('xpack.ml.dataframe.analytics.create.unableToFetchExplainDataMessage', {
+        defaultMessage: 'An error occurred fetching analysis fields data.',
+      }),
+      text: errorMessage,
+    };
+  }
+
+  return {
+    maxDistinctValuesErrorMessage,
+    unsupportedFieldsErrorMessage,
+    toastNotificationDanger,
+    toastNotificationWarning,
+  };
+};
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/index.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/index.ts
@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { RuntimeMappings } from './runtime_mappings';
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings.tsx
@ -0,0 +1,237 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC, useState, useEffect } from 'react';
+import {
+  EuiButton,
+  EuiButtonIcon,
+  EuiCopy,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiFormRow,
+  EuiSpacer,
+  EuiSwitch,
+  EuiText,
+} from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { XJsonMode } from '@kbn/ace';
+import { RuntimeField } from '../../../../../../../../../../src/plugins/data/common/index_patterns';
+import { useMlContext } from '../../../../../contexts/ml';
+import { CreateAnalyticsFormProps } from '../../../analytics_management/hooks/use_create_analytics_form';
+import { XJson } from '../../../../../../../../../../src/plugins/es_ui_shared/public';
+import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common';
+import { isPopulatedObject } from '../../../../../../../common/util/object_utils';
+import { RuntimeMappingsEditor } from './runtime_mappings_editor';
+
+const advancedEditorsSidebarWidth = '220px';
+const COPY_TO_CLIPBOARD_RUNTIME_MAPPINGS = i18n.translate(
+  'xpack.ml.dataframe.analytics.createWizard.indexPreview.copyRuntimeMappingsClipboardTooltip',
+  {
+    defaultMessage: 'Copy Dev Console statement of the runtime mappings to the clipboard.',
+  }
+);
+
+const { useXJsonMode } = XJson;
+const xJsonMode = new XJsonMode();
+
+interface Props {
+  actions: CreateAnalyticsFormProps['actions'];
+  state: CreateAnalyticsFormProps['state'];
+}
+
+type RuntimeMappings = Record<string, RuntimeField>;
+
+export const RuntimeMappings: FC<Props> = ({ actions, state }) => {
+  const [isRuntimeMappingsEditorEnabled, setIsRuntimeMappingsEditorEnabled] = useState<boolean>(
+    false
+  );
+  const [
+    isRuntimeMappingsEditorApplyButtonEnabled,
+    setIsRuntimeMappingsEditorApplyButtonEnabled,
+  ] = useState<boolean>(false);
+  const [
+    advancedEditorRuntimeMappingsLastApplied,
+    setAdvancedEditorRuntimeMappingsLastApplied,
+  ] = useState<string>();
+  const [advancedEditorRuntimeMappings, setAdvancedEditorRuntimeMappings] = useState<string>();
+
+  const { setFormState } = actions;
+  const { jobType, previousRuntimeMapping, runtimeMappings } = state.form;
+
+  const {
+    convertToJson,
+    setXJson: setAdvancedRuntimeMappingsConfig,
+    xJson: advancedRuntimeMappingsConfig,
+  } = useXJsonMode(runtimeMappings || '');
+
+  const mlContext = useMlContext();
+  const { currentIndexPattern } = mlContext;
+
+  const applyChanges = () => {
+    const removeRuntimeMappings = advancedRuntimeMappingsConfig === '';
+    const parsedRuntimeMappings = removeRuntimeMappings
+      ? undefined
+      : JSON.parse(advancedRuntimeMappingsConfig);
+    const prettySourceConfig = removeRuntimeMappings
+      ? ''
+      : JSON.stringify(parsedRuntimeMappings, null, 2);
+    const previous =
+      previousRuntimeMapping === undefined && runtimeMappings === undefined
+        ? parsedRuntimeMappings
+        : runtimeMappings;
+    setFormState({
+      runtimeMappings: parsedRuntimeMappings,
+      runtimeMappingsUpdated: true,
+      previousRuntimeMapping: previous,
+    });
+    setAdvancedEditorRuntimeMappings(prettySourceConfig);
+    setAdvancedEditorRuntimeMappingsLastApplied(prettySourceConfig);
+    setIsRuntimeMappingsEditorApplyButtonEnabled(false);
+  };
+
+  // If switching to KQL after updating via editor - reset search
+  const toggleEditorHandler = (reset = false) => {
+    if (reset === true) {
+      setFormState({ runtimeMappingsUpdated: false });
+    }
+    if (isRuntimeMappingsEditorEnabled === false) {
+      setAdvancedEditorRuntimeMappingsLastApplied(advancedEditorRuntimeMappings);
+    }
+
+    setIsRuntimeMappingsEditorEnabled(!isRuntimeMappingsEditorEnabled);
+    setIsRuntimeMappingsEditorApplyButtonEnabled(false);
+  };
+
+  useEffect(function getInitialRuntimeMappings() {
+    const combinedRuntimeMappings = getCombinedRuntimeMappings(
+      currentIndexPattern,
+      runtimeMappings
+    );
+
+    if (combinedRuntimeMappings) {
+      setAdvancedRuntimeMappingsConfig(JSON.stringify(combinedRuntimeMappings, null, 2));
+      setFormState({
+        runtimeMappings: combinedRuntimeMappings,
+      });
+    }
+  }, []);
+
+  return (
+    <>
+      <EuiSpacer size="s" />
+      <EuiFormRow
+        fullWidth={true}
+        label={i18n.translate('xpack.ml.dataframe.analytics.createWizard.runtimeMappingsLabel', {
+          defaultMessage: 'Runtime mappings',
+        })}
+      >
+        <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
+          <EuiFlexItem grow={true}>
+            {isPopulatedObject(runtimeMappings) ? (
+              <EuiText size="s" grow={false}>
+                {Object.keys(runtimeMappings).join(',')}
+              </EuiText>
+            ) : (
+              <FormattedMessage
+                id="xpack.ml.dataframe.analytics.createWizard.noRuntimeMappingsLabel"
+                defaultMessage="No runtime mapping"
+              />
+            )}
+
+            {isRuntimeMappingsEditorEnabled && (
+              <>
+                <EuiSpacer size="s" />
+                <RuntimeMappingsEditor
+                  advancedEditorRuntimeMappingsLastApplied={
+                    advancedEditorRuntimeMappingsLastApplied
+                  }
+                  advancedRuntimeMappingsConfig={advancedRuntimeMappingsConfig}
+                  setIsRuntimeMappingsEditorApplyButtonEnabled={
+                    setIsRuntimeMappingsEditorApplyButtonEnabled
+                  }
+                  setAdvancedRuntimeMappingsConfig={setAdvancedRuntimeMappingsConfig}
+                  convertToJson={convertToJson}
+                  xJsonMode={xJsonMode}
+                />
+              </>
+            )}
+          </EuiFlexItem>
+
+          <EuiFlexItem grow={false} style={{ width: advancedEditorsSidebarWidth }}>
+            <EuiFlexGroup gutterSize="xs" direction="column" justifyContent="spaceBetween">
+              <EuiFlexItem grow={false}>
+                <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
+                  <EuiFlexItem grow={false}>
+                    <EuiSwitch
+                      disabled={jobType === undefined}
+                      label={i18n.translate(
+                        'xpack.ml.dataframe.analytics.createWizard.advancedEditorRuntimeMappingsSwitchLabel',
+                        {
+                          defaultMessage: 'Edit runtime mappings',
+                        }
+                      )}
+                      checked={isRuntimeMappingsEditorEnabled}
+                      onChange={() => toggleEditorHandler()}
+                      data-test-subj="mlDataFrameAnalyticsRuntimeMappingsEditorSwitch"
+                    />
+                  </EuiFlexItem>
+                  <EuiFlexItem grow={false}>
+                    <EuiCopy
+                      beforeMessage={COPY_TO_CLIPBOARD_RUNTIME_MAPPINGS}
+                      textToCopy={advancedRuntimeMappingsConfig ?? ''}
+                    >
+                      {(copy: () => void) => (
+                        <EuiButtonIcon
+                          onClick={copy}
+                          iconType="copyClipboard"
+                          aria-label={COPY_TO_CLIPBOARD_RUNTIME_MAPPINGS}
+                        />
+                      )}
+                    </EuiCopy>
+                  </EuiFlexItem>
+                </EuiFlexGroup>
+              </EuiFlexItem>
+
+              {isRuntimeMappingsEditorEnabled && (
+                <EuiFlexItem style={{ width: advancedEditorsSidebarWidth }}>
+                  <EuiSpacer size="s" />
+                  <EuiText size="xs">
+                    {i18n.translate(
+                      'xpack.ml.dataframe.analytics.createWizard.advancedRuntimeMappingsEditorHelpText',
+                      {
+                        defaultMessage:
+                          'The advanced editor allows you to edit the runtime mappings of the source.',
+                      }
+                    )}
+                  </EuiText>
+                  <EuiSpacer size="s" />
+                  <EuiButton
+                    style={{ width: 'fit-content' }}
+                    size="s"
+                    fill
+                    onClick={applyChanges}
+                    disabled={!isRuntimeMappingsEditorApplyButtonEnabled}
+                    data-test-subj="mlDataFrameAnalyticsRuntimeMappingsApplyButton"
+                  >
+                    {i18n.translate(
+                      'xpack.ml.dataframe.analytics.createWizard.advancedSourceEditorApplyButtonText',
+                      {
+                        defaultMessage: 'Apply changes',
+                      }
+                    )}
+                  </EuiButton>
+                </EuiFlexItem>
+              )}
+            </EuiFlexGroup>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiFormRow>
+      <EuiSpacer size="s" />
+    </>
+  );
+};
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings_editor.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/runtime_mappings/runtime_mappings_editor.tsx
@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isEqual } from 'lodash';
+import React, { memo, FC } from 'react';
+import { EuiCodeEditor } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { isRuntimeMappings } from '../../../../../../../common/util/runtime_field_utils';
+
+interface Props {
+  convertToJson: (data: string) => string;
+  setAdvancedRuntimeMappingsConfig: React.Dispatch<string>;
+  setIsRuntimeMappingsEditorApplyButtonEnabled: React.Dispatch<React.SetStateAction<boolean>>;
+  advancedEditorRuntimeMappingsLastApplied: string | undefined;
+  advancedRuntimeMappingsConfig: string;
+  xJsonMode: any;
+}
+
+export const RuntimeMappingsEditor: FC<Props> = memo(
+  ({
+    convertToJson,
+    xJsonMode,
+    setAdvancedRuntimeMappingsConfig,
+    setIsRuntimeMappingsEditorApplyButtonEnabled,
+    advancedEditorRuntimeMappingsLastApplied,
+    advancedRuntimeMappingsConfig,
+  }) => {
+    return (
+      <EuiCodeEditor
+        data-test-subj="mlDataFrameAnalyticsAdvancedRuntimeMappingsEditor"
+        style={{ border: '1px solid #e3e6ef' }}
+        height="250px"
+        width="100%"
+        mode={xJsonMode}
+        value={advancedRuntimeMappingsConfig}
+        onChange={(d: string) => {
+          setAdvancedRuntimeMappingsConfig(d);
+
+          // Disable the "Apply"-Button if the config hasn't changed.
+          if (advancedEditorRuntimeMappingsLastApplied === d) {
+            setIsRuntimeMappingsEditorApplyButtonEnabled(false);
+            return;
+          }
+
+          // Enable Apply button so user can remove previously created runtime field
+          if (d === '') {
+            setIsRuntimeMappingsEditorApplyButtonEnabled(true);
+            return;
+          }
+
+          // Try to parse the string passed on from the editor.
+          // If parsing fails, the "Apply"-Button will be disabled
+          try {
+            const parsedJson = JSON.parse(convertToJson(d));
+            setIsRuntimeMappingsEditorApplyButtonEnabled(isRuntimeMappings(parsedJson));
+          } catch (e) {
+            setIsRuntimeMappingsEditorApplyButtonEnabled(false);
+          }
+        }}
+        setOptions={{
+          fontSize: '12px',
+        }}
+        theme="textmate"
+        aria-label={i18n.translate(
+          'xpack.ml.dataframe.analytics.createWizard.runtimeMappings.advancedEditorAriaLabel',
+          {
+            defaultMessage: 'Advanced runtime editor',
+          }
+        )}
+      />
+    );
+  },
+  (prevProps, nextProps) => isEqual(pickProps(prevProps), pickProps(nextProps))
+);
+
+function pickProps(props: Props) {
+  return [props.advancedEditorRuntimeMappingsLastApplied, props.advancedRuntimeMappingsConfig];
+}
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts
@ -5,20 +5,23 @@
 * 2.0.
 */

-import { useEffect, useMemo } from 'react';
+import { useEffect, useMemo, useState } from 'react';

 import { estypes } from '@elastic/elasticsearch';
 import { EuiDataGridColumn } from '@elastic/eui';
-
 import { CoreSetup } from 'src/core/public';

 import { IndexPattern } from '../../../../../../../../../src/plugins/data/public';
+import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils';
+import { RuntimeMappings, RuntimeField } from '../../../../../../common/types/fields';
+import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms';

 import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';

 import {
  getFieldType,
  getDataGridSchemaFromKibanaFieldType,
+  getDataGridSchemaFromESFieldType,
  getFieldsFromKibanaIndexPattern,
  showDataGridColumnChartErrorMessageToast,
  useDataGrid,
@ -26,31 +29,51 @@ import {
  EsSorting,
  UseIndexDataReturnType,
  getProcessedFields,
+  getCombinedRuntimeMappings,
 } from '../../../../components/data_grid';
 import { extractErrorMessage } from '../../../../../../common/util/errors';
 import { INDEX_STATUS } from '../../../common/analytics';
 import { ml } from '../../../../services/ml_api_service';
-import { getRuntimeFieldsMapping } from '../../../../components/data_grid/common';

 type IndexSearchResponse = estypes.SearchResponse;

+interface MLEuiDataGridColumn extends EuiDataGridColumn {
+  isRuntimeFieldColumn?: boolean;
+}
+
+function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) {
+  return Object.keys(runtimeMappings).map((id) => {
+    const field = runtimeMappings[id];
+    const schema = getDataGridSchemaFromESFieldType(field.type as RuntimeField['type']);
+    return { id, schema, isExpandable: schema !== 'boolean', isRuntimeFieldColumn: true };
+  });
+}
+
 export const useIndexData = (
  indexPattern: IndexPattern,
  query: Record<string, any> | undefined,
-  toastNotifications: CoreSetup['notifications']['toasts']
+  toastNotifications: CoreSetup['notifications']['toasts'],
+  runtimeMappings?: RuntimeMappings
 ): UseIndexDataReturnType => {
  const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
    indexPattern,
  ]);

-  // EuiDataGrid State
-  const columns: EuiDataGridColumn[] = [
+  const [columns, setColumns] = useState<MLEuiDataGridColumn[]>([
    ...indexPatternFields.map((id) => {
      const field = indexPattern.fields.getByName(id);
-      const schema = getDataGridSchemaFromKibanaFieldType(field);
-      return { id, schema, isExpandable: schema !== 'boolean' };
+      const isRuntimeFieldColumn = field?.runtimeField !== undefined;
+      const schema = isRuntimeFieldColumn
+        ? getDataGridSchemaFromESFieldType(field?.type as RuntimeField['type'])
+        : getDataGridSchemaFromKibanaFieldType(field);
+      return {
+        id,
+        schema,
+        isExpandable: schema !== 'boolean',
+        isRuntimeFieldColumn,
+      };
    }),
-  ];
+  ]);

  const dataGrid = useDataGrid(columns);

@ -75,6 +98,8 @@ export const useIndexData = (
    setErrorMessage('');
    setStatus(INDEX_STATUS.LOADING);

+    const combinedRuntimeMappings = getCombinedRuntimeMappings(indexPattern, runtimeMappings);
+
    const sort: EsSorting = sortingColumns.reduce((s, column) => {
      s[column.id] = { order: column.direction };
      return s;
@ -88,14 +113,37 @@ export const useIndexData = (
        fields: ['*'],
        _source: false,
        ...(Object.keys(sort).length > 0 ? { sort } : {}),
-        ...getRuntimeFieldsMapping(indexPatternFields, indexPattern),
+        ...(isRuntimeMappings(combinedRuntimeMappings)
+          ? { runtime_mappings: combinedRuntimeMappings }
+          : {}),
      },
    };

    try {
      const resp: IndexSearchResponse = await ml.esSearch(esSearchRequest);
-
      const docs = resp.hits.hits.map((d) => getProcessedFields(d.fields ?? {}));
+
+      if (isRuntimeMappings(runtimeMappings)) {
+        // remove old runtime field from columns
+        const updatedColumns = columns.filter((col) => col.isRuntimeFieldColumn === false);
+        setColumns([
+          ...updatedColumns,
+          ...(combinedRuntimeMappings ? getRuntimeFieldColumns(combinedRuntimeMappings) : []),
+        ]);
+      } else {
+        setColumns([
+          ...indexPatternFields.map((id) => {
+            const field = indexPattern.fields.getByName(id);
+            const schema = getDataGridSchemaFromKibanaFieldType(field);
+            return {
+              id,
+              schema,
+              isExpandable: schema !== 'boolean',
+              isRuntimeFieldColumn: field?.runtimeField !== undefined,
+            };
+          }),
+        ]);
+      }
      setRowCount(typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total.value);
      setRowCountRelation(
        typeof resp.hits.total === 'number'
@ -115,13 +163,18 @@ export const useIndexData = (
      getIndexData();
    }
    // custom comparison
-  }, [indexPattern.title, indexPatternFields, JSON.stringify([query, pagination, sortingColumns])]);
+  }, [
+    indexPattern.title,
+    indexPatternFields,
+    JSON.stringify([query, pagination, sortingColumns, runtimeMappings]),
+  ]);

  const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [
    indexPattern,
  ]);

  const fetchColumnChartsData = async function (fieldHistogramsQuery: Record<string, any>) {
+    const combinedRuntimeMappings = getCombinedRuntimeMappings(indexPattern, runtimeMappings);
    try {
      const columnChartsData = await dataLoader.loadFieldHistograms(
        columns
@ -130,7 +183,9 @@ export const useIndexData = (
            fieldName: cT.id,
            type: getFieldType(cT.schema),
          })),
-        fieldHistogramsQuery
+        fieldHistogramsQuery,
+        DEFAULT_SAMPLER_SHARD_SIZE,
+        combinedRuntimeMappings
      );
      dataGrid.setColumnCharts(columnChartsData);
    } catch (e) {
@ -146,7 +201,7 @@ export const useIndexData = (
  }, [
    dataGrid.chartsVisible,
    indexPattern.title,
-    JSON.stringify([query, dataGrid.visibleColumns]),
+    JSON.stringify([query, dataGrid.visibleColumns, runtimeMappings]),
  ]);

  const renderCellValue = useRenderCellValue(indexPattern, pagination, tableItems);
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/page.tsx
@ -104,6 +104,7 @@ export const Page: FC<Props> = ({ jobId }) => {
      children: (
        <ConfigurationStep
          {...createAnalyticsForm}
+          isClone={jobId !== undefined}
          setCurrentStep={setCurrentStep}
          step={currentStep}
          stepActivated={activatedSteps[ANALYTICS_STEPS.CONFIGURATION]}
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx
@ -289,6 +289,11 @@ const getAnalyticsJobMeta = (config: CloneDataFrameAnalyticsConfig): AnalyticsJo
        match_all: {},
      },
    },
+    runtime_mappings: {
+      optional: true,
+      formKey: 'runtimeMappings',
+      defaultValue: undefined,
+    },
    _source: {
      optional: true,
    },
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/reducer.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/reducer.ts
@ -6,7 +6,7 @@
 */

 import { i18n } from '@kbn/i18n';
-import { memoize } from 'lodash';
+import { memoize, isEqual } from 'lodash';
 // @ts-ignore
 import numeral from '@elastic/numeral';
 import { isValidIndexName } from '../../../../../../../common/util/es_utils';
@ -470,7 +470,11 @@ export function reducer(state: State, action: Action): State {
      let disableSwitchToForm = false;
      try {
        resultJobConfig = JSON.parse(collapseLiteralStrings(action.advancedEditorRawString));
-        disableSwitchToForm = isAdvancedConfig(resultJobConfig);
+        const runtimeMappingsChanged =
+          state.form.runtimeMappings &&
+          resultJobConfig.source.runtime_mappings &&
+          !isEqual(state.form.runtimeMappings, resultJobConfig.source.runtime_mappings);
+        disableSwitchToForm = isAdvancedConfig(resultJobConfig) || runtimeMappingsChanged;
      } catch (e) {
        return {
          ...state,
--- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts
+++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts
@ -5,9 +5,11 @@
 * 2.0.
 */

+import { RuntimeMappings } from '../../../../../../../common/types/fields';
 import { DeepPartial, DeepReadonly } from '../../../../../../../common/types/common';
 import { checkPermission } from '../../../../../capabilities/check_capabilities';
 import { mlNodesAvailable } from '../../../../../ml_nodes_check';
+import { isRuntimeMappings } from '../../../../../../../common/util/runtime_field_utils';

 import { defaultSearchQuery, getAnalysisType } from '../../../../common/analytics';
 import { CloneDataFrameAnalyticsConfig } from '../../components/action_clone';
@ -94,6 +96,9 @@ export interface State {
    requiredFieldsError: string | undefined;
    randomizeSeed: undefined | number;
    resultsField: undefined | string;
+    runtimeMappings: undefined | RuntimeMappings;
+    runtimeMappingsUpdated: boolean;
+    previousRuntimeMapping: undefined | RuntimeMappings;
    softTreeDepthLimit: undefined | number;
    softTreeDepthTolerance: undefined | number;
    sourceIndex: EsIndexName;
@ -171,6 +176,9 @@ export const getInitialState = (): State => ({
    requiredFieldsError: undefined,
    randomizeSeed: undefined,
    resultsField: undefined,
+    runtimeMappings: undefined,
+    runtimeMappingsUpdated: false,
+    previousRuntimeMapping: undefined,
    softTreeDepthLimit: undefined,
    softTreeDepthTolerance: undefined,
    sourceIndex: '',
@ -212,6 +220,9 @@ export const getJobConfigFromFormState = (
        ? formState.sourceIndex.split(',').map((d) => d.trim())
        : formState.sourceIndex,
      query: formState.jobConfigQuery,
+      ...(isRuntimeMappings(formState.runtimeMappings)
+        ? { runtime_mappings: formState.runtimeMappings }
+        : {}),
    },
    dest: {
      index: formState.destinationIndex,
@ -340,6 +351,7 @@ export function getFormStateFromJobConfig(
    sourceIndex: Array.isArray(analyticsJobConfig.source.index)
      ? analyticsJobConfig.source.index.join(',')
      : analyticsJobConfig.source.index,
+    runtimeMappings: analyticsJobConfig.source.runtime_mappings,
    modelMemoryLimit: analyticsJobConfig.model_memory_limit,
    maxNumThreads: analyticsJobConfig.max_num_threads,
    includes: analyticsJobConfig.analyzed_fields?.includes ?? [],
--- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts
+++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts
@ -110,14 +110,15 @@ export class DataLoader {
  async loadFieldHistograms(
    fields: FieldHistogramRequestConfig[],
    query: string | SavedSearchQuery,
-    samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE
+    samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE,
+    editorRuntimeMappings?: RuntimeMappings
  ): Promise<any[]> {
    const stats = await ml.getVisualizerFieldHistograms({
      indexPatternTitle: this._indexPatternTitle,
      query,
      fields,
      samplerShardSize,
-      runtimeMappings: this._runtimeMappings,
+      runtimeMappings: editorRuntimeMappings || this._runtimeMappings,
    });

    return stats;
--- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
+++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts
@ -6,6 +6,7 @@
 */

 import { schema } from '@kbn/config-schema';
+import { runtimeMappingsSchema } from './runtime_mappings_schema';

 export const dataAnalyticsJobConfigSchema = schema.object({
  description: schema.maybe(schema.string()),
@ -16,6 +17,7 @@ export const dataAnalyticsJobConfigSchema = schema.object({
  source: schema.object({
    index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
    query: schema.maybe(schema.any()),
+    runtime_mappings: runtimeMappingsSchema,
    _source: schema.maybe(
      schema.object({
        /** Fields to include in results */
@ -51,6 +53,7 @@ export const dataAnalyticsExplainSchema = schema.object({
  source: schema.object({
    index: schema.oneOf([schema.string(), schema.arrayOf(schema.string())]),
    query: schema.maybe(schema.any()),
+    runtime_mappings: runtimeMappingsSchema,
  }),
  analysis: schema.any(),
  analyzed_fields: schema.maybe(schema.any()),
--- a/x-pack/plugins/translations/translations/ja-JP.json
+++ b/x-pack/plugins/translations/translations/ja-JP.json
@ -13329,7 +13329,6 @@
    "xpack.ml.dataframe.analytics.create.etaInputAriaLabel": "縮小が重みに適用されました。",
    "xpack.ml.dataframe.analytics.create.etaLabel": "Eta",
    "xpack.ml.dataframe.analytics.create.etaText": "縮小が重みに適用されました。0.001から1の範囲でなければなりません。",
-    "xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg": "{count}以上",
    "xpack.ml.dataframe.analytics.create.featureBagFractionInputAriaLabel": "各候補分割のランダムなbagを選択したときに使用される特徴量の割合",
    "xpack.ml.dataframe.analytics.create.featureBagFractionLabel": "特徴量bag割合",
    "xpack.ml.dataframe.analytics.create.featureBagFractionText": "各候補分割のランダムなbagを選択したときに使用される特徴量の割合。",
--- a/x-pack/plugins/translations/translations/zh-CN.json
+++ b/x-pack/plugins/translations/translations/zh-CN.json
@ -13500,7 +13500,6 @@
    "xpack.ml.dataframe.analytics.create.etaInputAriaLabel": "缩小量已应用于权重。",
    "xpack.ml.dataframe.analytics.create.etaLabel": "Eta",
    "xpack.ml.dataframe.analytics.create.etaText": "缩小量已应用于权重。必须介于 0.001 和 1 之间。",
-    "xpack.ml.dataframe.analytics.create.extraUnsupportedRuntimeFieldsMsg": "及另外 {count} 个",
    "xpack.ml.dataframe.analytics.create.featureBagFractionInputAriaLabel": "选择为每个候选拆分选择随机袋时使用的特征比例",
    "xpack.ml.dataframe.analytics.create.featureBagFractionLabel": "特征袋比例",
    "xpack.ml.dataframe.analytics.create.featureBagFractionText": "选择为每个候选拆分选择随机袋时使用的特征比例。",
@ -13604,7 +13603,6 @@
    "xpack.ml.dataframe.analytics.create.trainingPercentLabel": "训练百分比",
    "xpack.ml.dataframe.analytics.create.unableToFetchExplainDataMessage": "提取分析字段数据时发生错误。",
    "xpack.ml.dataframe.analytics.create.unsupportedFieldsError": "无效。{message}",
-    "xpack.ml.dataframe.analytics.create.unsupportedRuntimeFieldsCallout": "不支持分析运行时{runtimeFieldsCount, plural, other {字段}} {unsupportedRuntimeFields} {extraCountMsg}。",
    "xpack.ml.dataframe.analytics.create.useEstimatedMmlLabel": "使用估计的模型内存限制",
    "xpack.ml.dataframe.analytics.create.UseResultsFieldDefaultLabel": "使用结果字段默认值“{defaultValue}”",
    "xpack.ml.dataframe.analytics.create.viewResultsCardDescription": "查看分析作业的结果。",