[8.12] [Transform] Fix retention policy field name not setting by default correctly in Transform creation wizard (#172609) (#173617)

# Backport This will backport the following commits from `main` to `8.12`: - [[Transform] Fix retention policy field name not setting by default correctly in Transform creation wizard (#172609)](https://github.com/elastic/kibana/pull/172609)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)  Co-authored-by: Quynh Nguyen (Quinn) <43350163+qn895@users.noreply.github.com>
2025-04-24 01:38:56 -04:00 · 2023-12-19 08:55:02 -05:00 · 2023-12-19 08:55:02 -05:00 · d36413a10c
commit d36413a10c
parent 7e6ab21b0a
5 changed files with 149 additions and 97 deletions
--- a/x-pack/plugins/transform/public/app/common/request.ts
+++ b/x-pack/plugins/transform/public/app/common/request.ts
@ -271,7 +271,9 @@ export const getCreateTransformRequestBody = (
      }
    : {}),
  // conditionally add retention policy settings
-  ...(transformDetailsState.isRetentionPolicyEnabled
+  ...(transformDetailsState.isRetentionPolicyEnabled &&
+  transformDetailsState.retentionPolicyDateField !== '' &&
+  transformDetailsState.retentionPolicyMaxAge !== ''
    ? {
        retention_policy: {
          time: {
--- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx
+++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_details/step_details_form.tsx
@ -106,9 +106,38 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
    const [createDataView, setCreateDataView] = useState(
      canCreateDataView === false ? false : defaults.createDataView
    );
-    const [dataViewAvailableTimeFields, setDataViewAvailableTimeFields] = useState<string[]>([]);
    const [dataViewTimeField, setDataViewTimeField] = useState<string | undefined>();

+    const previewRequest = useMemo(() => {
+      const { searchQuery, previewRequest: partialPreviewRequest } = stepDefineState;
+      const transformConfigQuery = getTransformConfigQuery(searchQuery);
+      return getPreviewTransformRequestBody(
+        searchItems.dataView,
+        transformConfigQuery,
+        partialPreviewRequest,
+        stepDefineState.runtimeMappings
+      );
+    }, [searchItems.dataView, stepDefineState]);
+
+    const { error: transformsPreviewError, data: transformPreview } =
+      useGetTransformsPreview(previewRequest);
+
+    const destIndexAvailableTimeFields = useMemo<string[]>(() => {
+      if (!transformPreview) return [];
+      const properties = transformPreview.generated_dest_index.mappings.properties;
+      const timeFields: string[] = Object.keys(properties).filter(
+        (col) => properties[col].type === 'date'
+      );
+      return timeFields;
+    }, [transformPreview]);
+
+    useEffect(
+      function resetDataViewTimeField() {
+        setDataViewTimeField(destIndexAvailableTimeFields[0]);
+      },
+      [destIndexAvailableTimeFields]
+    );
+
    const onTimeFieldChanged = React.useCallback(
      (e: React.ChangeEvent<HTMLSelectElement>) => {
        const value = e.target.value;
@ -118,11 +147,11 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
        }
        // Find the time field based on the selected value
        // this is to account for undefined when user chooses not to use a date field
-        const timeField = dataViewAvailableTimeFields.find((col) => col === value);
+        const timeField = destIndexAvailableTimeFields.find((col) => col === value);

        setDataViewTimeField(timeField);
      },
-      [setDataViewTimeField, dataViewAvailableTimeFields]
+      [setDataViewTimeField, destIndexAvailableTimeFields]
    );

    const {
@ -146,31 +175,6 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
      // eslint-disable-next-line react-hooks/exhaustive-deps
    }, [transformsError]);

-    const previewRequest = useMemo(() => {
-      const { searchQuery, previewRequest: partialPreviewRequest } = stepDefineState;
-      const transformConfigQuery = getTransformConfigQuery(searchQuery);
-      return getPreviewTransformRequestBody(
-        searchItems.dataView,
-        transformConfigQuery,
-        partialPreviewRequest,
-        stepDefineState.runtimeMappings
-      );
-    }, [searchItems.dataView, stepDefineState]);
-    const { error: transformsPreviewError, data: transformPreview } =
-      useGetTransformsPreview(previewRequest);
-
-    useEffect(() => {
-      if (transformPreview) {
-        const properties = transformPreview.generated_dest_index.mappings.properties;
-        const timeFields: string[] = Object.keys(properties).filter(
-          (col) => properties[col].type === 'date'
-        );
-
-        setDataViewAvailableTimeFields(timeFields);
-        setDataViewTimeField(timeFields[0]);
-      }
-    }, [transformPreview]);
-
    useEffect(() => {
      if (transformsPreviewError !== null) {
        toastNotifications.addDanger({
@ -242,24 +246,24 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
      }
    }, [dataViewTitlesError]);

-    const dateFieldNames = searchItems.dataView.fields
+    const sourceIndexDateFieldNames = searchItems.dataView.fields
      .filter((f) => f.type === KBN_FIELD_TYPES.DATE)
      .map((f) => f.name)
      .sort();

    // Continuous Mode
-    const isContinuousModeAvailable = dateFieldNames.length > 0;
+    const isContinuousModeAvailable = sourceIndexDateFieldNames.length > 0;
    const [isContinuousModeEnabled, setContinuousModeEnabled] = useState(
      defaults.isContinuousModeEnabled
    );
    const [continuousModeDateField, setContinuousModeDateField] = useState(
-      isContinuousModeAvailable ? dateFieldNames[0] : ''
+      isContinuousModeAvailable ? sourceIndexDateFieldNames[0] : ''
    );
    const [continuousModeDelay, setContinuousModeDelay] = useState(defaults.continuousModeDelay);
    const isContinuousModeDelayValid = continuousModeDelayValidator(continuousModeDelay);

    // Retention Policy
-    const isRetentionPolicyAvailable = dateFieldNames.length > 0;
+    const isRetentionPolicyAvailable = destIndexAvailableTimeFields.length > 0;
    const [isRetentionPolicyEnabled, setRetentionPolicyEnabled] = useState(
      defaults.isRetentionPolicyEnabled
    );
@ -272,15 +276,27 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
    const retentionPolicyMaxAgeEmpty = retentionPolicyMaxAge === '';
    const isRetentionPolicyMaxAgeValid = retentionPolicyMaxAgeValidator(retentionPolicyMaxAge);

-    // Reset retention policy settings when the user disables the whole option
    useEffect(() => {
+      // Reset retention policy settings when the user disables the whole option
      if (!isRetentionPolicyEnabled) {
        setRetentionPolicyDateField(
-          isRetentionPolicyAvailable ? dataViewAvailableTimeFields[0] : ''
+          isRetentionPolicyAvailable ? destIndexAvailableTimeFields[0] : ''
        );
        setRetentionPolicyMaxAge('');
      }
-    }, [isRetentionPolicyEnabled]);
+
+      // When retention policy is first enabled, pick a default option
+      if (
+        isRetentionPolicyAvailable &&
+        isRetentionPolicyEnabled &&
+        retentionPolicyDateField === ''
+      ) {
+        // If a time field '@timestamp' exists, prioritize that
+        const prioritizeTimestamp = destIndexAvailableTimeFields.find((d) => d === '@timestamp');
+        // else pick the first available option
+        setRetentionPolicyDateField(prioritizeTimestamp ?? destIndexAvailableTimeFields[0]);
+      }
+    }, [isRetentionPolicyEnabled, isRetentionPolicyAvailable, destIndexAvailableTimeFields]);

    const transformIdExists = transformIds.some((id) => transformId === id);
    const transformIdEmpty = transformId === '';
@ -537,7 +553,7 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
            createDataView={createDataView}
            dataViewTitleExists={dataViewTitleExists}
            setCreateDataView={setCreateDataView}
-            dataViewAvailableTimeFields={dataViewAvailableTimeFields}
+            dataViewAvailableTimeFields={destIndexAvailableTimeFields}
            dataViewTimeField={dataViewTimeField}
            onTimeFieldChanged={onTimeFieldChanged}
          />
@ -582,7 +598,7 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
                )}
              >
                <EuiSelect
-                  options={dateFieldNames.map((text: string) => ({ text, value: text }))}
+                  options={sourceIndexDateFieldNames.map((text: string) => ({ text, value: text }))}
                  value={continuousModeDateField}
                  onChange={(e) => setContinuousModeDateField(e.target.value)}
                  data-test-subj="transformContinuousDateFieldSelect"
@ -652,7 +668,7 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
              data-test-subj="transformRetentionPolicySwitch"
            />
          </EuiFormRow>
-          {isRetentionPolicyEnabled && (
+          {isRetentionPolicyEnabled && isRetentionPolicyAvailable && (
            <>
              <EuiFormRow
                label={i18n.translate(
@ -670,7 +686,7 @@ export const StepDetailsForm: FC<StepDetailsFormProps> = React.memo(
                )}
              >
                <EuiSelect
-                  options={dataViewAvailableTimeFields.map((text: string) => ({ text }))}
+                  options={destIndexAvailableTimeFields.map((text: string) => ({ text }))}
                  value={retentionPolicyDateField}
                  onChange={(e) => setRetentionPolicyDateField(e.target.value)}
                  data-test-subj="transformRetentionPolicyDateFieldSelect"
--- a/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_retention_policy.tsx
+++ b/x-pack/plugins/transform/public/app/sections/transform_management/components/edit_transform_flyout/edit_transform_retention_policy.tsx
@ -5,80 +5,108 @@
 * 2.0.
 */

-import React, { useEffect, useMemo, useState, type FC } from 'react';
+import React, { useEffect, useMemo, type FC } from 'react';

 import { i18n } from '@kbn/i18n';

 import { EuiFormRow, EuiSelect, EuiSpacer, EuiSwitch } from '@elastic/eui';
-
-import { KBN_FIELD_TYPES } from '@kbn/field-types';
-
-import { useAppDependencies } from '../../../../app_dependencies';
+import { toMountPoint } from '@kbn/react-kibana-mount';
+import { useAppDependencies, useToastNotifications } from '../../../../app_dependencies';
+import { ToastNotificationText } from '../../../../components';
+import { isLatestTransform, isPivotTransform } from '../../../../../../common/types/transform';
+import { useGetTransformsPreview } from '../../../../hooks';

 import { EditTransformFlyoutFormTextInput } from './edit_transform_flyout_form_text_input';
 import { useEditTransformFlyout } from './use_edit_transform_flyout';
+import { getErrorMessage } from '../../../../../../common/utils/errors';

 export const EditTransformRetentionPolicy: FC = () => {
-  const appDeps = useAppDependencies();
-  const dataViewsClient = appDeps.data.dataViews;
+  const { i18n: i18nStart, theme } = useAppDependencies();
+
+  const toastNotifications = useToastNotifications();

  const dataViewId = useEditTransformFlyout('dataViewId');
  const formSections = useEditTransformFlyout('stateFormSection');
  const retentionPolicyField = useEditTransformFlyout('retentionPolicyField');
  const { formField, formSection } = useEditTransformFlyout('actions');
+  const requestConfig = useEditTransformFlyout('config');

-  const [dateFieldNames, setDateFieldNames] = useState<string[]>([]);
+  const previewRequest = useMemo(() => {
+    return {
+      source: requestConfig.source,
+      ...(isPivotTransform(requestConfig) ? { pivot: requestConfig.pivot } : {}),
+      ...(isLatestTransform(requestConfig) ? { latest: requestConfig.latest } : {}),
+    };
+  }, [requestConfig]);

-  useEffect(
-    function getDateFields() {
-      let unmounted = false;
-      if (dataViewId !== undefined) {
-        dataViewsClient.get(dataViewId).then((dataView) => {
-          if (dataView) {
-            const dateTimeFields = dataView.fields
-              .filter((f) => f.type === KBN_FIELD_TYPES.DATE)
-              .map((f) => f.name)
-              .sort();
-            if (!unmounted) {
-              setDateFieldNames(dateTimeFields);
-            }
-          }
-        });
-        return () => {
-          unmounted = true;
-        };
-      }
-    },
-    [dataViewId, dataViewsClient]
-  );
+  const { error: transformsPreviewError, data: transformPreview } =
+    useGetTransformsPreview(previewRequest);

-  const isRetentionPolicyAvailable = dateFieldNames.length > 0;
+  const destIndexAvailableTimeFields = useMemo<string[]>(() => {
+    if (transformPreview?.generated_dest_index) {
+      const properties = transformPreview.generated_dest_index.mappings.properties;
+      const timeFields: string[] = Object.keys(properties).filter(
+        (col) => properties[col].type === 'date'
+      );
+
+      return timeFields;
+    }
+    return [];
+  }, [transformPreview]);
+
+  useEffect(() => {
+    if (transformsPreviewError !== null) {
+      toastNotifications.addDanger({
+        title: i18n.translate('xpack.transform.transformList.errorGettingTransformPreview', {
+          defaultMessage: 'An error occurred fetching the transform preview',
+        }),
+        text: toMountPoint(
+          <ToastNotificationText text={getErrorMessage(transformsPreviewError)} />,
+          { theme, i18n: i18nStart }
+        ),
+      });
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [transformsPreviewError]);
+
+  const isRetentionPolicyAvailable = destIndexAvailableTimeFields.length > 0;
  const retentionDateFieldOptions = useMemo(() => {
-    return Array.isArray(dateFieldNames)
-      ? dateFieldNames.map((text: string) => ({ text, value: text }))
+    return Array.isArray(destIndexAvailableTimeFields)
+      ? destIndexAvailableTimeFields.map((text: string) => ({ text, value: text }))
      : [];
-  }, [dateFieldNames]);
+  }, [destIndexAvailableTimeFields]);

  return (
    <>
-      <EuiSwitch
-        name="transformEditRetentionPolicySwitch"
-        label={i18n.translate(
-          'xpack.transform.transformList.editFlyoutFormRetentionPolicySwitchLabel',
-          {
-            defaultMessage: 'Retention policy',
-          }
-        )}
-        checked={formSections.retentionPolicy.enabled}
-        onChange={(e) =>
-          formSection({
-            section: 'retentionPolicy',
-            enabled: e.target.checked,
-          })
+      <EuiFormRow
+        helpText={
+          isRetentionPolicyAvailable === false
+            ? i18n.translate('xpack.transform.transformList.editFlyoutFormEetentionPolicyError', {
+                defaultMessage:
+                  'Retention policy settings are not available for indices without date fields.',
+              })
+            : ''
        }
-        disabled={!isRetentionPolicyAvailable}
-        data-test-subj="transformEditRetentionPolicySwitch"
-      />
+      >
+        <EuiSwitch
+          name="transformEditRetentionPolicySwitch"
+          label={i18n.translate(
+            'xpack.transform.transformList.editFlyoutFormRetentionPolicySwitchLabel',
+            {
+              defaultMessage: 'Retention policy',
+            }
+          )}
+          checked={formSections.retentionPolicy.enabled}
+          onChange={(e) =>
+            formSection({
+              section: 'retentionPolicy',
+              enabled: e.target.checked,
+            })
+          }
+          disabled={!isRetentionPolicyAvailable}
+          data-test-subj="transformEditRetentionPolicySwitch"
+        />
+      </EuiFormRow>
      {formSections.retentionPolicy.enabled && (
        <div data-test-subj="transformEditRetentionPolicyContent">
          <EuiSpacer size="m" />
--- a/x-pack/test/functional/apps/transform/edit_clone/editing.ts
+++ b/x-pack/test/functional/apps/transform/edit_clone/editing.ts
@ -59,6 +59,7 @@ export default function ({ getService }: FtrProviderContext) {
        transformDocsPerSecond: '1000',
        transformFrequency: '10m',
        resetRetentionPolicy: false,
+        retentionPolicyAvailable: false,
        transformRetentionPolicyField: 'order_date',
        transformRetentionPolicyMaxAge: '1d',
        numFailureRetries: '0',
@ -84,6 +85,7 @@ export default function ({ getService }: FtrProviderContext) {
        transformDocsPerSecond: '1000',
        transformFrequency: '10m',
        resetRetentionPolicy: true,
+        retentionPolicyAvailable: true,
        numFailureRetries: '7',
        expected: {
          messageText: 'updated transform.',
@ -172,15 +174,19 @@ export default function ({ getService }: FtrProviderContext) {
          );

          await transform.testExecution.logTestStep('should update the transform retention policy');
-          await transform.editFlyout.clickTransformEditRetentionPolicySettings(
-            !testData.resetRetentionPolicy
-          );

+          await transform.editFlyout.assertTransformEditFlyoutRetentionPolicySwitchEnabled(
+            testData.retentionPolicyAvailable
+          );
          if (
-            !testData.resetRetentionPolicy &&
+            testData.retentionPolicyAvailable &&
            testData?.transformRetentionPolicyField &&
            testData?.transformRetentionPolicyMaxAge
          ) {
+            await transform.editFlyout.clickTransformEditRetentionPolicySettings(
+              !testData.resetRetentionPolicy
+            );
+
            await transform.editFlyout.assertTransformEditFlyoutRetentionPolicyFieldSelectEnabled(
              true
            );
@ -242,7 +248,7 @@ export default function ({ getService }: FtrProviderContext) {

          await transform.table.assertTransformExpandedRowJson(
            'retention_policy',
-            !testData.resetRetentionPolicy
+            testData.retentionPolicyAvailable
          );
          await transform.table.assertTransformExpandedRowJson('updated description');
          await transform.table.assertTransformExpandedRowMessages(testData.expected.messageText);
--- a/x-pack/test/functional/apps/transform/permissions/full_transform_access.ts
+++ b/x-pack/test/functional/apps/transform/permissions/full_transform_access.ts
@ -158,7 +158,7 @@ export default function ({ getService }: FtrProviderContext) {
        await transform.testExecution.logTestStep(
          'should have the retention policy switch enabled'
        );
-        await transform.editFlyout.assertTransformEditFlyoutRetentionPolicySwitchEnabled(true);
+        await transform.editFlyout.assertTransformEditFlyoutRetentionPolicySwitchEnabled(false);

        await transform.testExecution.logTestStep(
          'should have the advanced settings inputs enabled'