[8.x] [Streams 🌊] Schema editor bug fixes (#218225) (#218269)

# Backport This will backport the following commits from `main` to `8.x`: - [[Streams 🌊] Schema editor bug fixes (#218225)](https://github.com/elastic/kibana/pull/218225)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: Marco Antonio Ghiani <marcoantonio.ghiani01@gmail.com>
2025-04-23 09:19:04 -04:00 · 2025-04-15 18:27:12 +02:00 · 2025-04-15 18:27:12 +02:00 · 0e10b2ea53
commit 0e10b2ea53
parent cdd4855188
10 changed files with 226 additions and 108 deletions
--- a/x-pack/platform/plugins/shared/streams/server/routes/internal/streams/schema/route.ts
+++ b/x-pack/platform/plugins/shared/streams/server/routes/internal/streams/schema/route.ts
@ -171,13 +171,7 @@ export const schemaFieldsSimulationRoute = createServerRoute({
    }

    const propertiesForSimulation = Object.fromEntries(
-      userFieldDefinitions.map((field) => [
-        field.name,
-        {
-          type: field.type,
-          ...(field.format ? { format: field.format } : {}),
-        },
-      ])
+      userFieldDefinitions.map(({ name, ...field }) => [name, field])
    );

    const fieldDefinitionKeys = Object.keys(propertiesForSimulation);
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/constants.ts
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/constants.ts
@ -14,41 +14,49 @@ export const FIELD_TYPE_MAP = {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableBooleanType', {
      defaultMessage: 'Boolean',
    }),
+    readonly: false,
  },
  date: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableDateType', {
      defaultMessage: 'Date',
    }),
+    readonly: false,
  },
  keyword: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableKeywordType', {
      defaultMessage: 'Keyword',
    }),
+    readonly: false,
  },
  match_only_text: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableTextType', {
      defaultMessage: 'Text (match_only_text)',
    }),
+    readonly: false,
  },
  long: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableNumberType', {
      defaultMessage: 'Number (long)',
    }),
+    readonly: false,
  },
  double: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableNumberType', {
      defaultMessage: 'Number (double)',
    }),
+    readonly: false,
  },
  ip: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableIpType', {
      defaultMessage: 'IP',
    }),
+    readonly: false,
  },
  system: {
    label: i18n.translate('xpack.streams.streamDetailSchemaEditorFieldsTableSystemType', {
      defaultMessage: 'System managed',
    }),
+    readonly: true,
  },
 } as const;

--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/filters/filter_group.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/filters/filter_group.tsx
@ -15,6 +15,7 @@ import {
 } from '@elastic/eui';
 import { useBoolean } from '@kbn/react-hooks';
 import React from 'react';
+import { css } from '@emotion/react';

 export const FilterGroup = ({
  filterGroupButtonLabel,
@ -57,9 +58,9 @@ export const FilterGroup = ({
        <EuiSelectable aria-label={filterGroupButtonLabel} options={items} onChange={onChange}>
          {(list) => (
            <div
-              css={{
-                minWidth: '200px',
-              }}
+              css={css`
+                min-width: 220px;
+              `}
            >
              {list}
            </div>
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/filters/status_filter_group.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/filters/status_filter_group.tsx
@ -5,14 +5,15 @@
 * 2.0.
 */

-import { useCallback, useState } from 'react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import { uniq } from 'lodash';
 import { i18n } from '@kbn/i18n';
-import React from 'react';
 import { EuiSelectableOption, EuiSelectableProps } from '@elastic/eui';
 import { FilterGroup } from './filter_group';
 import { FIELD_STATUS_MAP } from '../constants';
 import { TControlsChangeHandler } from '../hooks/use_controls';
 import { SchemaFieldStatus } from '../types';
+import { useSchemaEditorContext } from '../schema_editor_context';

 const BUTTON_LABEL = i18n.translate(
  'xpack.streams.streamDetailSchemaEditor.fieldStatusFilterGroupButtonLabel',
@ -20,14 +21,28 @@ const BUTTON_LABEL = i18n.translate(
 );

 export const FieldStatusFilterGroup = ({ onChange }: { onChange: TControlsChangeHandler }) => {
-  const [items, setItems] = useState<EuiSelectableOption[]>(() =>
-    Object.entries(FIELD_STATUS_MAP).map(([key, value]) => {
-      return {
-        label: value.label,
-        key,
-      };
-    })
-  );
+  const { fields } = useSchemaEditorContext();
+
+  const fieldStatus = useMemo(() => uniq(fields.map((field) => field.status)), [fields]);
+
+  const [items, setItems] = useState<EuiSelectableOption[]>(() => getStatusOptions(fieldStatus));
+
+  // This side effect is due to the fact that the available field status can be updated once the unmapped fields are fetched.
+  useEffect(() => {
+    setItems((prevItems) => {
+      const prevSelection = new Map(prevItems.map((item) => [item.key, item.checked]));
+
+      const nextItems = getStatusOptions(fieldStatus);
+
+      nextItems.forEach((item) => {
+        if (prevSelection.has(item.key)) {
+          item.checked = prevSelection.get(item.key);
+        }
+      });
+
+      return nextItems;
+    });
+  }, [fieldStatus]);

  const onChangeItems = useCallback<Required<EuiSelectableProps>['onChange']>(
    (nextItems) => {
@ -45,3 +60,10 @@ export const FieldStatusFilterGroup = ({ onChange }: { onChange: TControlsChange
    <FilterGroup items={items} filterGroupButtonLabel={BUTTON_LABEL} onChange={onChangeItems} />
  );
 };
+
+const getStatusOptions = (fieldStatus: SchemaFieldStatus[]): EuiSelectableOption[] => {
+  return fieldStatus.map((key) => ({
+    label: FIELD_STATUS_MAP[key].label,
+    key,
+  }));
+};
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/filters/type_filter_group.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/filters/type_filter_group.tsx
@ -5,14 +5,14 @@
 * 2.0.
 */

-import { useCallback, useState } from 'react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import { uniq } from 'lodash';
 import { i18n } from '@kbn/i18n';
-import React from 'react';
 import { EuiSelectableOption, EuiSelectableProps } from '@elastic/eui';
 import { FilterGroup } from './filter_group';
-import { FIELD_TYPE_MAP } from '../constants';
+import { FIELD_TYPE_MAP, FieldTypeOption } from '../constants';
 import { TControlsChangeHandler } from '../hooks/use_controls';
-import { SchemaFieldType } from '../types';
+import { useSchemaEditorContext } from '../schema_editor_context';

 const BUTTON_LABEL = i18n.translate(
  'xpack.streams.streamDetailSchemaEditor.fieldTypeFilterGroupButtonLabel',
@ -20,22 +20,44 @@ const BUTTON_LABEL = i18n.translate(
 );

 export const FieldTypeFilterGroup = ({ onChange }: { onChange: TControlsChangeHandler }) => {
-  const [items, setItems] = useState<EuiSelectableOption[]>(() =>
-    Object.entries(FIELD_TYPE_MAP).map(([key, value]) => {
-      return {
-        label: value.label,
-        key,
-      };
-    })
+  const { fields } = useSchemaEditorContext();
+
+  const fieldTypes = useMemo(
+    () =>
+      uniq(
+        fields
+          .map((field) => field.type)
+          .filter((type): type is FieldTypeOption => type !== undefined)
+      ),
+    [fields]
  );

+  const [items, setItems] = useState<EuiSelectableOption[]>(() => getTypeOptions(fieldTypes));
+
+  // This side effect is due to the fact that the available field status can be updated once the unmapped fields are fetched.
+  useEffect(() => {
+    setItems((prevItems) => {
+      const prevSelection = new Map(prevItems.map((item) => [item.key, item.checked]));
+
+      const nextItems = getTypeOptions(fieldTypes);
+
+      nextItems.forEach((item) => {
+        if (prevSelection.has(item.key)) {
+          item.checked = prevSelection.get(item.key);
+        }
+      });
+
+      return nextItems;
+    });
+  }, [fieldTypes]);
+
  const onChangeItems = useCallback<Required<EuiSelectableProps>['onChange']>(
    (nextItems) => {
      setItems(nextItems);
      onChange({
        type: nextItems
          .filter((nextItem) => nextItem.checked === 'on')
-          .map((item) => item.key as SchemaFieldType),
+          .map((item) => item.key as FieldTypeOption),
      });
    },
    [onChange]
@ -45,3 +67,10 @@ export const FieldTypeFilterGroup = ({ onChange }: { onChange: TControlsChangeHa
    <FilterGroup items={items} filterGroupButtonLabel={BUTTON_LABEL} onChange={onChangeItems} />
  );
 };
+
+const getTypeOptions = (fieldStatus: FieldTypeOption[]): EuiSelectableOption[] => {
+  return fieldStatus.map((key) => ({
+    label: FIELD_TYPE_MAP[key].label,
+    key,
+  }));
+};
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/advanced_field_mapping_options.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/advanced_field_mapping_options.tsx
@ -7,17 +7,21 @@
 import {
  EuiAccordion,
  EuiCodeBlock,
+  EuiFormRow,
  EuiLink,
  EuiPanel,
-  EuiSpacer,
-  EuiText,
  useGeneratedHtmlId,
 } from '@elastic/eui';
 import { CodeEditor } from '@kbn/code-editor';
 import { i18n } from '@kbn/i18n';
 import React, { useMemo } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
-import { getAdvancedParameters } from '@kbn/streams-schema';
+import {
+  FieldDefinitionConfigAdvancedParameters,
+  isSchema,
+  recursiveRecord,
+} from '@kbn/streams-schema';
+import { useBoolean } from '@kbn/react-hooks';
 import { SchemaField } from '../types';
 import { useKibana } from '../../../../hooks/use_kibana';

@ -28,16 +32,23 @@ const label = i18n.translate('xpack.streams.advancedFieldMappingOptions.label',
 export const AdvancedFieldMappingOptions = ({
  field,
  onChange,
+  onValidate,
  isEditing,
 }: {
  field: SchemaField;
  onChange: (field: Partial<SchemaField>) => void;
+  onValidate?: (isValid: boolean) => void;
  isEditing: boolean;
 }) => {
  const { core } = useKibana();

  const accordionId = useGeneratedHtmlId({ prefix: 'accordionID' });

+  const [hasParsingError, { on: markAsParsingError, off: resetParsingErrorFlag }] =
+    useBoolean(false);
+
+  const isInvalid = hasParsingError || !getValidFlag(field.additionalParameters);
+
  const jsonOptions = useMemo(
    () => (field.additionalParameters ? JSON.stringify(field.additionalParameters, null, 2) : ''),
    [field.additionalParameters]
@ -46,50 +57,73 @@ export const AdvancedFieldMappingOptions = ({
  return (
    <EuiAccordion id={accordionId} buttonContent={label}>
      <EuiPanel color="subdued">
-        <EuiText size="xs">
-          <FormattedMessage
-            id="xpack.streams.advancedFieldMappingOptions.docs.label"
-            defaultMessage="Parameters can be defined with JSON. {link}"
-            values={{
-              link: (
-                <EuiLink
-                  data-test-subj="streamsAppAdvancedFieldMappingOptionsViewDocumentationLink"
-                  href={core.docLinks.links.elasticsearch.docsBase.concat('mapping-params.html')}
-                  target="_blank"
-                  external
-                >
-                  <FormattedMessage
-                    id="xpack.streams.indexPattern.randomSampling.learnMore"
-                    defaultMessage="View documentation."
-                  />
-                </EuiLink>
-              ),
-            }}
-          />
-        </EuiText>
-        <EuiSpacer size="s" />
-        {isEditing ? (
-          <CodeEditor
-            height={120}
-            languageId="json"
-            value={jsonOptions}
-            onChange={(value) => {
-              try {
-                onChange({
-                  additionalParameters:
-                    value === '' ? undefined : getAdvancedParameters(field.name, JSON.parse(value)),
-                });
-              } catch (error: unknown) {
-                // do nothing
-              }
-            }}
-          />
-        ) : (
-          <EuiCodeBlock language="json" isCopyable>
-            {jsonOptions ?? ''}
-          </EuiCodeBlock>
-        )}
+        <EuiFormRow
+          isInvalid={isInvalid}
+          error={
+            isInvalid
+              ? i18n.translate('xpack.streams.advancedFieldMappingOptions.error', {
+                  defaultMessage:
+                    'Invalid advanced field mapping parameters. It should be defined as a JSON object.',
+                })
+              : undefined
+          }
+          label={
+            <FormattedMessage
+              id="xpack.streams.advancedFieldMappingOptions.docs.label"
+              defaultMessage="Parameters can be defined with JSON. {link}"
+              values={{
+                link: (
+                  <EuiLink
+                    data-test-subj="streamsAppAdvancedFieldMappingOptionsViewDocumentationLink"
+                    href={core.docLinks.links.elasticsearch.docsBase.concat('mapping-params.html')}
+                    target="_blank"
+                    external
+                  >
+                    <FormattedMessage
+                      id="xpack.streams.indexPattern.randomSampling.learnMore"
+                      defaultMessage="View documentation."
+                    />
+                  </EuiLink>
+                ),
+              }}
+            />
+          }
+        >
+          {isEditing ? (
+            <CodeEditor
+              height={120}
+              languageId="json"
+              value={jsonOptions}
+              onChange={(value) => {
+                try {
+                  const additionalParameters =
+                    value === ''
+                      ? undefined
+                      : (JSON.parse(value) as FieldDefinitionConfigAdvancedParameters);
+                  onChange({ additionalParameters });
+                  if (onValidate) onValidate(getValidFlag(additionalParameters));
+                  resetParsingErrorFlag();
+                } catch (error: unknown) {
+                  markAsParsingError();
+                  if (onValidate) onValidate(false);
+                }
+              }}
+            />
+          ) : (
+            <EuiCodeBlock language="json" isCopyable>
+              {jsonOptions}
+            </EuiCodeBlock>
+          )}
+        </EuiFormRow>
      </EuiPanel>
    </EuiAccordion>
  );
 };
+
+const getValidFlag = (additionalParameters?: FieldDefinitionConfigAdvancedParameters) => {
+  return Boolean(
+    !additionalParameters ||
+      additionalParameters === '' ||
+      isSchema(recursiveRecord, additionalParameters)
+  );
+};
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/field_form_type.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/field_form_type.tsx
@ -67,6 +67,13 @@ interface FieldTypeSelectorProps {
  value?: FieldTypeOption;
 }

+const typeSelectorOptions = Object.entries(FIELD_TYPE_MAP)
+  .filter(([_, { readonly }]) => !readonly)
+  .map(([optionKey, { label }]) => ({
+    text: label,
+    value: optionKey,
+  }));
+
 const FieldTypeSelector = ({ value, onChange, isLoading = false }: FieldTypeSelectorProps) => {
  return (
    <EuiSelect
@ -77,10 +84,7 @@ const FieldTypeSelector = ({ value, onChange, isLoading = false }: FieldTypeSele
        onChange(event.target.value as FieldTypeOption);
      }}
      value={value}
-      options={Object.entries(FIELD_TYPE_MAP).map(([optionKey, optionConfig]) => ({
-        text: optionConfig.label,
-        value: optionKey,
-      }))}
+      options={typeSelectorOptions}
    />
  );
 };
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/field_summary.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/field_summary.tsx
@ -108,6 +108,7 @@ export const FieldSummary = (props: FieldSummaryProps) => {
                        tab: 'schemaEditor',
                      },
                    })}
+                    target="_blank"
                  >
                    {i18n.translate('xpack.streams.fieldSummary.editInParentButtonLabel', {
                      defaultMessage: 'Edit in parent stream',
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/index.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/index.tsx
@ -15,7 +15,7 @@ import {
  EuiTitle,
  EuiButton,
 } from '@elastic/eui';
-import React, { useReducer } from 'react';
+import React, { useReducer, useState } from 'react';
 import { i18n } from '@kbn/i18n';
 import { IngestStreamDefinition } from '@kbn/streams-schema';
 import useAsyncFn from 'react-use/lib/useAsyncFn';
@ -43,6 +43,8 @@ export const SchemaEditorFlyout = ({
  withFieldSimulation = false,
 }: SchemaEditorFlyoutProps) => {
  const [isEditing, toggleEditMode] = useToggle(isEditingByDefault);
+  const [isValidAdvancedFieldMappings, setValidAdvancedFieldMappings] = useState(true);
+  const [isValidSimulation, setValidSimulation] = useState(true);

  const [nextField, setNextField] = useReducer(
    (prev: SchemaField, updated: Partial<SchemaField>) =>
@ -53,6 +55,8 @@ export const SchemaEditorFlyout = ({
    field
  );

+  const hasValidFieldType = nextField.type !== undefined;
+
  const [{ loading: isSaving }, saveChanges] = useAsyncFn(async () => {
    await onSave(nextField);
    if (onClose) onClose();
@ -78,39 +82,52 @@ export const SchemaEditorFlyout = ({
          <AdvancedFieldMappingOptions
            field={nextField}
            onChange={setNextField}
+            onValidate={setValidAdvancedFieldMappings}
            isEditing={isEditing}
          />
          {withFieldSimulation && (
            <EuiFlexItem grow={false}>
-              <SamplePreviewTable stream={stream} nextField={nextField} />
+              <SamplePreviewTable
+                stream={stream}
+                nextField={nextField}
+                onValidate={setValidSimulation}
+              />
            </EuiFlexItem>
          )}
        </EuiFlexGroup>
      </EuiFlyoutBody>

-      <EuiFlyoutFooter>
-        <EuiFlexGroup justifyContent="spaceBetween">
-          <EuiButtonEmpty
-            data-test-subj="streamsAppSchemaEditorFlyoutCloseButton"
-            iconType="cross"
-            onClick={onClose}
-            flush="left"
-          >
-            {i18n.translate('xpack.streams.schemaEditorFlyout.closeButtonLabel', {
-              defaultMessage: 'Cancel',
-            })}
-          </EuiButtonEmpty>
-          <EuiButton
-            data-test-subj="streamsAppSchemaEditorFieldSaveButton"
-            isLoading={isSaving}
-            onClick={saveChanges}
-          >
-            {i18n.translate('xpack.streams.fieldForm.saveButtonLabel', {
-              defaultMessage: 'Save changes',
-            })}
-          </EuiButton>
-        </EuiFlexGroup>
-      </EuiFlyoutFooter>
+      {isEditing && (
+        <EuiFlyoutFooter>
+          <EuiFlexGroup justifyContent="spaceBetween">
+            <EuiButtonEmpty
+              data-test-subj="streamsAppSchemaEditorFlyoutCloseButton"
+              iconType="cross"
+              onClick={onClose}
+              flush="left"
+            >
+              {i18n.translate('xpack.streams.schemaEditorFlyout.closeButtonLabel', {
+                defaultMessage: 'Cancel',
+              })}
+            </EuiButtonEmpty>
+            <EuiButton
+              data-test-subj="streamsAppSchemaEditorFieldSaveButton"
+              disabled={
+                isSaving ||
+                !hasValidFieldType ||
+                !isValidAdvancedFieldMappings ||
+                !isValidSimulation
+              }
+              isLoading={isSaving}
+              onClick={saveChanges}
+            >
+              {i18n.translate('xpack.streams.fieldForm.saveButtonLabel', {
+                defaultMessage: 'Save changes',
+              })}
+            </EuiButton>
+          </EuiFlexGroup>
+        </EuiFlyoutFooter>
+      )}
    </>
  );
 };
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/sample_preview_table.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/schema_editor/flyout/sample_preview_table.tsx
@ -5,7 +5,7 @@
 * 2.0.
 */

-import React, { useMemo } from 'react';
+import React, { useEffect, useMemo } from 'react';
 import { css } from '@emotion/react';
 import { i18n } from '@kbn/i18n';
 import { EuiCallOut } from '@elastic/eui';
@ -21,6 +21,7 @@ import { convertToFieldDefinitionConfig } from '../utils';
 interface SamplePreviewTableProps {
  stream: IngestStreamDefinition;
  nextField: SchemaField;
+  onValidate?: (isValid: boolean) => void;
 }

 export const SamplePreviewTable = (props: SamplePreviewTableProps) => {
@ -37,6 +38,7 @@ const SAMPLE_DOCUMENTS_TO_SHOW = 20;
 const SamplePreviewTableContent = ({
  stream,
  nextField,
+  onValidate,
 }: SamplePreviewTableProps & { nextField: MappedSchemaField }) => {
  const { streamsRepositoryClient } = useKibana().dependencies.start.streams;

@ -63,6 +65,12 @@ const SamplePreviewTableContent = ({
    { disableToastOnError: true }
  );

+  useEffect(() => {
+    if (onValidate) {
+      onValidate(value?.status === 'failure' || error ? false : true);
+    }
+  }, [value, error, onValidate]);
+
  const columns = useMemo(() => {
    return [nextField.name];
  }, [nextField.name]);