[Streams] Schema Editor advanced field mapping options (#210667)

## Summary Closes https://github.com/elastic/streams-program/issues/88 Adds JSON advanced field mapping parameters to the Schema Editor. Main questions here are around the types and data structure. In this PR these are added as an `additionalProperties` property, but we may also want to have all of these parameters top level (like `type` and `format`). This version makes separating concerns easier in the UI and separating "first class" options vs advanced options, I could see pros and cons to both, and also things might be "upgraded" from advanced to first class later on. Also an open question on whether the `MappingProperty` type needs to be explicitly redefined for Zod (ES will obviously reject anything that isn't supported here). ![Screenshot 2025-02-12 at 13 01 35](https://github.com/user-attachments/assets/7082fed7-f445-476f-abb7-8f41d693d378) ![json_params](https://github.com/user-attachments/assets/521df9bf-cbd4-468a-9385-5787cdd5f906)
2025-04-24 17:59:23 -04:00 · 2025-02-14 13:52:18 +00:00 · 2025-02-14 13:52:18 +00:00 · c9dfe9aab1
commit c9dfe9aab1
parent 791be62934
12 changed files with 192 additions and 25 deletions
--- a/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/field_definition.ts
+++ b/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/field_definition.ts
@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { omit } from 'lodash';
+import { FieldDefinitionConfig } from '../models';
+
+// Parameters that we consider first class and provide a curated experience for
+const FIRST_CLASS_PARAMETERS = ['type', 'format'];
+
+// Advanced parameters that we provide a generic experience (JSON blob) for
+export const getAdvancedParameters = (fieldName: string, fieldConfig: FieldDefinitionConfig) => {
+  // @timestamp can't ignore malformed dates as it's used for sorting in logsdb
+  const additionalOmissions = fieldName === '@timestamp' ? ['ignore_malformed'] : [];
+  return omit(fieldConfig, FIRST_CLASS_PARAMETERS.concat(additionalOmissions));
+};
--- a/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts
+++ b/x-pack/solutions/observability/packages/kbn-streams-schema/src/helpers/index.ts
@ -10,3 +10,4 @@ export * from './hierarchy';
 export * from './lifecycle';
 export * from './condition_fields';
 export * from './condition_to_query_dsl';
+export * from './field_definition';
--- a/x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts
+++ b/x-pack/solutions/observability/packages/kbn-streams-schema/src/models/ingest/fields/index.ts
@ -5,6 +5,7 @@
 * 2.0.
 */

+import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
 import { z } from '@kbn/zod';
 import { NonEmptyString } from '@kbn/zod-helpers';

@ -20,15 +21,25 @@ export const FIELD_DEFINITION_TYPES = [

 export type FieldDefinitionType = (typeof FIELD_DEFINITION_TYPES)[number];

-export interface FieldDefinitionConfig {
+// We redefine "first class" parameters
+export type FieldDefinitionConfig = MappingProperty & {
  type: FieldDefinitionType;
  format?: string;
-}
+};

-export const fieldDefinitionConfigSchema: z.Schema<FieldDefinitionConfig> = z.object({
-  type: z.enum(FIELD_DEFINITION_TYPES),
-  format: z.optional(NonEmptyString),
-});
+// Parameters that we provide a generic (JSON blob) experience for
+export type FieldDefinitionConfigAdvancedParameters = Omit<
+  FieldDefinitionConfig,
+  'type' | 'format'
+>;
+
+export const fieldDefinitionConfigSchema: z.Schema<FieldDefinitionConfig> = z.intersection(
+  z.record(z.string(), z.unknown()),
+  z.object({
+    type: z.enum(FIELD_DEFINITION_TYPES),
+    format: z.optional(NonEmptyString),
+  })
+);

 export interface FieldDefinition {
  [x: string]: FieldDefinitionConfig;
@ -39,9 +50,9 @@ export const fieldDefinitionSchema: z.Schema<FieldDefinition> = z.record(
  fieldDefinitionConfigSchema
 );

-export interface InheritedFieldDefinitionConfig extends FieldDefinitionConfig {
+export type InheritedFieldDefinitionConfig = FieldDefinitionConfig & {
  from: string;
-}
+};

 export interface InheritedFieldDefinition {
  [x: string]: InheritedFieldDefinitionConfig;
@ -52,9 +63,9 @@ export const inheritedFieldDefinitionSchema: z.Schema<InheritedFieldDefinition>
  z.intersection(fieldDefinitionConfigSchema, z.object({ from: NonEmptyString }))
 );

-export interface NamedFieldDefinitionConfig extends FieldDefinitionConfig {
+export type NamedFieldDefinitionConfig = FieldDefinitionConfig & {
  name: string;
-}
+};

 export const namedFieldDefinitionConfigSchema: z.Schema<NamedFieldDefinitionConfig> =
  z.intersection(
--- a/x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts
+++ b/x-pack/solutions/observability/plugins/streams/server/lib/streams/component_templates/generate_layer.ts
@ -10,7 +10,13 @@ import {
  MappingDateProperty,
  MappingProperty,
 } from '@elastic/elasticsearch/lib/api/types';
-import { WiredStreamDefinition, isDslLifecycle, isIlmLifecycle, isRoot } from '@kbn/streams-schema';
+import {
+  WiredStreamDefinition,
+  getAdvancedParameters,
+  isDslLifecycle,
+  isIlmLifecycle,
+  isRoot,
+} from '@kbn/streams-schema';
 import { ASSET_VERSION } from '../../../../common/constants';
 import { logsSettings } from './logs_layer';
 import { getComponentTemplateName } from './name';
@ -25,6 +31,13 @@ export function generateLayer(
    const property: MappingProperty = {
      type: props.type,
    };
+
+    const advancedParameters = getAdvancedParameters(field, props);
+
+    if (Object.keys(advancedParameters).length > 0) {
+      Object.assign(property, advancedParameters);
+    }
+
    if (field === '@timestamp') {
      // @timestamp can't ignore malformed dates as it's used for sorting in logsdb
      (property as MappingDateProperty).ignore_malformed = false;
@ -32,6 +45,7 @@ export function generateLayer(
    if (props.type === 'date' && props.format) {
      (property as MappingDateProperty).format = props.format;
    }
+
    properties[field] = property;
  });

--- a/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts
+++ b/x-pack/solutions/observability/plugins/streams/server/routes/streams/schema/route.ts
@ -165,7 +165,10 @@ export const schemaFieldsSimulationRoute = createServerRoute({
    const propertiesForSimulation = Object.fromEntries(
      params.body.field_definitions.map((field) => [
        field.name,
-        { type: field.type, ...(field.format ? { format: field.format } : {}) },
+        {
+          type: field.type,
+          ...(field.format ? { format: field.format } : {}),
+        },
      ])
    );

--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/advanced_field_mapping_options.tsx
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/advanced_field_mapping_options.tsx
@ -0,0 +1,95 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import {
+  EuiAccordion,
+  EuiCodeBlock,
+  EuiLink,
+  EuiPanel,
+  EuiSpacer,
+  EuiText,
+  useGeneratedHtmlId,
+} from '@elastic/eui';
+import { CodeEditor } from '@kbn/code-editor';
+import { i18n } from '@kbn/i18n';
+import React, { useMemo } from 'react';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { getAdvancedParameters } from '@kbn/streams-schema';
+import { SchemaField } from '../types';
+import { useKibana } from '../../../hooks/use_kibana';
+
+const label = i18n.translate('xpack.streams.advancedFieldMappingOptions.label', {
+  defaultMessage: 'Advanced field mapping parameters',
+});
+
+export const AdvancedFieldMappingOptions = ({
+  field,
+  onChange,
+  isEditing,
+}: {
+  field: SchemaField;
+  onChange: (field: Partial<SchemaField>) => void;
+  isEditing: boolean;
+}) => {
+  const { core } = useKibana();
+
+  const accordionId = useGeneratedHtmlId({ prefix: 'accordionID' });
+
+  const jsonOptions = useMemo(
+    () => (field.additionalParameters ? JSON.stringify(field.additionalParameters, null, 2) : ''),
+    [field.additionalParameters]
+  );
+
+  return (
+    <EuiAccordion id={accordionId} buttonContent={label}>
+      <EuiPanel color="subdued">
+        <EuiText size="xs">
+          <FormattedMessage
+            id="xpack.streams.advancedFieldMappingOptions.docs.label"
+            defaultMessage="Parameters can be defined with JSON. {link}"
+            values={{
+              link: (
+                <EuiLink
+                  data-test-subj="streamsAppAdvancedFieldMappingOptionsViewDocumentationLink"
+                  href={core.docLinks.links.elasticsearch.docsBase.concat('mapping-params.html')}
+                  target="_blank"
+                  external
+                >
+                  <FormattedMessage
+                    id="xpack.streams.indexPattern.randomSampling.learnMore"
+                    defaultMessage="View documentation."
+                  />
+                </EuiLink>
+              ),
+            }}
+          />
+        </EuiText>
+        <EuiSpacer size="s" />
+        {isEditing ? (
+          <CodeEditor
+            height={120}
+            languageId="json"
+            value={jsonOptions}
+            onChange={(value) => {
+              try {
+                onChange({
+                  additionalParameters:
+                    value === '' ? undefined : getAdvancedParameters(field.name, JSON.parse(value)),
+                });
+              } catch (error: unknown) {
+                // do nothing
+              }
+            }}
+          />
+        ) : (
+          <EuiCodeBlock language="json" isCopyable>
+            {jsonOptions ?? ''}
+          </EuiCodeBlock>
+        )}
+      </EuiPanel>
+    </EuiAccordion>
+  );
+};
--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/field_summary.tsx
@ -16,7 +16,6 @@ import {
 } from '@elastic/eui';
 import React from 'react';
 import { i18n } from '@kbn/i18n';
-import useToggle from 'react-use/lib/useToggle';
 import { WiredStreamDefinition } from '@kbn/streams-schema';
 import { useStreamsAppRouter } from '../../../hooks/use_streams_app_router';
 import { FieldParent } from '../field_parent';
@ -56,18 +55,17 @@ const FIELD_SUMMARIES = {

 interface FieldSummaryProps {
  field: SchemaField;
-  isEditingByDefault: boolean;
+  isEditing: boolean;
+  toggleEditMode: () => void;
  stream: WiredStreamDefinition;
  onChange: (field: Partial<SchemaField>) => void;
 }

 export const FieldSummary = (props: FieldSummaryProps) => {
-  const { field, isEditingByDefault, onChange, stream } = props;
+  const { field, isEditing, toggleEditMode, onChange, stream } = props;

  const router = useStreamsAppRouter();

-  const [isEditing, toggleEditMode] = useToggle(isEditingByDefault);
-
  return (
    <>
      <EuiFlexGroup direction="column" gutterSize="s">
--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/flyout/index.tsx
@ -19,9 +19,11 @@ import React, { useReducer } from 'react';
 import { i18n } from '@kbn/i18n';
 import { WiredStreamDefinition } from '@kbn/streams-schema';
 import useAsyncFn from 'react-use/lib/useAsyncFn';
+import useToggle from 'react-use/lib/useToggle';
 import { SamplePreviewTable } from './sample_preview_table';
 import { FieldSummary } from './field_summary';
 import { SchemaField } from '../types';
+import { AdvancedFieldMappingOptions } from './advanced_field_mapping_options';

 export interface SchemaEditorFlyoutProps {
  field: SchemaField;
@ -40,6 +42,8 @@ export const SchemaEditorFlyout = ({
  isEditingByDefault = false,
  withFieldSimulation = false,
 }: SchemaEditorFlyoutProps) => {
+  const [isEditing, toggleEditMode] = useToggle(isEditingByDefault);
+
  const [nextField, setNextField] = useReducer(
    (prev: SchemaField, updated: Partial<SchemaField>) =>
      ({
@ -66,10 +70,16 @@ export const SchemaEditorFlyout = ({
        <EuiFlexGroup direction="column">
          <FieldSummary
            field={nextField}
-            isEditingByDefault={isEditingByDefault}
+            isEditing={isEditing}
+            toggleEditMode={toggleEditMode}
            onChange={setNextField}
            stream={stream}
          />
+          <AdvancedFieldMappingOptions
+            field={nextField}
+            onChange={setNextField}
+            isEditing={isEditing}
+          />
          {withFieldSimulation && (
            <EuiFlexItem grow={false}>
              <SamplePreviewTable stream={stream} nextField={nextField} />
--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/hooks/use_schema_fields.ts
@ -7,7 +7,11 @@

 import { i18n } from '@kbn/i18n';
 import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller';
-import { NamedFieldDefinitionConfig, WiredStreamGetResponse } from '@kbn/streams-schema';
+import {
+  NamedFieldDefinitionConfig,
+  WiredStreamGetResponse,
+  getAdvancedParameters,
+} from '@kbn/streams-schema';
 import { isEqual, omit } from 'lodash';
 import { useMemo, useCallback } from 'react';
 import { useStreamsAppFetch } from '../../../hooks/use_streams_app_fetch';
@ -59,6 +63,7 @@ export const useSchemaFields = ({
        name,
        type: field.type,
        format: field.format,
+        additionalParameters: getAdvancedParameters(name, field),
        parent: field.from,
        status: 'inherited',
      })
@ -69,6 +74,7 @@ export const useSchemaFields = ({
        name,
        type: field.type,
        format: field.format,
+        additionalParameters: getAdvancedParameters(name, field),
        parent: definition.stream.name,
        status: 'mapped',
      })
@ -132,12 +138,12 @@ export const useSchemaFields = ({

        refreshFields();
      } catch (error) {
-        toasts.addError(error, {
+        toasts.addError(new Error(error.body.message), {
          title: i18n.translate('xpack.streams.streamDetailSchemaEditorEditErrorToast', {
            defaultMessage: 'Something went wrong editing the {field} field',
            values: { field: field.name },
          }),
-          toastMessage: error.message,
+          toastMessage: error.body.message,
          toastLifeTimeMs: 5000,
        });
      }
--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/schema_editor_table.tsx
@ -107,7 +107,7 @@ const createCellRenderer =
      return <FieldStatusBadge status={status} />;
    }

-    return field[columnId as keyof SchemaField] || EMPTY_CONTENT;
+    return <>{field[columnId as keyof SchemaField] || EMPTY_CONTENT}</>;
  };

 const createFieldActionsCellRenderer = (fields: SchemaField[]): EuiDataGridControlColumn => ({
--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/types.ts
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/types.ts
@ -5,7 +5,11 @@
 * 2.0.
 */

-import { FieldDefinitionConfig, WiredStreamDefinition } from '@kbn/streams-schema';
+import {
+  FieldDefinitionConfig,
+  FieldDefinitionConfigAdvancedParameters,
+  WiredStreamDefinition,
+} from '@kbn/streams-schema';

 export type SchemaFieldStatus = 'inherited' | 'mapped' | 'unmapped';
 export type SchemaFieldType = FieldDefinitionConfig['type'];
@ -13,16 +17,19 @@ export type SchemaFieldType = FieldDefinitionConfig['type'];
 export interface BaseSchemaField extends Omit<FieldDefinitionConfig, 'type'> {
  name: string;
  parent: string;
+  format?: string;
 }

 export interface MappedSchemaField extends BaseSchemaField {
  status: 'inherited' | 'mapped';
  type: SchemaFieldType;
+  additionalParameters?: FieldDefinitionConfigAdvancedParameters;
 }

 export interface UnmappedSchemaField extends BaseSchemaField {
  status: 'unmapped';
-  type?: SchemaFieldType | undefined;
+  type?: SchemaFieldType;
+  additionalParameters?: FieldDefinitionConfigAdvancedParameters;
 }

 export type SchemaField = MappedSchemaField | UnmappedSchemaField;
--- a/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/utils.ts
+++ b/x-pack/solutions/observability/plugins/streams_app/public/components/schema_editor/utils.ts
@ -12,5 +12,8 @@ export const convertToFieldDefinitionConfig = (
  field: MappedSchemaField
 ): FieldDefinitionConfig => ({
  type: field.type,
-  ...(field.format && field.type === 'date' ? { format: field.format } : {}),
+  ...(field.format && field.type === 'date' ? { format: field.format as string } : {}),
+  ...(field.additionalParameters && Object.keys(field.additionalParameters).length > 0
+    ? field.additionalParameters
+    : {}),
 });