[Streams 🌊] Add warning for dotted field names (#216154)

## 📓 Summary Closes #215887 Until the access to dotted fields is not supported, we'll warn the user about the unreliability of the simulation outcome when using those fields in processor. configurations. The unsupported fields that will make the warning appear are derived by the sample docs, deriving a list of existing fields that have some nested dot-separated field names. https://github.com/user-attachments/assets/46228821-601c-4a32-995c-1699be6c4ce3 ## 🧪 Test To reproduce it, ingest docs manually with ```tsx POST logs-mytest.otel-default/_doc { "body": { "text": "This is the message" }, "severity_text": "WARN", "resource": { "attributes": { "host.name": "my-host", "host.arch": "arm" } } } ``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Mike Birnstiehl <114418652+mdbirnstiehl@users.noreply.github.com>
2025-04-23 17:28:26 -04:00 · 2025-04-02 13:12:34 +02:00 · 2025-04-02 13:12:34 +02:00 · 9797e95289
commit 9797e95289
parent c05dda37e2
5 changed files with 157 additions and 36 deletions
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/processor_field_selector.tsx
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/processor_field_selector.tsx
@ -5,13 +5,22 @@
 * 2.0.
 */

-import { EuiFormRow, EuiFieldText } from '@elastic/eui';
+import { EuiFormRow, EuiFieldText, EuiCallOut, useEuiTheme } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import React from 'react';
 import { useController } from 'react-hook-form';
+import { css } from '@emotion/react';
 import { ProcessorFormState } from '../types';
+import { useSimulatorSelector } from '../state_management/stream_enrichment_state_machine';
+import { selectUnsupportedDottedFields } from '../state_management/simulation_state_machine/selectors';

 export const ProcessorFieldSelector = () => {
+  const { euiTheme } = useEuiTheme();
+
+  const unsupportedFields = useSimulatorSelector((state) =>
+    selectUnsupportedDottedFields(state.context)
+  );
+
  const { field, fieldState } = useController<ProcessorFormState, 'field'>({
    name: 'field',
    rules: {
@ -22,28 +31,70 @@ export const ProcessorFieldSelector = () => {
    },
  });

-  const { ref, ...inputProps } = field;
+  const { ref, value, ...inputProps } = field;
  const { invalid, error } = fieldState;

+  const isUnsupported = unsupportedFields.some((unsupportedField) =>
+    value.startsWith(unsupportedField)
+  );
+
  return (
-    <EuiFormRow
-      label={i18n.translate(
-        'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorLabel',
-        { defaultMessage: 'Field' }
-      )}
-      helpText={i18n.translate(
-        'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorHelpText',
-        { defaultMessage: 'Field to search for matches.' }
-      )}
-      isInvalid={invalid}
-      error={error?.message}
-    >
-      <EuiFieldText
-        data-test-subj="streamsAppProcessorFieldSelectorFieldText"
-        {...inputProps}
-        inputRef={ref}
+    <>
+      <EuiFormRow
+        label={i18n.translate(
+          'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorLabel',
+          { defaultMessage: 'Field' }
+        )}
+        helpText={i18n.translate(
+          'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorHelpText',
+          { defaultMessage: 'Field to search for matches.' }
+        )}
        isInvalid={invalid}
-      />
-    </EuiFormRow>
+        error={error?.message}
+      >
+        <EuiFieldText
+          data-test-subj="streamsAppProcessorFieldSelectorFieldText"
+          {...inputProps}
+          value={value}
+          inputRef={ref}
+          isInvalid={invalid}
+        />
+      </EuiFormRow>
+      {isUnsupported && (
+        <EuiCallOut
+          color="warning"
+          iconType="alert"
+          title={i18n.translate(
+            'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorUnsupportedDottedFieldsWarning.title',
+            {
+              defaultMessage: 'Dot-separated field names are not supported.',
+            }
+          )}
+          css={css`
+            margin-top: ${euiTheme.size.s};
+            margin-bottom: ${euiTheme.size.m};
+          `}
+        >
+          <p>
+            {i18n.translate(
+              'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorUnsupportedDottedFieldsWarning.p1',
+              {
+                defaultMessage:
+                  'Dot-separated field names in processors can produce misleading simulation results.',
+              }
+            )}
+          </p>
+          <p>
+            {i18n.translate(
+              'xpack.streams.streamDetailView.managementTab.enrichment.processor.fieldSelectorUnsupportedDottedFieldsWarning.p2',
+              {
+                defaultMessage:
+                  'For accurate results, avoid dot-separated field names or expand them into nested objects.',
+              }
+            )}
+          </p>
+        </EuiCallOut>
+      )}
+    </>
  );
 };
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/samples_fetcher_actor.ts
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/samples_fetcher_actor.ts
@ -6,8 +6,7 @@
 */

 import { i18n } from '@kbn/i18n';
-import { flattenObjectNestedLast } from '@kbn/object-utils';
-import { Condition, FlattenRecord } from '@kbn/streams-schema';
+import { Condition, SampleDocument } from '@kbn/streams-schema';
 import { fromPromise, ErrorActorEvent } from 'xstate5';
 import { errors as esErrors } from '@elastic/elasticsearch';
 import { DateRangeContext } from '../../../../../state_management/date_range_state_machine';
@ -22,7 +21,7 @@ export interface SamplesFetchInput {
 export function createSamplesFetchActor({
  streamsRepositoryClient,
 }: Pick<SimulationMachineDeps, 'streamsRepositoryClient'>) {
-  return fromPromise<FlattenRecord[], SamplesFetchInput>(async ({ input, signal }) => {
+  return fromPromise<SampleDocument[], SamplesFetchInput>(async ({ input, signal }) => {
    const samplesBody = await streamsRepositoryClient.fetch(
      'POST /internal/streams/{name}/_sample',
      {
@ -39,7 +38,7 @@ export function createSamplesFetchActor({
      }
    );

-    return samplesBody.documents.map(flattenObjectNestedLast) as FlattenRecord[];
+    return samplesBody.documents;
  });
 }

--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/selectors.ts
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/selectors.ts
@ -4,23 +4,88 @@
 * 2.0; you may not use this file except in compliance with the Elastic License
 * 2.0.
 */
+
 import { createSelector } from 'reselect';
+
+import { FlattenRecord, SampleDocument } from '@kbn/streams-schema';
+import { isPlainObject, uniq } from 'lodash';
+import { SimulationContext } from './types';
 import { filterSimulationDocuments } from './utils';
-import { SimulationActorSnapshot } from './simulation_state_machine';

 const EMPTY_ARRAY: [] = [];

+/**
+ * Selects the documents used for the data preview table.
+ */
 export const selectPreviewDocuments = createSelector(
  [
-    (snapshot: SimulationActorSnapshot['context']) => snapshot.samples,
-    (snapshot: SimulationActorSnapshot['context']) => snapshot.previewDocsFilter,
-    (snapshot: SimulationActorSnapshot['context']) => snapshot.simulation?.documents,
+    (context: SimulationContext) => context.samples,
+    (context: SimulationContext) => context.previewDocsFilter,
+    (context: SimulationContext) => context.simulation?.documents,
  ],
  (samples, previewDocsFilter, documents) => {
    return (
-      (previewDocsFilter && documents
+      ((previewDocsFilter && documents
        ? filterSimulationDocuments(documents, previewDocsFilter)
-        : samples) || EMPTY_ARRAY
+        : samples) as FlattenRecord[]) || EMPTY_ARRAY
    );
  }
 );
+
+/**
+ * Selects the set of dotted fields that are not supported by the current simulation.
+ */
+export const selectUnsupportedDottedFields = createSelector(
+  [(context: SimulationContext) => context.samples],
+  (samples) => {
+    const properties = samples.flatMap(getDottedFieldPrefixes);
+
+    return uniq(properties);
+  }
+);
+
+const isPlainObj = isPlainObject as (value: unknown) => value is Record<string, unknown>;
+
+/**
+ * Returns a list of all dotted properties prefixes in the given object.
+ */
+function getDottedFieldPrefixes(obj: SampleDocument): string[] {
+  const result: string[] = [];
+
+  function traverse(currentObj: SampleDocument, path: string[]): boolean {
+    let foundDot = false;
+
+    for (const key in currentObj) {
+      if (Object.hasOwn(currentObj, key)) {
+        const value = currentObj[key];
+        const newPath = [...path, key];
+
+        // Check if current key contains a dot
+        if (key.includes('.')) {
+          const newKey = newPath.join('.');
+          // For objects with dotted keys, add trailing dot
+          if (isPlainObj(value)) {
+            result.push(newKey.concat('.'));
+          } else {
+            result.push(newKey);
+          }
+          foundDot = true;
+          continue; // Skip further traversal for this key
+        }
+
+        // If it's an object, traverse deeper
+        if (isPlainObj(value) && traverse(value, newPath)) {
+          // If traversal found a dot, don't continue with siblings
+          foundDot = true;
+          continue;
+        }
+      }
+    }
+
+    return foundDot;
+  }
+
+  traverse(obj, []);
+
+  return result;
+}
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/simulation_state_machine.ts
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/simulation_state_machine.ts
@ -6,8 +6,14 @@
 */
 import { ActorRefFrom, MachineImplementationsFrom, SnapshotFrom, assign, setup } from 'xstate5';
 import { getPlaceholderFor } from '@kbn/xstate-utils';
-import { FlattenRecord, isSchema, processorDefinitionSchema } from '@kbn/streams-schema';
+import {
+  FlattenRecord,
+  SampleDocument,
+  isSchema,
+  processorDefinitionSchema,
+} from '@kbn/streams-schema';
 import { isEmpty, isEqual } from 'lodash';
+import { flattenObjectNestedLast } from '@kbn/object-utils';
 import {
  dateRangeMachine,
  createDateRangeMachineImplementations,
@ -38,7 +44,7 @@ export interface ProcessorEventParams {
  processors: ProcessorDefinitionWithUIAttributes[];
 }

-const hasSamples = (samples: FlattenRecord[]) => !isEmpty(samples);
+const hasSamples = (samples: SampleDocument[]) => !isEmpty(samples);

 const isValidProcessor = (processor: ProcessorDefinitionWithUIAttributes) =>
  isSchema(processorDefinitionSchema, processorConverter.toAPIDefinition(processor));
@ -66,7 +72,7 @@ export const simulationMachine = setup({
    storeProcessors: assign((_, params: ProcessorEventParams) => ({
      processors: params.processors,
    })),
-    storeSamples: assign((_, params: { samples: FlattenRecord[] }) => ({
+    storeSamples: assign((_, params: { samples: SampleDocument[] }) => ({
      samples: params.samples,
    })),
    storeSimulation: assign((_, params: { simulation: Simulation | undefined }) => ({
@ -231,7 +237,7 @@ export const simulationMachine = setup({
        src: 'runSimulation',
        input: ({ context }) => ({
          streamName: context.streamName,
-          documents: context.samples,
+          documents: context.samples.map(flattenObjectNestedLast) as FlattenRecord[],
          processors: context.processors,
        }),
        onDone: {
--- a/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/types.ts
+++ b/x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/types.ts
@ -5,7 +5,7 @@
 * 2.0.
 */

-import { Condition, FlattenRecord } from '@kbn/streams-schema';
+import { Condition, FlattenRecord, SampleDocument } from '@kbn/streams-schema';
 import { APIReturnType, StreamsRepositoryClient } from '@kbn/streams-plugin/public/api';
 import { IToasts } from '@kbn/core/public';
 import { DataPublicPluginStart } from '@kbn/data-plugin/public';
@ -46,7 +46,7 @@ export interface SimulationContext {
  previewDocsFilter: PreviewDocsFilterOption;
  previewDocuments: FlattenRecord[];
  processors: ProcessorDefinitionWithUIAttributes[];
-  samples: FlattenRecord[];
+  samples: SampleDocument[];
  samplingCondition?: Condition;
  simulation?: Simulation;
  streamName: string;