github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Streams 🌊] Add processors validation and simulation gate (#206566)

Browse source 
## 📓 Summary

Closes https://github.com/elastic/streams-program/issues/66

This work adds changes to prevent invalid processors from being
submitted.
The main rule is that a simulation is performed before any add/edit
submission to guarantee that the processor config is valid.

This work also updates the simulation API to detect whether there is a
non-additive change in any simulated document.

@patpscal error reporting UI for add/edit is different since the
simulator is not visible for edit, I used a callout but we can easily
update this once there is a final design in place.

### Form validation + simulation


https://github.com/user-attachments/assets/f7fc351b-6efc-4500-8490-b7f1c85139bf

### Non-additive processors


https://github.com/user-attachments/assets/47b5b739-c2cf-4a74-93a8-6ef43521c7d4
This commit is contained in:
Marco Antonio Ghiani 2025-01-15 16:09:02 +01:00 • committed by GitHub
parent b4342f44f0
commit 6429c53597
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 454 additions and 216 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/platform/packages/shared/kbn-object-utils/README.md
View file

@ -44,6 +44,7 @@ Result:
},
},
},
updated: {}
}
*/
```

23
src/platform/packages/shared/kbn-object-utils/src/calculate_object_diff.test.ts
View file

@ -10,25 +10,34 @@
import { calculateObjectDiff } from './calculate_object_diff';
describe('calculateObjectDiff', () => {
it('should return the added and removed parts between 2 objects', () => {
const { added, removed } = calculateObjectDiff({ alpha: 1, beta: 2 }, { alpha: 1, gamma: 3 });
it('should return the added, removed and updated parts between 2 objects', () => {
const { added, removed, updated } = calculateObjectDiff(
{ alpha: 1, beta: 2, sigma: 4 },
{ alpha: 1, gamma: 3, sigma: 5 }
);
expect(added).toEqual({ gamma: 3 });
expect(removed).toEqual({ beta: 2 });
expect(updated).toEqual({ sigma: 5 });
});
it('should work on nested objects', () => {
const { added, removed } = calculateObjectDiff(
{ alpha: 1, beta: { gamma: 2, delta: { sigma: 7 } } },
{ alpha: 1, beta: { gamma: 2, eta: 4 } }
const { added, removed, updated } = calculateObjectDiff(
{ alpha: 1, beta: { gamma: 2, delta: { sigma: 7, omega: 8 } } },
{ alpha: 1, beta: { gamma: 2, delta: { omega: 9 }, eta: 4 } }
);
expect(added).toEqual({ beta: { eta: 4 } });
expect(removed).toEqual({ beta: { delta: { sigma: 7 } } });
expect(updated).toEqual({ beta: { delta: { omega: 9 } } });
});
it('should return empty added/removed when the objects are the same', () => {
const { added, removed } = calculateObjectDiff({ alpha: 1, beta: 2 }, { alpha: 1, beta: 2 });
it('should return empty added/removed/updated when the objects are the same', () => {
const { added, removed, updated } = calculateObjectDiff(
{ alpha: 1, beta: 2 },
{ alpha: 1, beta: 2 }
);
expect(added).toEqual({});
expect(removed).toEqual({});
expect(updated).toEqual({});
});
});

23
src/platform/packages/shared/kbn-object-utils/src/calculate_object_diff.ts
View file

@ -22,6 +22,9 @@ type DeepPartial<TInputObj> = {
interface ObjectDiffResult<TBase, TCompare> {
added: DeepPartial<TCompare>;
removed: DeepPartial<TBase>;
updated: {
[K in keyof TBase & keyof TCompare]?: TBase[K] extends TCompare[K] ? never : TCompare[K];
};
}
/**
@ -34,16 +37,18 @@ export function calculateObjectDiff<TBase extends Obj, TCompare extends Obj>(
oldObj: TBase,
newObj?: TCompare
): ObjectDiffResult<TBase, TCompare> {
const added: DeepPartial<TCompare> = {};
const removed: DeepPartial<TBase> = {};
const added: ObjectDiffResult<TBase, TCompare>['added'] = {};
const removed: ObjectDiffResult<TBase, TCompare>['removed'] = {};
const updated: ObjectDiffResult<TBase, TCompare>['updated'] = {};
if (!newObj) return { added, removed };
if (!newObj) return { added, removed, updated };
function diffRecursive(
base: Obj,
compare: Obj,
addedMap: DeepPartial<Obj>,
removedMap: DeepPartial<Obj>
removedMap: DeepPartial<Obj>,
updatedMap: DeepPartial<Obj>
): void {
for (const key in compare) {
if (!(key in base)) {
@ -51,14 +56,18 @@ export function calculateObjectDiff<TBase extends Obj, TCompare extends Obj>(
} else if (isPlainObject(base[key]) && isPlainObject(compare[key])) {
addedMap[key] = {};
removedMap[key] = {};
updatedMap[key] = {};
diffRecursive(
base[key] as Obj,
compare[key] as Obj,
addedMap[key] as Obj,
removedMap[key] as Obj
removedMap[key] as Obj,
updatedMap[key] as Obj
);
if (isEmpty(addedMap[key])) delete addedMap[key];
if (isEmpty(removedMap[key])) delete removedMap[key];
} else if (base[key] !== compare[key]) {
updatedMap[key] = compare[key];
}
}
@ -69,7 +78,7 @@ export function calculateObjectDiff<TBase extends Obj, TCompare extends Obj>(
}
}
diffRecursive(oldObj, newObj, added, removed);
diffRecursive(oldObj, newObj, added, removed, updated);
return { added, removed };
return { added, removed, updated };
}

13
x-pack/solutions/observability/plugins/streams/server/lib/streams/errors/non_additive_processor.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export class NonAdditiveProcessor extends Error {
constructor(message: string) {
super(message);
this.name = 'NonAdditiveProcessor';
}
}

69
x-pack/solutions/observability/plugins/streams/server/routes/streams/processing/simulate.ts
View file

@ -13,6 +13,8 @@ import {
IngestSimulateResponse,
IngestSimulateSimulateDocumentResult,
} from '@elastic/elasticsearch/lib/api/types';
import { isEmpty } from 'lodash';
import { NonAdditiveProcessor } from '../../../lib/streams/errors/non_additive_processor';
import { SimulationFailed } from '../../../lib/streams/errors/simulation_failed';
import { formatToIngestProcessors } from '../../../lib/streams/helpers/processing';
import { createServerRoute } from '../../create_server_route';
@ -62,8 +64,17 @@ export const simulateProcessorRoute = createServerRoute({
throw new SimulationFailed(error);
}
const simulationDiffs = computeSimulationDiffs(simulationResult, docs);
const updatedFields = computeUpdatedFields(simulationDiffs);
if (!isEmpty(updatedFields)) {
throw new NonAdditiveProcessor(
`The processor is not additive to the documents. It might update fields [${updatedFields.join()}]`
);
}
const documents = computeSimulationDocuments(simulationResult, docs);
const detectedFields = computeDetectedFields(simulationResult, docs);
const detectedFields = computeDetectedFields(simulationDiffs);
const successRate = computeSuccessRate(simulationResult);
const failureRate = 1 - successRate;
@ -78,7 +89,7 @@ export const simulateProcessorRoute = createServerRoute({
throw notFound(error);
}
if (error instanceof SimulationFailed) {
if (error instanceof SimulationFailed || error instanceof NonAdditiveProcessor) {
throw badRequest(error);
}
@ -87,6 +98,35 @@ export const simulateProcessorRoute = createServerRoute({
},
});
const computeSimulationDiffs = (
simulation: IngestSimulateResponse,
sampleDocs: Array<{ _source: Record<string, unknown> }>
) => {
// Since we filter out failed documents, we need to map the simulation docs to the sample docs for later retrieval
const samplesToSimulationMap = new Map(simulation.docs.map((doc, id) => [doc, sampleDocs[id]]));
const diffs = simulation.docs.filter(isSuccessfulDocument).map((doc) => {
const sample = samplesToSimulationMap.get(doc);
if (sample) {
return calculateObjectDiff(sample._source, doc.processor_results.at(-1)?.doc?._source);
}
return calculateObjectDiff({});
});
return diffs;
};
const computeUpdatedFields = (simulationDiff: ReturnType<typeof computeSimulationDiffs>) => {
const diffs = simulationDiff
.map((simulatedDoc) => flattenObject(simulatedDoc.updated))
.flatMap(Object.keys);
const uniqueFields = [...new Set(diffs)];
return uniqueFields;
};
const computeSimulationDocuments = (
simulation: IngestSimulateResponse,
sampleDocs: Array<{ _source: Record<string, unknown> }>
@ -108,31 +148,14 @@ const computeSimulationDocuments = (
};
const computeDetectedFields = (
simulation: IngestSimulateResponse,
sampleDocs: Array<{ _source: Record<string, unknown> }>
simulationDiff: ReturnType<typeof computeSimulationDiffs>
): Array<{
name: string;
type: FieldDefinitionConfig['type'] | 'unmapped';
}> => {
// Since we filter out failed documents, we need to map the simulation docs to the sample docs for later retrieval
const samplesToSimulationMap = new Map(simulation.docs.map((doc, id) => [doc, sampleDocs[id]]));
const diffs = simulation.docs
.filter(isSuccessfulDocument)
.map((doc) => {
const sample = samplesToSimulationMap.get(doc);
if (sample) {
const { added } = calculateObjectDiff(
sample._source,
doc.processor_results.at(-1)?.doc?._source
);
return flattenObject(added);
}
return {};
})
.map(Object.keys)
.flat();
const diffs = simulationDiff
.map((simulatedDoc) => flattenObject(simulatedDoc.added))
.flatMap(Object.keys);
const uniqueFields = [...new Set(diffs)];

15
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/dissect/dissect_pattern_definition.tsx
View file

@ -17,7 +17,17 @@ export const DissectPatternDefinition = () => {
const { core } = useKibana();
const esDocUrl = core.docLinks.links.ingest.dissectKeyModifiers;
const { field, fieldState } = useController({ name: 'pattern' });
const { field, fieldState } = useController({
name: 'pattern',
rules: {
required: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.dissectPatternRequiredError',
{ defaultMessage: 'A pattern is required.' }
),
},
});
const { invalid, error } = fieldState;
return (
<EuiFormRow
@ -41,7 +51,8 @@ export const DissectPatternDefinition = () => {
}}
/>
}
isInvalid={fieldState.invalid}
isInvalid={invalid}
error={error?.message}
fullWidth
>
<CodeEditor

23
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/dissect/index.tsx
View file

@ -7,13 +7,12 @@
import React from 'react';
import { EuiSpacer } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { DissectAppendSeparator } from './dissect_append_separator';
import { DissectPatternDefinition } from './dissect_pattern_definition';
import { ProcessorFieldSelector } from '../processor_field_selector';
import { ToggleField } from '../toggle_field';
import { OptionalFieldsAccordion } from '../optional_fields_accordion';
import { ProcessorConditionEditor } from '../processor_condition_editor';
import { IgnoreFailureToggle, IgnoreMissingToggle } from '../ignore_toggles';
export const DissectProcessorForm = () => {
return (
@ -27,24 +26,8 @@ export const DissectProcessorForm = () => {
<ProcessorConditionEditor />
</OptionalFieldsAccordion>
<EuiSpacer size="m" />
<ToggleField
name="ignore_failure"
label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreFailuresLabel',
{ defaultMessage: 'Ignore failures for this processor' }
)}
/>
<ToggleField
name="ignore_missing"
label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreMissingLabel',
{ defaultMessage: 'Ignore missing' }
)}
helpText={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreMissingHelpText',
{ defaultMessage: 'Ignore documents with a missing field.' }
)}
/>
<IgnoreFailureToggle />
<IgnoreMissingToggle />
</>
);
};

91
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/grok/grok_patterns_editor.tsx
View file

@ -6,7 +6,13 @@
*/
import React from 'react';
import { useFormContext, useFieldArray, UseFormRegisterReturn } from 'react-hook-form';
import {
useFormContext,
useFieldArray,
UseFormRegisterReturn,
FieldError,
FieldErrorsImpl,
} from 'react-hook-form';
import {
DragDropContextProps,
EuiFormRow,
@ -24,11 +30,23 @@ import { SortableList } from '../../sortable_list';
import { GrokFormState } from '../../types';
export const GrokPatternsEditor = () => {
const { register } = useFormContext();
const {
formState: { errors },
register,
} = useFormContext();
const { fields, append, remove, move } = useFieldArray<Pick<GrokFormState, 'patterns'>>({
name: 'patterns',
});
const fieldsWithError = fields.map((field, id) => {
return {
...field,
error: (errors.patterns as unknown as FieldErrorsImpl[])?.[id]?.value as
| FieldError
| undefined,
};
});
const handlerPatternDrag: DragDropContextProps['onDragEnd'] = ({ source, destination }) => {
if (source && destination) {
move(source.index, destination.index);
@ -50,13 +68,18 @@ export const GrokPatternsEditor = () => {
>
<EuiPanel color="subdued" paddingSize="m">
<SortableList onDragItem={handlerPatternDrag}>
{fields.map((field, idx) => (
{fieldsWithError.map((field, idx) => (
<DraggablePatternInput
key={field.id}
pattern={field}
field={field}
idx={idx}
onRemove={getRemovePatternHandler(idx)}
inputProps={register(`patterns.${idx}.value`)}
inputProps={register(`patterns.${idx}.value`, {
required: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.grokEditorRequiredError',
{ defaultMessage: 'A pattern is required.' }
),
})}
/>
))}
</SortableList>
@ -73,25 +96,28 @@ export const GrokPatternsEditor = () => {
};
interface DraggablePatternInputProps {
field: GrokFormState['patterns'][number] & { id: string; error?: FieldError };
idx: number;
inputProps: UseFormRegisterReturn<`patterns.${number}.value`>;
onRemove: ((idx: number) => void) | null;
pattern: GrokFormState['patterns'][number] & { id: string };
}
const DraggablePatternInput = ({
field,
idx,
inputProps,
onRemove,
pattern,
}: DraggablePatternInputProps) => {
const { ref, ...inputPropsWithoutRef } = inputProps;
const { error, id } = field;
const isInvalid = Boolean(error);
return (
<EuiDraggable
index={idx}
spacing="m"
draggableId={pattern.id}
draggableId={id}
hasInteractiveChildren
customDragHandle
style={{
@ -100,28 +126,35 @@ const DraggablePatternInput = ({
}}
>
{(provided) => (
<EuiFlexGroup gutterSize="m" responsive={false} alignItems="center">
<EuiPanel
color="transparent"
paddingSize="xs"
{...provided.dragHandleProps}
aria-label="Drag Handle"
>
<EuiIcon type="grab" />
</EuiPanel>
<EuiFieldText {...inputPropsWithoutRef} inputRef={ref} compressed />
{onRemove && (
<EuiButtonIcon
iconType="minusInCircle"
color="danger"
onClick={() => onRemove(idx)}
aria-label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.grokEditor.removePattern',
{ defaultMessage: 'Remove grok pattern' }
)}
<EuiFormRow isInvalid={isInvalid} error={error?.message}>
<EuiFlexGroup gutterSize="m" responsive={false} alignItems="center">
<EuiPanel
color="transparent"
paddingSize="xs"
{...provided.dragHandleProps}
aria-label="Drag Handle"
>
<EuiIcon type="grab" />
</EuiPanel>
<EuiFieldText
{...inputPropsWithoutRef}
inputRef={ref}
compressed
isInvalid={isInvalid}
/>
)}
</EuiFlexGroup>
{onRemove && (
<EuiButtonIcon
iconType="minusInCircle"
color="danger"
onClick={() => onRemove(idx)}
aria-label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.grokEditor.removePattern',
{ defaultMessage: 'Remove grok pattern' }
)}
/>
)}
</EuiFlexGroup>
</EuiFormRow>
)}
</EuiDraggable>
);

23
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/grok/index.tsx
View file

@ -7,13 +7,12 @@
import React from 'react';
import { EuiSpacer } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { GrokPatternDefinition } from './grok_pattern_definition';
import { GrokPatternsEditor } from './grok_patterns_editor';
import { ProcessorFieldSelector } from '../processor_field_selector';
import { ToggleField } from '../toggle_field';
import { OptionalFieldsAccordion } from '../optional_fields_accordion';
import { ProcessorConditionEditor } from '../processor_condition_editor';
import { IgnoreFailureToggle, IgnoreMissingToggle } from '../ignore_toggles';
export const GrokProcessorForm = () => {
return (
@ -27,24 +26,8 @@ export const GrokProcessorForm = () => {
<ProcessorConditionEditor />
</OptionalFieldsAccordion>
<EuiSpacer size="m" />
<ToggleField
name="ignore_failure"
label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreFailuresLabel',
{ defaultMessage: 'Ignore failures for this processor' }
)}
/>
<ToggleField
name="ignore_missing"
label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreMissingLabel',
{ defaultMessage: 'Ignore missing' }
)}
helpText={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreMissingHelpText',
{ defaultMessage: 'Ignore documents with a missing field.' }
)}
/>
<IgnoreFailureToggle />
<IgnoreMissingToggle />
</>
);
};

56
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/ignore_toggles.tsx Normal file
View file

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import React from 'react';
import { i18n } from '@kbn/i18n';
import { useWatch } from 'react-hook-form';
import { EuiCode, EuiText } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n-react';
import { ToggleField } from './toggle_field';
export const IgnoreFailureToggle = () => {
const value = useWatch({ name: 'ignore_failure' });
return (
<ToggleField
name="ignore_failure"
label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreFailuresLabel',
{ defaultMessage: 'Ignore failures for this processor' }
)}
helpText={
!value ? (
<EuiText component="span" size="relative" color="warning">
<FormattedMessage
id="xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreFailuresWarning"
defaultMessage="Disabling the {ignoreField} option could lead to unexpected pipeline failures."
values={{
ignoreField: <EuiCode>ignore_failure</EuiCode>,
}}
/>
</EuiText>
) : undefined
}
/>
);
};
export const IgnoreMissingToggle = () => {
return (
<ToggleField
name="ignore_missing"
label={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreMissingLabel',
{ defaultMessage: 'Ignore missing' }
)}
helpText={i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.ignoreMissingHelpText',
{ defaultMessage: 'Ignore documents with a missing field.' }
)}
/>
);
};

44
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/index.tsx
View file

@ -19,6 +19,7 @@ import { DangerZone } from './danger_zone';
import { DissectProcessorForm } from './dissect';
import { GrokProcessorForm } from './grok';
import { convertFormStateToProcessing, getDefaultFormState } from '../utils';
import { useProcessingSimulator } from '../hooks/use_processing_simulator';
const ProcessorOutcomePreview = dynamic(() =>
import(/* webpackChunkName: "management_processor_outcome" */ './processor_outcome_preview').then(
@ -29,11 +30,11 @@ const ProcessorOutcomePreview = dynamic(() =>
);
export interface ProcessorFlyoutProps {
definition: ReadStreamDefinition;
onClose: () => void;
}
export interface AddProcessorFlyoutProps extends ProcessorFlyoutProps {
definition: ReadStreamDefinition;
onAddProcessor: (newProcessing: ProcessingDefinition, newFields?: DetectedField[]) => void;
}
export interface EditProcessorFlyoutProps extends ProcessorFlyoutProps {
@ -49,7 +50,7 @@ export function AddProcessorFlyout({
}: AddProcessorFlyoutProps) {
const defaultValues = useMemo(() => getDefaultFormState('grok'), []);
const methods = useForm<ProcessorFormState>({ defaultValues });
const methods = useForm<ProcessorFormState>({ defaultValues, mode: 'onChange' });
const formFields = methods.watch();
@ -58,11 +59,21 @@ export function AddProcessorFlyout({
[defaultValues, formFields]
);
const handleSubmit: SubmitHandler<ProcessorFormState> = (data) => {
const { error, isLoading, refreshSamples, simulation, samples, simulate } =
useProcessingSimulator({
definition,
condition: { field: formFields.field, operator: 'exists' },
});
const handleSubmit: SubmitHandler<ProcessorFormState> = async (data) => {
const processingDefinition = convertFormStateToProcessing(data);
onAddProcessor(processingDefinition, data.detected_fields);
onClose();
simulate(processingDefinition).then((responseBody) => {
if (responseBody instanceof Error) return;
onAddProcessor(processingDefinition, data.detected_fields);
onClose();
});
};
return (
@ -74,7 +85,10 @@ export function AddProcessorFlyout({
{ defaultMessage: 'Add processor' }
)}
confirmButton={
<EuiButton onClick={methods.handleSubmit(handleSubmit)}>
<EuiButton
onClick={methods.handleSubmit(handleSubmit)}
disabled={!methods.formState.isValid && methods.formState.isSubmitted}
>
{i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.confirmAddProcessor',
{ defaultMessage: 'Add processor' }
@ -90,7 +104,16 @@ export function AddProcessorFlyout({
{formFields.type === 'dissect' && <DissectProcessorForm />}
</EuiForm>
<EuiHorizontalRule />
<ProcessorOutcomePreview definition={definition} formFields={formFields} />
<ProcessorOutcomePreview
definition={definition}
formFields={formFields}
simulation={simulation}
samples={samples}
onSimulate={simulate}
onRefreshSamples={refreshSamples}
simulationError={error}
isLoading={isLoading}
/>
</FormProvider>
</ProcessorFlyoutTemplate>
);
@ -107,7 +130,7 @@ export function EditProcessorFlyout({
[processor]
);
const methods = useForm<ProcessorFormState>({ defaultValues });
const methods = useForm<ProcessorFormState>({ defaultValues, mode: 'onChange' });
const formFields = methods.watch();
@ -146,7 +169,10 @@ export function EditProcessorFlyout({
/>
}
confirmButton={
<EuiButton onClick={methods.handleSubmit(handleSubmit)} disabled={!hasChanges}>
<EuiButton
onClick={methods.handleSubmit(handleSubmit)}
disabled={!methods.formState.isValid}
>
{i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.confirmEditProcessor',
{ defaultMessage: 'Update processor' }

20
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/processor_field_selector.tsx
View file

@ -8,11 +8,21 @@
import { EuiFormRow, EuiFieldText } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import React from 'react';
import { useFormContext } from 'react-hook-form';
import { useController } from 'react-hook-form';
export const ProcessorFieldSelector = () => {
const { register } = useFormContext();
const { ref, ...inputProps } = register(`field`);
const { field, fieldState } = useController({
name: 'field',
rules: {
required: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.fieldSelectorRequiredError',
{ defaultMessage: 'A field value is required.' }
),
},
});
const { ref, ...inputProps } = field;
const { invalid, error } = fieldState;
return (
<EuiFormRow
@ -24,8 +34,10 @@ export const ProcessorFieldSelector = () => {
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.fieldSelectorHelpText',
{ defaultMessage: 'Field to search for matches.' }
)}
isInvalid={invalid}
error={error?.message}
>
<EuiFieldText {...inputProps} inputRef={ref} />
<EuiFieldText {...inputProps} inputRef={ref} isInvalid={invalid} />
</EuiFormRow>
);
};

127
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/processor_outcome_preview.tsx
View file

@ -35,107 +35,45 @@ import { useController, useFieldArray } from 'react-hook-form';
import { css } from '@emotion/react';
import { flattenObject } from '@kbn/object-utils';
import { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public';
import { useStreamsAppFetch } from '../../../hooks/use_streams_app_fetch';
import { useKibana } from '../../../hooks/use_kibana';
import { StreamsAppSearchBar, StreamsAppSearchBarProps } from '../../streams_app_search_bar';
import { PreviewTable } from '../../preview_table';
import { convertFormStateToProcessing, isCompleteProcessingDefinition } from '../utils';
import { convertFormStateToProcessing } from '../utils';
import { DetectedField, ProcessorFormState } from '../types';
import { UseProcessingSimulatorReturnType } from '../hooks/use_processing_simulator';
interface ProcessorOutcomePreviewProps {
definition: ReadStreamDefinition;
formFields: ProcessorFormState;
isLoading: UseProcessingSimulatorReturnType['isLoading'];
simulation: UseProcessingSimulatorReturnType['simulation'];
samples: UseProcessingSimulatorReturnType['samples'];
onRefreshSamples: UseProcessingSimulatorReturnType['refreshSamples'];
onSimulate: UseProcessingSimulatorReturnType['simulate'];
simulationError: UseProcessingSimulatorReturnType['error'];
}
export const ProcessorOutcomePreview = ({
definition,
formFields,
isLoading,
simulation,
samples,
onRefreshSamples,
onSimulate,
simulationError,
}: ProcessorOutcomePreviewProps) => {
const { dependencies } = useKibana();
const {
data,
streams: { streamsRepositoryClient },
} = dependencies.start;
const { data } = dependencies.start;
const {
timeRange,
absoluteTimeRange: { start, end },
setTimeRange,
} = useDateRange({ data });
const { timeRange, setTimeRange } = useDateRange({ data });
const [selectedDocsFilter, setSelectedDocsFilter] =
useState<DocsFilterOption>('outcome_filter_all');
const {
value: samples,
loading: isLoadingSamples,
refresh: refreshSamples,
} = useStreamsAppFetch(
({ signal }) => {
if (!definition || !formFields.field) {
return { documents: [] };
}
return streamsRepositoryClient.fetch('POST /api/streams/{id}/_sample', {
signal,
params: {
path: { id: definition.name },
body: {
condition: { field: formFields.field, operator: 'exists' },
start: start?.valueOf(),
end: end?.valueOf(),
number: 100,
},
},
});
},
[definition, formFields.field, streamsRepositoryClient, start, end],
{ disableToastOnError: true }
);
const {
value: simulation,
loading: isLoadingSimulation,
error,
refresh: refreshSimulation,
} = useStreamsAppFetch(
async ({ signal }) => {
if (!definition || !samples || isEmpty(samples.documents)) {
return Promise.resolve(null);
}
const processingDefinition = convertFormStateToProcessing(formFields);
if (!isCompleteProcessingDefinition(processingDefinition)) {
return Promise.resolve(null);
}
const simulationResult = await streamsRepositoryClient.fetch(
'POST /api/streams/{id}/processing/_simulate',
{
signal,
params: {
path: { id: definition.name },
body: {
documents: samples.documents as Array<Record<PropertyKey, unknown>>,
processing: [processingDefinition],
},
},
}
);
return simulationResult;
},
[definition, samples, streamsRepositoryClient],
{ disableToastOnError: true }
);
const simulationError = error as IHttpFetchError<ResponseErrorBody> | undefined;
const simulationDocuments = useMemo(() => {
if (!simulation?.documents) {
const docs = (samples?.documents ?? []) as Array<Record<PropertyKey, unknown>>;
return docs.map((doc) => flattenObject(doc));
return samples.map((doc) => flattenObject(doc));
}
const filterDocuments = (filter: DocsFilterOption) => {
@ -151,11 +89,24 @@ export const ProcessorOutcomePreview = ({
};
return filterDocuments(selectedDocsFilter).map((doc) => doc.value);
}, [samples?.documents, simulation?.documents, selectedDocsFilter]);
}, [samples, simulation?.documents, selectedDocsFilter]);
const detectedFieldsColumns = simulation?.detected_fields
? simulation.detected_fields.map((field) => field.name)
: [];
const detectedFieldsColumns = useMemo(
() =>
simulation?.detected_fields ? simulation.detected_fields.map((field) => field.name) : [],
[simulation?.detected_fields]
);
const tableColumns = useMemo(() => {
switch (selectedDocsFilter) {
case 'outcome_filter_unmatched':
return [formFields.field];
case 'outcome_filter_matched':
case 'outcome_filter_all':
default:
return [formFields.field, ...detectedFieldsColumns];
}
}, [formFields.field, detectedFieldsColumns, selectedDocsFilter]);
const detectedFieldsEnabled =
isWiredReadStream(definition) && simulation && !isEmpty(simulation.detected_fields);
@ -175,8 +126,8 @@ export const ProcessorOutcomePreview = ({
iconType="play"
color="accentSecondary"
size="s"
onClick={refreshSimulation}
isLoading={isLoadingSimulation}
onClick={() => onSimulate(convertFormStateToProcessing(formFields))}
isLoading={isLoading}
>
{i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processorFlyout.runSimulation',
@ -191,16 +142,16 @@ export const ProcessorOutcomePreview = ({
onDocsFilterChange={setSelectedDocsFilter}
timeRange={timeRange}
onTimeRangeChange={setTimeRange}
onTimeRangeRefresh={refreshSamples}
onTimeRangeRefresh={onRefreshSamples}
simulationFailureRate={simulation?.failure_rate}
simulationSuccessRate={simulation?.success_rate}
/>
<EuiSpacer size="m" />
<OutcomePreviewTable
documents={simulationDocuments}
columns={[formFields.field, ...detectedFieldsColumns]}
columns={tableColumns}
error={simulationError}
isLoading={isLoadingSamples || isLoadingSimulation}
isLoading={isLoading}
/>
</EuiPanel>
);

4
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/flyout/toggle_field.tsx
View file

@ -7,10 +7,10 @@
import React from 'react';
import { useController } from 'react-hook-form';
import { EuiFormRow, EuiSwitch, htmlIdGenerator } from '@elastic/eui';
import { EuiFormRow, EuiFormRowProps, EuiSwitch, htmlIdGenerator } from '@elastic/eui';
interface ToggleFieldProps {
helpText?: string;
helpText?: EuiFormRowProps['helpText'];
id?: string;
label: string;
name: string;

106
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/hooks/use_processing_simulator.ts Normal file
View file

@ -0,0 +1,106 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { useAbortController } from '@kbn/observability-utils-browser/hooks/use_abort_controller';
import { ReadStreamDefinition, ProcessingDefinition, Condition } from '@kbn/streams-schema';
import useAsyncFn from 'react-use/lib/useAsyncFn';
import { IHttpFetchError, ResponseErrorBody } from '@kbn/core/public';
import { useDateRange } from '@kbn/observability-utils-browser/hooks/use_date_range';
import { APIReturnType } from '@kbn/streams-plugin/public/api';
import { useStreamsAppFetch } from '../../../hooks/use_streams_app_fetch';
import { useKibana } from '../../../hooks/use_kibana';
type Simulation = APIReturnType<'POST /api/streams/{id}/processing/_simulate'>;
export interface UseProcessingSimulatorReturnType {
error?: IHttpFetchError<ResponseErrorBody>;
isLoading: boolean;
refreshSamples: () => void;
samples: Array<Record<PropertyKey, unknown>>;
simulate: (processing: ProcessingDefinition) => Promise<Simulation | null>;
simulation?: Simulation | null;
}
export const useProcessingSimulator = ({
definition,
condition,
}: {
definition: ReadStreamDefinition;
condition?: Condition;
}): UseProcessingSimulatorReturnType => {
const { dependencies } = useKibana();
const {
data,
streams: { streamsRepositoryClient },
} = dependencies.start;
const {
absoluteTimeRange: { start, end },
} = useDateRange({ data });
const abortController = useAbortController();
const serializedCondition = JSON.stringify(condition);
const {
loading: isLoadingSamples,
value: samples,
refresh: refreshSamples,
} = useStreamsAppFetch(
({ signal }) => {
if (!definition) {
return { documents: [] };
}
return streamsRepositoryClient.fetch('POST /api/streams/{id}/_sample', {
signal,
params: {
path: { id: definition.name },
body: {
condition,
start: start?.valueOf(),
end: end?.valueOf(),
number: 100,
},
},
});
},
[definition, streamsRepositoryClient, start, end, serializedCondition],
{ disableToastOnError: true }
);
const sampleDocs = (samples?.documents ?? []) as Array<Record<PropertyKey, unknown>>;
const [{ loading: isLoadingSimulation, error, value }, simulate] = useAsyncFn(
(processingDefinition: ProcessingDefinition) => {
if (!definition) {
return Promise.resolve(null);
}
return streamsRepositoryClient.fetch('POST /api/streams/{id}/processing/_simulate', {
signal: abortController.signal,
params: {
path: { id: definition.name },
body: {
documents: sampleDocs,
processing: [processingDefinition],
},
},
});
},
[definition, sampleDocs]
);
return {
isLoading: isLoadingSamples || isLoadingSimulation,
error: error as IHttpFetchError<ResponseErrorBody> | undefined,
refreshSamples,
simulate,
simulation: value,
samples: sampleDocs,
};
};

1
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/page_content.tsx
View file

@ -118,6 +118,7 @@ export function StreamDetailEnrichmentContent({
<DraggableProcessorListItem
key={processor.id}
idx={idx}
definition={definition}
processor={processor}
onUpdateProcessor={updateProcessor}
onDeleteProcessor={deleteProcessor}

10
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/processors_list.tsx
View file

@ -17,7 +17,12 @@ import {
EuiButtonIcon,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { getProcessorType, isDissectProcessor, isGrokProcessor } from '@kbn/streams-schema';
import {
ReadStreamDefinition,
getProcessorType,
isDissectProcessor,
isGrokProcessor,
} from '@kbn/streams-schema';
import { useBoolean } from '@kbn/react-hooks';
import { css } from '@emotion/react';
import { EditProcessorFlyout, EditProcessorFlyoutProps } from './flyout';
@ -45,6 +50,7 @@ export const DraggableProcessorListItem = ({
);
interface ProcessorListItemProps {
definition: ReadStreamDefinition;
processor: ProcessorDefinition;
hasShadow: EuiPanelProps['hasShadow'];
onUpdateProcessor: EditProcessorFlyoutProps['onUpdateProcessor'];
@ -52,6 +58,7 @@ interface ProcessorListItemProps {
}
const ProcessorListItem = ({
definition,
processor,
hasShadow = false,
onUpdateProcessor,
@ -93,6 +100,7 @@ const ProcessorListItem = ({
{isEditProcessorOpen && (
<EditProcessorFlyout
key={`edit-processor`}
definition={definition}
processor={processor}
onClose={closeEditProcessor}
onUpdateProcessor={onUpdateProcessor}

21
x-pack/solutions/observability/plugins/streams_app/public/components/stream_detail_enrichment/utils.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
/* eslint-disable @typescript-eslint/naming-convention */
import {
DissectProcessingDefinition,
GrokProcessingDefinition,
@ -84,7 +86,8 @@ export const convertFormStateToProcessing = (
formState: ProcessorFormState
): ProcessingDefinition => {
if (formState.type === 'grok') {
const { condition, patterns, ...grokConfig } = formState;
const { condition, patterns, field, pattern_definitions, ignore_failure, ignore_missing } =
formState;
return {
condition: isCompleteCondition(condition) ? condition : undefined,
@ -93,19 +96,29 @@ export const convertFormStateToProcessing = (
patterns: patterns
.filter(({ value }) => value.trim().length > 0)
.map(({ value }) => value),
...grokConfig,
field,
pattern_definitions,
ignore_failure,
ignore_missing,
},
},
};
}
if (formState.type === 'dissect') {
const { condition, ...dissectConfig } = formState;
const { condition, field, pattern, append_separator, ignore_failure, ignore_missing } =
formState;
return {
condition: isCompleteCondition(condition) ? condition : undefined,
config: {
dissect: dissectConfig,
dissect: {
field,
pattern,
append_separator,
ignore_failure,
ignore_missing,
},
},
};
}