github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-24 01:38:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

[Streams 🌊] Update enrichment sample filters (#216463)

Browse source 
## 📓 Summary

These changes update the samples filters available during a processors
simulation.

Although this is a temporary update before we get a more complete
filtering experience, it improve filtering the docs by the simulation
status.


https://github.com/user-attachments/assets/a5c4c22a-6833-4744-bee5-e90a2ac1a389
This commit is contained in:
Marco Antonio Ghiani 2025-04-03 14:46:08 +02:00 • committed by GitHub
parent ed058086e2
commit c2113c44b3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 200 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

62
x-pack/platform/plugins/shared/streams/server/routes/internal/streams/processing/simulation_handler.ts
View file

@ -76,9 +76,9 @@ export interface SimulationDocReport {
export interface ProcessorMetrics {
detected_fields: string[];
errors: SimulationError[];
failure_rate: number;
failed_rate: number;
skipped_rate: number;
success_rate: number;
parsed_rate: number;
}
// Narrow down the type to only successful processor results
@ -397,7 +397,7 @@ const computePipelineSimulationResult = (
processorsMap[procId].errors.push(error);
if (error.type !== 'non_additive_processor_failure') {
processorsMap[procId].failure_rate++;
processorsMap[procId].failed_rate++;
}
});
@ -425,9 +425,9 @@ const initProcessorMetricsMap = (
{
detected_fields: [],
errors: [],
failure_rate: 0,
failed_rate: 0,
skipped_rate: 0,
success_rate: 1,
parsed_rate: 1,
},
]);
@ -442,18 +442,18 @@ const extractProcessorMetrics = ({
sampleSize: number;
}) => {
return mapValues(processorsMap, (metrics) => {
const failureRate = metrics.failure_rate / sampleSize;
const failureRate = metrics.failed_rate / sampleSize;
const skippedRate = metrics.skipped_rate / sampleSize;
const successRate = 1 - skippedRate - failureRate;
const parsedRate = 1 - skippedRate - failureRate;
const detected_fields = uniq(metrics.detected_fields);
const errors = uniqBy(metrics.errors, (error) => error.message);
return {
detected_fields,
errors,
failure_rate: parseFloat(failureRate.toFixed(2)),
failed_rate: parseFloat(failureRate.toFixed(2)),
skipped_rate: parseFloat(skippedRate.toFixed(2)),
success_rate: parseFloat(successRate.toFixed(2)),
parsed_rate: parseFloat(parsedRate.toFixed(2)),
};
});
};
@ -575,9 +575,13 @@ const prepareSimulationResponse = async (
processorsMetrics: Record<string, ProcessorMetrics>,
detectedFields: DetectedField[]
) => {
const successRate = computeSuccessRate(docReports);
const skippedRate = computeSkippedRate(docReports);
const failureRate = 1 - skippedRate - successRate;
const calculateRateByStatus = getRateCalculatorForDocs(docReports);
const parsedRate = calculateRateByStatus('parsed');
const partiallyParsedRate = calculateRateByStatus('partially_parsed');
const skippedRate = calculateRateByStatus('skipped');
const failureRate = calculateRateByStatus('failed');
const isNotAdditiveSimulation = some(processorsMetrics, (metrics) =>
metrics.errors.some(isNonAdditiveSimulationError)
);
@ -586,9 +590,12 @@ const prepareSimulationResponse = async (
detected_fields: detectedFields,
documents: docReports,
processors_metrics: processorsMetrics,
failure_rate: parseFloat(failureRate.toFixed(2)),
skipped_rate: parseFloat(skippedRate.toFixed(2)),
success_rate: parseFloat(successRate.toFixed(2)),
documents_metrics: {
failed_rate: parseFloat(failureRate.toFixed(2)),
partially_parsed_rate: parseFloat(partiallyParsedRate.toFixed(2)),
skipped_rate: parseFloat(skippedRate.toFixed(2)),
parsed_rate: parseFloat(parsedRate.toFixed(2)),
},
is_non_additive_simulation: isNotAdditiveSimulation,
};
};
@ -601,14 +608,17 @@ const prepareSimulationFailureResponse = (error: SimulationError) => {
[error.processor_id]: {
detected_fields: [],
errors: [error],
failure_rate: 1,
failed_rate: 1,
skipped_rate: 0,
success_rate: 0,
parsed_rate: 0,
},
},
failure_rate: 1,
skipped_rate: 0,
success_rate: 0,
documents_metrics: {
failed_rate: 1,
partially_parsed_rate: 0,
skipped_rate: 0,
parsed_rate: 0,
},
is_non_additive_simulation: isNonAdditiveSimulationError(error),
};
};
@ -659,16 +669,10 @@ const computeDetectedFields = async (
});
};
const computeSuccessRate = (docs: SimulationDocReport[]) => {
const successfulCount = docs.reduce((rate, doc) => (rate += doc.status === 'parsed' ? 1 : 0), 0);
const getRateCalculatorForDocs = (docs: SimulationDocReport[]) => (status: DocSimulationStatus) => {
const matchCount = docs.reduce((rate, doc) => (rate += doc.status === status ? 1 : 0), 0);
return successfulCount / docs.length;
};
const computeSkippedRate = (docs: SimulationDocReport[]) => {
const skippedCount = docs.reduce((rate, doc) => (rate += doc.status === 'skipped' ? 1 : 0), 0);
return skippedCount / docs.length;
return matchCount / docs.length;
};
const computeMappingProperties = (detectedFields: NamedFieldDefinitionConfig[]) => {

8
x-pack/platform/plugins/shared/streams/server/routes/internal/streams/processing/suggestions_handler.test.ts
View file

@ -15,7 +15,9 @@ jest.mock('./simulation_handler', () => ({
simulateProcessing: jest.fn((params) =>
Promise.resolve({
is_non_additive_simulation: false,
success_rate: 1,
documents_metrics: {
parsed_rate: 1,
},
simulationField: 'dummy',
// include any simulation-specific response details if necessary
})
@ -168,7 +170,9 @@ describe('handleProcessingSuggestion', () => {
(simulateProcessing as jest.Mock).mockImplementationOnce(async () => ({
is_non_additive_simulation: false,
success_rate: 0,
documents_metrics: {
parsed_rate: 0,
},
simulationField: 'dummy',
}));

15
x-pack/platform/plugins/shared/streams/server/routes/internal/streams/processing/suggestions_handler.ts
View file

@ -40,19 +40,18 @@ export const handleProcessingSuggestion = async (
const deduplicatedSimulations = uniqBy(
results.flatMap((result) => result.simulations),
(simulation) => simulation!.pattern
(simulation) => simulation.pattern
);
return {
patterns: deduplicatedSimulations.map((simulation) => simulation!.pattern),
simulations: deduplicatedSimulations as SimulationWithPattern[],
patterns: deduplicatedSimulations.map((simulation) => simulation.pattern),
simulations: deduplicatedSimulations,
};
};
type SimulationWithPattern = ReturnType<typeof simulateProcessing> & {
export interface SimulationWithPattern extends Awaited<ReturnType<typeof simulateProcessing>> {
pattern: string;
success_rate: number;
};
}
export function extractAndGroupPatterns(samples: FlattenRecord[], field: string) {
const evalPattern = (sample: string) => {
@ -195,7 +194,7 @@ async function processPattern(
streamsClient,
});
if (simulationResult.success_rate === 0) {
if (simulationResult.documents_metrics.parsed_rate === 0) {
return null;
}
@ -207,7 +206,7 @@ async function processPattern(
};
})
)
).filter(Boolean) as Array<SimulationWithPattern | null>;
).filter((simulation): simulation is SimulationWithPattern => simulation !== null);
return {
chatResponse,

57
x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processor_outcome_preview.tsx
View file

@ -51,18 +51,29 @@ export const ProcessorOutcomePreview = () => {
</>
);
};
const formatter = new Intl.NumberFormat('en-US', {
style: 'percent',
maximumFractionDigits: 0,
});
const formatRateToPercentage = (rate?: number) =>
(rate ? formatter.format(rate) : undefined) as any; // This is a workaround for the type error, since the numFilters & numActiveFilters props are defined as number | undefined
const OutcomeControls = () => {
const { changePreviewDocsFilter } = useStreamEnrichmentEvents();
const previewDocsFilter = useSimulatorSelector((state) => state.context.previewDocsFilter);
const simulationFailureRate = useSimulatorSelector((state) =>
state.context.simulation
? state.context.simulation.failure_rate + state.context.simulation.skipped_rate
: undefined
const simulationFailedRate = useSimulatorSelector((state) =>
formatRateToPercentage(state.context.simulation?.documents_metrics.failed_rate)
);
const simulationSuccessRate = useSimulatorSelector(
(state) => state.context.simulation?.success_rate
const simulationSkippedRate = useSimulatorSelector((state) =>
formatRateToPercentage(state.context.simulation?.documents_metrics.skipped_rate)
);
const simulationPartiallyParsedRate = useSimulatorSelector((state) =>
formatRateToPercentage(state.context.simulation?.documents_metrics.partially_parsed_rate)
);
const simulationParsedRate = useSimulatorSelector((state) =>
formatRateToPercentage(state.context.simulation?.documents_metrics.parsed_rate)
);
const dateRangeRef = useSimulatorSelector((state) => state.context.dateRangeRef);
@ -101,22 +112,36 @@ const OutcomeControls = () => {
{previewDocsFilterOptions.outcome_filter_all.label}
</EuiFilterButton>
<EuiFilterButton
{...getFilterButtonPropsFor(previewDocsFilterOptions.outcome_filter_matched.id)}
{...getFilterButtonPropsFor(previewDocsFilterOptions.outcome_filter_parsed.id)}
badgeColor="success"
numActiveFilters={
simulationSuccessRate ? parseFloat((simulationSuccessRate * 100).toFixed(2)) : undefined
}
numFilters={simulationParsedRate}
numActiveFilters={simulationParsedRate}
>
{previewDocsFilterOptions.outcome_filter_matched.label}
{previewDocsFilterOptions.outcome_filter_parsed.label}
</EuiFilterButton>
<EuiFilterButton
{...getFilterButtonPropsFor(previewDocsFilterOptions.outcome_filter_unmatched.id)}
{...getFilterButtonPropsFor(previewDocsFilterOptions.outcome_filter_partially_parsed.id)}
badgeColor="accent"
numActiveFilters={
simulationFailureRate ? parseFloat((simulationFailureRate * 100).toFixed(2)) : undefined
}
numFilters={simulationPartiallyParsedRate}
numActiveFilters={simulationPartiallyParsedRate}
>
{previewDocsFilterOptions.outcome_filter_unmatched.label}
{previewDocsFilterOptions.outcome_filter_partially_parsed.label}
</EuiFilterButton>
<EuiFilterButton
{...getFilterButtonPropsFor(previewDocsFilterOptions.outcome_filter_skipped.id)}
badgeColor="accent"
numFilters={simulationSkippedRate}
numActiveFilters={simulationSkippedRate}
>
{previewDocsFilterOptions.outcome_filter_skipped.label}
</EuiFilterButton>
<EuiFilterButton
{...getFilterButtonPropsFor(previewDocsFilterOptions.outcome_filter_failed.id)}
badgeColor="accent"
numFilters={simulationFailedRate}
numActiveFilters={simulationFailedRate}
>
{previewDocsFilterOptions.outcome_filter_failed.label}
</EuiFilterButton>
</EuiFilterGroup>
<StreamsAppSearchBar

12
x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/grok/grok_ai_suggestions.tsx
View file

@ -29,6 +29,7 @@ import type { FindActionResult } from '@kbn/actions-plugin/server';
import { UseGenAIConnectorsResult } from '@kbn/observability-ai-assistant-plugin/public/hooks/use_genai_connectors';
import { useAbortController, useBoolean } from '@kbn/react-hooks';
import useObservable from 'react-use/lib/useObservable';
import { APIReturnType } from '@kbn/streams-plugin/public/api';
import { isEmpty } from 'lodash';
import { css } from '@emotion/css';
import { useStreamDetail } from '../../../../../hooks/use_stream_detail';
@ -152,6 +153,9 @@ function useAIFeatures() {
};
}
export type SuggestionsResponse =
APIReturnType<'POST /internal/streams/{name}/processing/_suggestions'>;
function InnerGrokAiSuggestions({
previewDocuments,
genAiConnectors,
@ -175,9 +179,7 @@ function InnerGrokAiSuggestions({
const [isLoadingSuggestions, setSuggestionsLoading] = useState(false);
const [suggestionsError, setSuggestionsError] = useState<Error | undefined>();
const [suggestions, setSuggestions] = useState<
{ patterns: string[]; simulations: any[] } | undefined
>();
const [suggestions, setSuggestions] = useState<SuggestionsResponse | undefined>();
const [blocklist, setBlocklist] = useState<Set<string>>(new Set());
const abortController = useAbortController();
@ -210,7 +212,7 @@ function InnerGrokAiSuggestions({
.then((response) => {
finishTrackingAndReport(
response.patterns.length || 0,
response.simulations.map((item) => item.success_rate)
response.simulations.map((simulation) => simulation.documents_metrics.parsed_rate)
);
setSuggestions(response);
setSuggestionsLoading(false);
@ -249,7 +251,7 @@ function InnerGrokAiSuggestions({
const filteredSuggestions = suggestions?.patterns
.map((pattern, i) => ({
pattern,
success_rate: suggestions.simulations[i].success_rate,
success_rate: suggestions.simulations[i].documents_metrics.parsed_rate,
detected_fields_count: suggestions.simulations[i].detected_fields.length,
}))
.filter(

37
x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/processors/processor_metrics.tsx
View file

@ -31,28 +31,28 @@ const formatter = new Intl.NumberFormat('en-US', {
export const ProcessorMetricBadges = ({
detected_fields,
failure_rate,
failed_rate,
skipped_rate,
success_rate,
parsed_rate,
}: ProcessorMetricBadgesProps) => {
const detectedFieldsCount = detected_fields.length;
const failureRate = failure_rate > 0 ? formatter.format(failure_rate) : null;
const parsedRate = parsed_rate > 0 ? formatter.format(parsed_rate) : null;
const skippedRate = skipped_rate > 0 ? formatter.format(skipped_rate) : null;
const successRate = success_rate > 0 ? formatter.format(success_rate) : null;
const failedRate = failed_rate > 0 ? formatter.format(failed_rate) : null;
return (
<EuiBadgeGroup gutterSize="xs">
{failureRate && (
{parsedRate && (
<EuiBadge
color="hollow"
iconType="warning"
title={i18n.translate('xpack.streams.processorMetricBadges.euiBadge.failureRate', {
iconType="check"
title={i18n.translate('xpack.streams.processorMetricBadges.euiBadge.parsedRate', {
defaultMessage:
'{failureRate} of the sampled documents were not parsed due to an error',
values: { failureRate },
'{parsedRate} of the sampled documents were successfully parsed by this processor',
values: { parsedRate },
})}
>
{failureRate}
{parsedRate}
</EuiBadge>
)}
{skippedRate && (
@ -68,17 +68,16 @@ export const ProcessorMetricBadges = ({
{skippedRate}
</EuiBadge>
)}
{successRate && (
{failedRate && (
<EuiBadge
color="hollow"
iconType="check"
title={i18n.translate('xpack.streams.processorMetricBadges.euiBadge.successRate', {
defaultMessage:
'{successRate} of the sampled documents were successfully parsed by this processor',
values: { successRate },
iconType="warning"
title={i18n.translate('xpack.streams.processorMetricBadges.euiBadge.failedRate', {
defaultMessage: '{failedRate} of the sampled documents were not parsed due to an error',
values: { failedRate },
})}
>
{successRate}
{failedRate}
</EuiBadge>
)}
{detectedFieldsCount > 0 && (
@ -106,7 +105,7 @@ const errorTitle = i18n.translate(
);
export const ProcessorErrors = ({ metrics }: { metrics: ProcessorMetrics }) => {
const { errors, success_rate } = metrics;
const { errors, parsed_rate } = metrics;
const { euiTheme } = useEuiTheme();
const [isErrorListExpanded, toggleErrorListExpanded] = useToggle(false);
@ -118,7 +117,7 @@ export const ProcessorErrors = ({ metrics }: { metrics: ProcessorMetrics }) => {
const getCalloutProps = (type: ProcessorMetrics['errors'][number]['type']): EuiCallOutProps => {
const isWarningError =
type === 'non_additive_processor_failure' ||
(type === 'generic_processor_failure' && success_rate > 0);
(type === 'generic_processor_failure' && parsed_rate > 0);
return {
color: isWarningError ? 'warning' : 'danger',

30
x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/preview_docs_filter.ts
View file

@ -15,18 +15,32 @@ export const previewDocsFilterOptions = {
{ defaultMessage: 'All samples' }
),
},
outcome_filter_matched: {
id: 'outcome_filter_matched',
outcome_filter_parsed: {
id: 'outcome_filter_parsed',
label: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processor.outcomeControls.matched',
{ defaultMessage: 'Matched' }
'xpack.streams.streamDetailView.managementTab.enrichment.processor.outcomeControls.parsed',
{ defaultMessage: 'Parsed' }
),
},
outcome_filter_unmatched: {
id: 'outcome_filter_unmatched',
outcome_filter_partially_parsed: {
id: 'outcome_filter_partially_parsed',
label: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processor.outcomeControls.unmatched',
{ defaultMessage: 'Unmatched' }
'xpack.streams.streamDetailView.managementTab.enrichment.processor.outcomeControls.partially_parsed',
{ defaultMessage: 'Partially parsed' }
),
},
outcome_filter_skipped: {
id: 'outcome_filter_skipped',
label: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processor.outcomeControls.skipped',
{ defaultMessage: 'Skipped' }
),
},
outcome_filter_failed: {
id: 'outcome_filter_failed',
label: i18n.translate(
'xpack.streams.streamDetailView.managementTab.enrichment.processor.outcomeControls.failed',
{ defaultMessage: 'Failed' }
),
},
} as const;

12
x-pack/platform/plugins/shared/streams_app/public/components/data_management/stream_detail_enrichment/state_management/simulation_state_machine/utils.ts
View file

@ -46,7 +46,7 @@ export function getTableColumns(
) {
const uniqueProcessorsFields = uniq(getSourceFields(processors));
if (filter === 'outcome_filter_unmatched') {
if (filter === 'outcome_filter_failed' || filter === 'outcome_filter_skipped') {
return uniqueProcessorsFields;
}
@ -60,10 +60,14 @@ export function filterSimulationDocuments(
filter: PreviewDocsFilterOption
) {
switch (filter) {
case 'outcome_filter_matched':
case 'outcome_filter_parsed':
return documents.filter((doc) => doc.status === 'parsed').map((doc) => doc.value);
case 'outcome_filter_unmatched':
return documents.filter((doc) => doc.status !== 'parsed').map((doc) => doc.value);
case 'outcome_filter_partially_parsed':
return documents.filter((doc) => doc.status === 'partially_parsed').map((doc) => doc.value);
case 'outcome_filter_skipped':
return documents.filter((doc) => doc.status === 'skipped').map((doc) => doc.value);
case 'outcome_filter_failed':
return documents.filter((doc) => doc.status === 'failed').map((doc) => doc.value);
case 'outcome_filter_all':
default:
return documents.map((doc) => doc.value);

79
x-pack/test/api_integration/deployment_agnostic/apis/observability/streams/processing_simulate.ts
View file

@ -112,8 +112,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
documents: [createTestDocument()],
});
expect(response.body.success_rate).to.be(1);
expect(response.body.failure_rate).to.be(0);
expect(response.body.documents_metrics.parsed_rate).to.be(1);
expect(response.body.documents_metrics.failed_rate).to.be(0);
const { detected_fields, errors, status, value } = response.body.documents[0];
expect(status).to.be('parsed');
@ -162,8 +162,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
documents: [createTestDocument(`${TEST_MESSAGE} 127.0.0.1`)],
});
expect(response.body.success_rate).to.be(1);
expect(response.body.failure_rate).to.be(0);
expect(response.body.documents_metrics.parsed_rate).to.be(1);
expect(response.body.documents_metrics.failed_rate).to.be(0);
const { detected_fields, status, value } = response.body.documents[0];
expect(status).to.be('parsed');
@ -195,8 +195,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
documents: [createTestDocument(`${TEST_MESSAGE} 127.0.0.1`)],
});
expect(response.body.success_rate).to.be(0);
expect(response.body.failure_rate).to.be(1);
expect(response.body.documents_metrics.parsed_rate).to.be(0);
expect(response.body.documents_metrics.partially_parsed_rate).to.be(1);
expect(response.body.documents_metrics.failed_rate).to.be(0);
const { detected_fields, status, value } = response.body.documents[0];
expect(status).to.be('partially_parsed');
@ -236,8 +237,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
'parsed_timestamp',
]);
expect(dissectMetrics.errors).to.eql([]);
expect(dissectMetrics.failure_rate).to.be(0);
expect(dissectMetrics.success_rate).to.be(1);
expect(dissectMetrics.failed_rate).to.be(0);
expect(dissectMetrics.parsed_rate).to.be(1);
expect(grokMetrics.detected_fields).to.eql([]);
expect(grokMetrics.errors).to.eql([
@ -247,11 +248,12 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
message: 'Provided Grok expressions do not match field value: [test 127.0.0.1]',
},
]);
expect(grokMetrics.failure_rate).to.be(1);
expect(grokMetrics.success_rate).to.be(0);
expect(grokMetrics.failed_rate).to.be(1);
expect(grokMetrics.parsed_rate).to.be(0);
expect(grokMetrics.skipped_rate).to.be(0);
});
it('should return accurate success/failure rates', async () => {
it('should return accurate rates', async () => {
const response = await simulateProcessingForStream(apiClient, 'logs.test', {
processing: [
basicDissectProcessor,
@ -272,8 +274,9 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
],
});
expect(response.body.success_rate).to.be(0.25);
expect(response.body.failure_rate).to.be(0.75);
expect(response.body.documents_metrics.parsed_rate).to.be(0.25);
expect(response.body.documents_metrics.partially_parsed_rate).to.be(0.5);
expect(response.body.documents_metrics.failed_rate).to.be(0.25);
expect(response.body.documents).to.have.length(4);
expect(response.body.documents[0].status).to.be('parsed');
expect(response.body.documents[1].status).to.be('partially_parsed');
@ -284,10 +287,44 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
const dissectMetrics = processorsMetrics['dissect-uuid'];
const grokMetrics = processorsMetrics.draft;
expect(dissectMetrics.failure_rate).to.be(0.25);
expect(dissectMetrics.success_rate).to.be(0.75);
expect(grokMetrics.failure_rate).to.be(0.75);
expect(grokMetrics.success_rate).to.be(0.25);
expect(dissectMetrics.failed_rate).to.be(0.25);
expect(dissectMetrics.parsed_rate).to.be(0.75);
expect(grokMetrics.failed_rate).to.be(0.75);
expect(grokMetrics.parsed_rate).to.be(0.25);
});
it('should return metrics for skipped documents due to non-hit condition', async () => {
const response = await simulateProcessingForStream(apiClient, 'logs.test', {
processing: [
{
...basicDissectProcessor,
dissect: {
...basicDissectProcessor.dissect,
if: { field: 'message', operator: 'contains', value: 'test' },
},
},
],
documents: [
createTestDocument(`${TEST_TIMESTAMP} info test`),
createTestDocument('invalid format'),
createTestDocument('invalid format'),
createTestDocument('invalid format'),
],
});
expect(response.body.documents_metrics.skipped_rate).to.be(0.75);
expect(response.body.documents).to.have.length(4);
expect(response.body.documents[0].status).to.be('parsed');
expect(response.body.documents[1].status).to.be('skipped');
expect(response.body.documents[2].status).to.be('skipped');
expect(response.body.documents[3].status).to.be('skipped');
const processorsMetrics = response.body.processors_metrics;
const dissectMetrics = processorsMetrics['dissect-uuid'];
expect(dissectMetrics.failed_rate).to.be(0);
expect(dissectMetrics.parsed_rate).to.be(0.25);
expect(dissectMetrics.skipped_rate).to.be(0.75);
});
it('should allow overriding fields detected by previous simulation processors (skip non-additive check)', async () => {
@ -306,8 +343,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
documents: [createTestDocument(`${TEST_MESSAGE} 127.0.0.1 greedy data message`)],
});
expect(response.body.success_rate).to.be(1);
expect(response.body.failure_rate).to.be(0);
expect(response.body.documents_metrics.parsed_rate).to.be(1);
expect(response.body.documents_metrics.failed_rate).to.be(0);
const { detected_fields, status, value } = response.body.documents[0];
expect(status).to.be('parsed');
@ -407,8 +444,8 @@ export default function ({ getService }: DeploymentAgnosticFtrProviderContext) {
},
]);
// Non-additive changes are not counted as error
expect(grokMetrics.success_rate).to.be(1);
expect(grokMetrics.failure_rate).to.be(0);
expect(grokMetrics.parsed_rate).to.be(1);
expect(grokMetrics.failed_rate).to.be(0);
});
it('should return the is_non_additive_simulation simulation flag', async () => {