mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 09:48:58 -04:00
[EDR Workflows] Insights - Rework defend insights evaluation function (#216462)
This PR updates the evaluator function that compares LLM output to predefined expected results. The update was needed after we agreed on a more structured way of evaluating prompts and the insights they produce. We now use three sets of expected paths: * Required – paths that must always be returned * Optional – paths that are nice to have but not mandatory * Excluded – paths that should never appear in the LLM response This structure is applied per-OS, and we’ve updated the LangSmith examples accordingly. The evaluator’s role is to compare these LangChain-side requirements to the actual LLM output. It starts by validating that the structure defined in the LangSmith Web UI matches what the code expects. It then performs several checks: comparing the number of antivirus groups, matching the returned groups to the expected ones, verifying the presence of required and optional paths, and flagging any unexpected results. The final score reflects how many of these checks passed.
This commit is contained in:
Konrad Szwarc
GitHub
parent
c80325db10
commit
f6d12f8384
3 changed files with 484 additions and 91 deletions
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
export const EVALUATOR_ERRORS = {
|
||||
INVALID_OUTPUT_STRUCTURE:
|
||||
'Invalid output structure: expected {results: [{ name: string; requiredPaths: string[]; optionalPaths: string[]; excludedPaths: string[]; }]}',
|
||||
NO_RESULTS: "No results found in the run's output",
|
||||
GROUPS_COUNT: 'Number of insight groups does not match number of requirements',
|
||||
};
|
||||
|
|
@ -5,79 +5,388 @@
|
|||
* 2.0.
|
||||
*/
|
||||
|
||||
import { customIncompatibleAntivirusEvaluator } from './customIncompatibleAntivirusEvaluator';
|
||||
import { Run, Example } from 'langsmith';
|
||||
import {
|
||||
customIncompatibleAntivirusEvaluator,
|
||||
ExampleOutput,
|
||||
isValidExampleOutput,
|
||||
} from './customIncompatibleAntivirusEvaluator';
|
||||
import { EVALUATOR_ERRORS } from './constants';
|
||||
import { Example, Run } from 'langsmith';
|
||||
|
||||
describe('customIncompatibleAntivirusEvaluator', () => {
|
||||
const baseEvent = {
|
||||
id: 'event-1',
|
||||
value: 'value-1',
|
||||
endpointId: 'endpoint-1',
|
||||
};
|
||||
describe('isValidExampleOutput', () => {
|
||||
it('returns true for empty path arrays', () => {
|
||||
const validOutput = {
|
||||
results: [
|
||||
{
|
||||
name: 'Windows Defender',
|
||||
requiredPaths: [],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(validOutput)).toBe(true);
|
||||
});
|
||||
|
||||
const createGroup = (group: string, events = [baseEvent]) => ({
|
||||
group,
|
||||
events,
|
||||
});
|
||||
it('returns true for valid ExampleOutput', () => {
|
||||
const validOutput = {
|
||||
results: [
|
||||
{
|
||||
name: 'Windows Defender',
|
||||
requiredPaths: ['/path/to/required'],
|
||||
optionalPaths: ['/path/to/optional'],
|
||||
excludedPaths: ['/path/to/excluded'],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(validOutput)).toBe(true);
|
||||
});
|
||||
|
||||
const makeExample = (insights: unknown[]): Example =>
|
||||
({
|
||||
outputs: { insights },
|
||||
} as unknown as Example);
|
||||
it('returns false for invalid ExampleOutput with missing name', () => {
|
||||
const invalidOutput = {
|
||||
results: [
|
||||
{
|
||||
requiredPaths: ['/path/to/required'],
|
||||
optionalPaths: ['/path/to/optional'],
|
||||
excludedPaths: ['/path/to/excluded'],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(invalidOutput as unknown as ExampleOutput)).toBe(false);
|
||||
});
|
||||
|
||||
const makeRun = (insights: unknown[]): Run =>
|
||||
({
|
||||
outputs: { insights },
|
||||
} as unknown as Run);
|
||||
it('returns false for invalid ExampleOutput with non-string name', () => {
|
||||
const invalidOutput = {
|
||||
results: [
|
||||
{
|
||||
name: 123,
|
||||
requiredPaths: ['/path/to/required'],
|
||||
optionalPaths: ['/path/to/optional'],
|
||||
excludedPaths: ['/path/to/excluded'],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(invalidOutput as unknown as ExampleOutput)).toBe(false);
|
||||
});
|
||||
|
||||
const evaluatorFunction = customIncompatibleAntivirusEvaluator as Function;
|
||||
it('returns false for invalid ExampleOutput with non-array results', () => {
|
||||
const invalidOutput = {
|
||||
results: 'not-an-array',
|
||||
};
|
||||
expect(isValidExampleOutput(invalidOutput as unknown as ExampleOutput)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return score 1 when insights match', () => {
|
||||
const example = makeExample([createGroup('GroupA')]);
|
||||
const run = makeRun([createGroup('GroupA')]);
|
||||
it('returns false for invalid ExampleOutput with non-string array in requiredPaths', () => {
|
||||
const invalidOutput = {
|
||||
results: [
|
||||
{
|
||||
name: 'Windows Defender',
|
||||
requiredPaths: [123],
|
||||
optionalPaths: ['/path/to/optional'],
|
||||
excludedPaths: ['/path/to/excluded'],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(invalidOutput as unknown as ExampleOutput)).toBe(false);
|
||||
});
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result).toEqual({
|
||||
key: 'correct',
|
||||
score: 1,
|
||||
comment: undefined,
|
||||
it('returns false for invalid ExampleOutput with non-string array in optionalPaths', () => {
|
||||
const invalidOutput = {
|
||||
results: [
|
||||
{
|
||||
name: 'Windows Defender',
|
||||
requiredPaths: ['/path/to/required'],
|
||||
optionalPaths: [123],
|
||||
excludedPaths: ['/path/to/excluded'],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(invalidOutput as unknown as ExampleOutput)).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false for invalid ExampleOutput with non-string array in excludedPaths', () => {
|
||||
const invalidOutput = {
|
||||
results: [
|
||||
{
|
||||
name: 'Windows Defender',
|
||||
requiredPaths: ['/path/to/required'],
|
||||
optionalPaths: ['/path/to/optional'],
|
||||
excludedPaths: [123],
|
||||
},
|
||||
],
|
||||
};
|
||||
expect(isValidExampleOutput(invalidOutput as unknown as ExampleOutput)).toBe(false);
|
||||
});
|
||||
});
|
||||
describe('customIncompatibleAntivirusEvaluator', () => {
|
||||
const buildRun = (insights: unknown) => ({ outputs: { insights } } as unknown as Run);
|
||||
|
||||
it('should return score 0 when number of groups mismatch', () => {
|
||||
const example = makeExample([createGroup('GroupA')]);
|
||||
const run = makeRun([createGroup('GroupA'), createGroup('GroupB')]);
|
||||
const buildExample = (results: ExampleOutput['results']) =>
|
||||
({ outputs: { results } } as unknown as Example);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.comment).toContain('Expected 1 insights, but got 2');
|
||||
});
|
||||
const evaluatorFunction = customIncompatibleAntivirusEvaluator as Function;
|
||||
|
||||
it('should return score 0 when group names mismatch', () => {
|
||||
const example = makeExample([createGroup('GroupA')]);
|
||||
const run = makeRun([createGroup('GroupB')]);
|
||||
it('returns INVALID_OUTPUT_STRUCTURE if output is not valid', () => {
|
||||
const run = buildRun([]);
|
||||
const example = { outputs: {} }; // Missing valid structure
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.comment).toContain('Mismatch in group name at index 0');
|
||||
});
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.comment).toContain(EVALUATOR_ERRORS.INVALID_OUTPUT_STRUCTURE);
|
||||
});
|
||||
|
||||
it('should return score 0 when number of events in group mismatch', () => {
|
||||
const example = makeExample([createGroup('GroupA', [baseEvent])]);
|
||||
const run = makeRun([createGroup('GroupA', [baseEvent, baseEvent])]);
|
||||
it('returns NO_RESULTS if run insights are missing or empty', () => {
|
||||
const run = buildRun([]);
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'TestAV',
|
||||
requiredPaths: ['C:/test.exe'],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.comment).toContain('Mismatch in number of events');
|
||||
});
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.comment).toContain(EVALUATOR_ERRORS.NO_RESULTS);
|
||||
});
|
||||
|
||||
it('should return score 0 when event data mismatches', () => {
|
||||
const modifiedEvent = { ...baseEvent, value: 'wrong-value' };
|
||||
const example = makeExample([createGroup('GroupA', [baseEvent])]);
|
||||
const run = makeRun([createGroup('GroupA', [modifiedEvent])]);
|
||||
it('passes all checks when everything matches', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'TestAV',
|
||||
events: [{ value: 'C:/test.exe' }, { value: 'C:/opt.log' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.comment).toContain("Mismatch in event at group 'GroupA'");
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'TestAV',
|
||||
requiredPaths: ['C:/test.exe'],
|
||||
optionalPaths: ['C:/opt.log'],
|
||||
excludedPaths: [],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(1);
|
||||
expect(result.comment).toBe('All checks passed');
|
||||
});
|
||||
|
||||
it('fails on missing required paths', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'TestAV',
|
||||
events: [{ value: 'C:/other.exe' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'TestAV',
|
||||
requiredPaths: ['C:/required.exe'],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeCloseTo(0.5);
|
||||
expect(result.comment).toContain('missing required paths');
|
||||
expect(result.comment).toContain('contains unexpected paths');
|
||||
});
|
||||
|
||||
it('fails on presence of excluded paths', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'TestAV',
|
||||
events: [{ value: 'C:/malware.exe' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'TestAV',
|
||||
requiredPaths: [],
|
||||
optionalPaths: [],
|
||||
excludedPaths: ['C:/malware.exe'],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeCloseTo(0.75);
|
||||
expect(result.comment).toContain('contains excluded paths');
|
||||
});
|
||||
|
||||
it('fails on unexpected paths', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'TestAV',
|
||||
events: [{ value: 'C:/weird.exe' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'TestAV',
|
||||
requiredPaths: [],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeCloseTo(0.75);
|
||||
expect(result.comment).toContain('contains unexpected paths');
|
||||
});
|
||||
|
||||
it('handles multiple failed checks with score < 1', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'TestAV',
|
||||
events: [{ value: 'C:/unexpected.exe' }, { value: 'C:/excluded.exe' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'TestAV',
|
||||
requiredPaths: ['C:/required.exe'],
|
||||
optionalPaths: [],
|
||||
excludedPaths: ['C:/excluded.exe'],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeCloseTo(0.25);
|
||||
expect(result.comment).toContain('missing required paths');
|
||||
expect(result.comment).toContain('contains excluded paths');
|
||||
expect(result.comment).toContain('contains unexpected paths');
|
||||
});
|
||||
it('passes when multiple requirements are all satisfied', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'AV A',
|
||||
events: [{ value: '/a/req1' }, { value: '/a/opt1' }],
|
||||
},
|
||||
{
|
||||
group: 'AV B',
|
||||
events: [{ value: '/b/req1' }, { value: '/b/opt1' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'AV A',
|
||||
requiredPaths: ['/a/req1'],
|
||||
optionalPaths: ['/a/opt1'],
|
||||
excludedPaths: ['/a/bad'],
|
||||
},
|
||||
{
|
||||
name: 'AV B',
|
||||
requiredPaths: ['/b/req1'],
|
||||
optionalPaths: ['/b/opt1'],
|
||||
excludedPaths: ['/b/bad'],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBe(1);
|
||||
expect(result.comment).toBe('All checks passed');
|
||||
});
|
||||
it('fails when one requirement is missing a required path', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'AV A',
|
||||
events: [{ value: '/a/req1' }],
|
||||
},
|
||||
{
|
||||
group: 'AV B',
|
||||
events: [{ value: '/b/opt1' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'AV A',
|
||||
requiredPaths: ['/a/req1'],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
{
|
||||
name: 'AV B',
|
||||
requiredPaths: ['/b/req1'],
|
||||
optionalPaths: ['/b/opt1'],
|
||||
excludedPaths: [],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeCloseTo(0.86);
|
||||
expect(result.comment).toContain('requirement "AV B" is missing required paths');
|
||||
});
|
||||
it('fails when one requirement has an excluded path present', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'AV A',
|
||||
events: [{ value: '/a/req1' }],
|
||||
},
|
||||
{
|
||||
group: 'AV B',
|
||||
events: [{ value: '/b/bad' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'AV A',
|
||||
requiredPaths: ['/a/req1'],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
{
|
||||
name: 'AV B',
|
||||
requiredPaths: [],
|
||||
optionalPaths: [],
|
||||
excludedPaths: ['/b/bad'],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeLessThan(1);
|
||||
expect(result.comment).toContain('requirement "AV B" contains excluded paths');
|
||||
});
|
||||
it('fails when a requirement encounters unexpected paths', () => {
|
||||
const run = buildRun([
|
||||
{
|
||||
group: 'AV A',
|
||||
events: [{ value: '/a/unknown' }],
|
||||
},
|
||||
{
|
||||
group: 'AV B',
|
||||
events: [{ value: '/b/req1' }],
|
||||
},
|
||||
]);
|
||||
|
||||
const example = buildExample([
|
||||
{
|
||||
name: 'AV A',
|
||||
requiredPaths: [],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
{
|
||||
name: 'AV B',
|
||||
requiredPaths: ['/b/req1'],
|
||||
optionalPaths: [],
|
||||
excludedPaths: [],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = evaluatorFunction(run, example);
|
||||
expect(result.score).toBeLessThan(1);
|
||||
expect(result.comment).toContain('requirement "AV A" contains unexpected paths');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,57 +7,128 @@
|
|||
|
||||
import { Example, Run } from 'langsmith';
|
||||
import { EvaluatorT } from 'langsmith/evaluation';
|
||||
import { DefendInsights } from '@kbn/elastic-assistant-common';
|
||||
import { EVALUATOR_ERRORS } from './constants';
|
||||
|
||||
export interface ExampleOutput {
|
||||
// Example is a LangSmith name for datasets
|
||||
results: Array<{
|
||||
name: string; // e.g. "Windows Defender"
|
||||
requiredPaths: string[];
|
||||
optionalPaths: string[];
|
||||
excludedPaths: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export const isValidExampleOutput = (output: ExampleOutput): output is ExampleOutput => {
|
||||
// Check if output is an object and has the expected structure. It's defined in LangSmith, hence needs validation.
|
||||
const isStringArray = (arr: string[] | unknown): arr is string[] =>
|
||||
Array.isArray(arr) && arr.every((item) => typeof item === 'string');
|
||||
|
||||
return (
|
||||
output &&
|
||||
Array.isArray(output.results) &&
|
||||
output.results.every(
|
||||
(res) =>
|
||||
typeof res.name === 'string' &&
|
||||
isStringArray(res.requiredPaths) &&
|
||||
isStringArray(res.optionalPaths) &&
|
||||
isStringArray(res.excludedPaths)
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
const failWithComment = (comment: string) => ({
|
||||
key: 'correct',
|
||||
score: 0,
|
||||
comment,
|
||||
});
|
||||
|
||||
export const customIncompatibleAntivirusEvaluator: EvaluatorT = (
|
||||
run: Run,
|
||||
example: Example | undefined
|
||||
) => {
|
||||
let error: string | undefined;
|
||||
const referenceInsights = example?.outputs?.insights ?? [];
|
||||
const actualInsights = run.outputs?.insights ?? [];
|
||||
const rawOutput = example?.outputs as ExampleOutput;
|
||||
if (!isValidExampleOutput(rawOutput)) {
|
||||
return failWithComment(EVALUATOR_ERRORS.INVALID_OUTPUT_STRUCTURE);
|
||||
}
|
||||
|
||||
if (referenceInsights.length !== actualInsights.length) {
|
||||
// Mismatch in number of insights
|
||||
error = `Expected ${referenceInsights.length} insights, but got ${actualInsights.length}`;
|
||||
} else {
|
||||
for (let i = 0; i < referenceInsights.length; i++) {
|
||||
const refGroup = referenceInsights[i];
|
||||
const actGroup = actualInsights[i];
|
||||
const { results: requirements } = rawOutput;
|
||||
const insights: DefendInsights = run.outputs?.insights ?? [];
|
||||
|
||||
if (refGroup.group !== actGroup.group) {
|
||||
// Mismatch in group name
|
||||
error = `Mismatch in group name at index ${i}: expected '${refGroup.group}', got '${actGroup.group}'`;
|
||||
break;
|
||||
if (!Array.isArray(insights) || insights.length === 0) {
|
||||
return failWithComment(EVALUATOR_ERRORS.NO_RESULTS);
|
||||
}
|
||||
|
||||
const failedChecks: Array<{ label: string; details?: string[] }> = [];
|
||||
let totalChecks = 0;
|
||||
|
||||
// Check: group count matches requirement count
|
||||
totalChecks += 1;
|
||||
if (insights.length !== requirements.length) {
|
||||
failedChecks.push({
|
||||
label: 'number of insight groups does not match number of requirements',
|
||||
details: [`insights: ${insights.length}`, `requirements: ${requirements.length}`],
|
||||
});
|
||||
}
|
||||
|
||||
for (const req of requirements) {
|
||||
const label = `requirement "${req.name}"`;
|
||||
|
||||
const matchedInsight = insights.find((insight) =>
|
||||
insight.group.toLowerCase().includes(req.name.toLowerCase())
|
||||
);
|
||||
|
||||
totalChecks += 3;
|
||||
|
||||
if (!matchedInsight) {
|
||||
failedChecks.push({
|
||||
label: `${label} did not match any insight group`,
|
||||
});
|
||||
} else {
|
||||
const eventPaths = (matchedInsight.events || []).map((e) => e.value);
|
||||
|
||||
const requiredSet = new Set(req.requiredPaths);
|
||||
const excludedSet = new Set(req.excludedPaths);
|
||||
const allowedSet = new Set([...req.requiredPaths, ...req.optionalPaths]);
|
||||
|
||||
const missingRequired = [...requiredSet].filter((p) => !eventPaths.includes(p));
|
||||
if (missingRequired.length) {
|
||||
failedChecks.push({
|
||||
label: `${label} is missing required paths`,
|
||||
details: missingRequired,
|
||||
});
|
||||
}
|
||||
|
||||
if (refGroup.events.length !== actGroup.events.length) {
|
||||
// Mismatch in number of events
|
||||
error = `Mismatch in number of events for group '${refGroup.group}': expected ${refGroup.events.length}, got ${actGroup.events.length}`;
|
||||
break;
|
||||
const foundExcluded = eventPaths.filter((p) => excludedSet.has(p));
|
||||
if (foundExcluded.length) {
|
||||
failedChecks.push({
|
||||
label: `${label} contains excluded paths`,
|
||||
details: foundExcluded,
|
||||
});
|
||||
}
|
||||
|
||||
for (let j = 0; j < refGroup.events.length; j++) {
|
||||
const refEvent = refGroup.events[j];
|
||||
const actEvent = actGroup.events[j];
|
||||
|
||||
if (
|
||||
refEvent.id !== actEvent.id ||
|
||||
refEvent.value !== actEvent.value ||
|
||||
refEvent.endpointId !== actEvent.endpointId
|
||||
) {
|
||||
// Mismatch in event
|
||||
error = `Mismatch in event at group '${refGroup.group}', index ${j}`;
|
||||
break;
|
||||
}
|
||||
const unexpected = eventPaths.filter((p) => !allowedSet.has(p) && !excludedSet.has(p));
|
||||
if (unexpected.length) {
|
||||
failedChecks.push({
|
||||
label: `${label} contains unexpected paths`,
|
||||
details: unexpected,
|
||||
});
|
||||
}
|
||||
|
||||
if (error) break;
|
||||
}
|
||||
}
|
||||
|
||||
const score = totalChecks === 0 ? 0 : Number((1 - failedChecks.length / totalChecks).toFixed(2));
|
||||
|
||||
const comment = failedChecks.length
|
||||
? `Failed checks:\n${failedChecks
|
||||
.map((c) => (c.details?.length ? `${c.label}:\n - ${c.details.join('\n - ')}` : c.label))
|
||||
.join('\n\n')}`
|
||||
: 'All checks passed';
|
||||
|
||||
return {
|
||||
key: 'correct',
|
||||
score: error ? 0 : 1,
|
||||
comment: error,
|
||||
score,
|
||||
comment,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue