[Security Solution][RAC] Threshold Rule Fixes (#117571) (#118045)

* Remove unused code

* Fix threshold field refs

* Fix import

* UI fixes for Rules

* Threshold Cypress test fixes

* Type fixes

* Threshold signal test fixes

* Handle legacy schema optionally

* Fix threshold integration test

* More test fixes

Co-authored-by: Madison Caldwell <madison.rey.caldwell@gmail.com>

x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts

@@ -99,7 +99,7 @@ describe('Detection rules, threshold', () => {
     waitForAlertsIndexToBeCreated();
   });
 
-  it.skip('Creates and activates a new threshold rule', () => {
+  it('Creates and activates a new threshold rule', () => {
     goToManageAlertsDetectionRules();
     waitForRulesTableToBeLoaded();
     goToCreateNewRule();
@@ -171,9 +171,7 @@ describe('Detection rules, threshold', () => {
     waitForAlertsToPopulate();
 
     cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text().split(' ')[0]).to.be.lt(100));
-    cy.get(ALERT_GRID_CELL).eq(3).contains(rule.name);
-    cy.get(ALERT_GRID_CELL).eq(4).contains(rule.severity.toLowerCase());
-    cy.get(ALERT_GRID_CELL).eq(5).contains(rule.riskScore);
+    cy.get(ALERT_GRID_CELL).contains(rule.name);
   });
 
   it('Preview results of keyword using "host.name"', () => {

x-pack/plugins/security_solution/public/cases/components/case_view/index.tsx

@@ -75,7 +75,6 @@ const InvestigateInTimelineActionComponent = (alertIds: string[]) => {
       alertIds={alertIds}
       key="investigate-in-timeline"
       ecsRowData={null}
-      nonEcsRowData={[]}
     />
   );
 };

x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx

@@ -76,7 +76,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: mockEcsDataWithAlert,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -92,7 +91,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: mockEcsDataWithAlert,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -249,7 +247,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: mockEcsDataWithAlert,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -267,7 +264,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: mockEcsDataWithAlert,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -301,7 +297,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMock,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -327,7 +322,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMock,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -357,7 +351,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMock,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });
@@ -398,7 +391,6 @@ describe('alert actions', () => {
         await sendAlertToTimelineAction({
           createTimeline,
           ecsData: ecsDataMock,
-          nonEcsData: [],
           updateTimelineIsLoading,
           searchStrategyClient,
         });

x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx

@@ -7,7 +7,7 @@
 
 /* eslint-disable complexity */
 
-import { get, getOr, isEmpty } from 'lodash/fp';
+import { getOr, isEmpty } from 'lodash/fp';
 import moment from 'moment';
 
 import dateMath from '@elastic/datemath';
@@ -37,7 +37,6 @@ import {
 } from './types';
 import { Ecs } from '../../../../common/ecs';
 import {
-  TimelineNonEcsData,
   TimelineEventsDetailsItem,
   TimelineEventsDetailsRequestOptions,
   TimelineEventsDetailsStrategyResponse,
@@ -75,26 +74,6 @@ export const getUpdateAlertsQuery = (eventIds: Readonly<string[]>) => {
   };
 };
 
-export const getFilterAndRuleBounds = (
-  data: TimelineNonEcsData[][]
-): [string[], number, number] => {
-  const stringFilter =
-    data?.[0].filter(
-      (d) => d.field === 'signal.rule.filters' || d.field === 'kibana.alert.rule.filters'
-    )?.[0]?.value ?? [];
-
-  const eventTimes = data
-    .flatMap(
-      (alert) =>
-        alert.filter(
-          (d) => d.field === 'signal.original_time' || d.field === 'kibana.alert.original_time'
-        )?.[0]?.value ?? []
-    )
-    .map((d) => moment(d));
-
-  return [stringFilter, moment.min(eventTimes).valueOf(), moment.max(eventTimes).valueOf()];
-};
-
 export const updateAlertStatusAction = async ({
   query,
   alertIds,
@@ -174,11 +153,7 @@ const getFiltersFromRule = (filters: string[]): Filter[] =>
     }
   }, [] as Filter[]);
 
-export const getThresholdAggregationData = (
-  ecsData: Ecs | Ecs[],
-  nonEcsData: TimelineNonEcsData[]
-): ThresholdAggregationData => {
-  // TODO: AAD fields
+export const getThresholdAggregationData = (ecsData: Ecs | Ecs[]): ThresholdAggregationData => {
   const thresholdEcsData: Ecs[] = Array.isArray(ecsData) ? ecsData : [ecsData];
   return thresholdEcsData.reduce<ThresholdAggregationData>(
     (outerAcc, thresholdData) => {
@@ -195,11 +170,9 @@ export const getThresholdAggregationData = (
       };
 
       try {
-        try {
-          thresholdResult = JSON.parse((thresholdData.signal?.threshold_result as string[])[0]);
-        } catch (err) {
-          thresholdResult = JSON.parse((get(ALERT_THRESHOLD_RESULT, thresholdData) as string[])[0]);
-        }
+        thresholdResult = JSON.parse(
+          (getField(thresholdData, ALERT_THRESHOLD_RESULT) as string[])[0]
+        );
         aggField = JSON.parse(threshold[0]).field;
       } catch (err) {
         // Legacy support
@@ -401,7 +374,6 @@ export const buildEqlDataProviderOrFilter = (
 export const sendAlertToTimelineAction = async ({
   createTimeline,
   ecsData: ecs,
-  nonEcsData,
   updateTimelineIsLoading,
   searchStrategyClient,
 }: SendAlertToTimelineActionProps) => {
@@ -498,10 +470,7 @@ export const sendAlertToTimelineAction = async ({
   }
 
   if (isThresholdRule(ecsData)) {
-    const { thresholdFrom, thresholdTo, dataProviders } = getThresholdAggregationData(
-      ecsData,
-      nonEcsData
-    );
+    const { thresholdFrom, thresholdTo, dataProviders } = getThresholdAggregationData(ecsData);
 
     return createTimeline({
       from: thresholdFrom,

x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/investigate_in_timeline_action.tsx

@@ -8,7 +8,6 @@
 import React from 'react';
 
 import { Ecs } from '../../../../../common/ecs';
-import { TimelineNonEcsData } from '../../../../../common/search_strategy/timeline';
 import { ActionIconItem } from '../../../../timelines/components/timeline/body/actions/action_icon_item';
 
 import {
@@ -19,7 +18,6 @@ import { useInvestigateInTimeline } from './use_investigate_in_timeline';
 
 interface InvestigateInTimelineActionProps {
   ecsRowData?: Ecs | Ecs[] | null;
-  nonEcsRowData?: TimelineNonEcsData[];
   ariaLabel?: string;
   alertIds?: string[];
   buttonType?: 'text' | 'icon';
@@ -30,13 +28,11 @@ const InvestigateInTimelineActionComponent: React.FC<InvestigateInTimelineAction
   ariaLabel = ACTION_INVESTIGATE_IN_TIMELINE_ARIA_LABEL,
   alertIds,
   ecsRowData,
-  nonEcsRowData,
   buttonType,
   onInvestigateInTimelineAlertClick,
 }) => {
   const { investigateInTimelineAlertClick } = useInvestigateInTimeline({
     ecsRowData,
-    nonEcsRowData,
     alertIds,
     onInvestigateInTimelineAlertClick,
   });

x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/use_investigate_in_timeline.tsx

@@ -30,7 +30,6 @@ interface UseInvestigateInTimelineActionProps {
 
 export const useInvestigateInTimeline = ({
   ecsRowData,
-  nonEcsRowData,
   alertIds,
   onInvestigateInTimelineAlertClick,
 }: UseInvestigateInTimelineActionProps) => {
@@ -90,7 +89,6 @@ export const useInvestigateInTimeline = ({
       await sendAlertToTimelineAction({
         createTimeline,
         ecsData: alertsEcsData,
-        nonEcsData: nonEcsRowData ?? [],
         searchStrategyClient,
         updateTimelineIsLoading,
       });
@@ -100,7 +98,6 @@ export const useInvestigateInTimeline = ({
       await sendAlertToTimelineAction({
         createTimeline,
         ecsData: ecsRowData,
-        nonEcsData: nonEcsRowData ?? [],
         searchStrategyClient,
         updateTimelineIsLoading,
       });
@@ -109,7 +106,6 @@ export const useInvestigateInTimeline = ({
     alertsEcsData,
     createTimeline,
     ecsRowData,
-    nonEcsRowData,
     onInvestigateInTimelineAlertClick,
     searchStrategyClient,
     updateTimelineIsLoading,

x-pack/plugins/security_solution/public/detections/components/alerts_table/types.ts

@@ -8,7 +8,6 @@
 import { ISearchStart } from '../../../../../../../src/plugins/data/public';
 import { Status } from '../../../../common/detection_engine/schemas/common/schemas';
 import { Ecs } from '../../../../common/ecs';
-import { TimelineNonEcsData } from '../../../../common/search_strategy/timeline';
 import { NoteResult } from '../../../../common/types/timeline/note';
 import { DataProvider } from '../../../timelines/components/timeline/data_providers/data_provider';
 import { TimelineModel } from '../../../timelines/store/timeline/model';
@@ -54,7 +53,6 @@ export interface UpdateAlertStatusActionProps {
 export interface SendAlertToTimelineActionProps {
   createTimeline: CreateTimeline;
   ecsData: Ecs | Ecs[];
-  nonEcsData: TimelineNonEcsData[];
   updateTimelineIsLoading: UpdateTimelineLoading;
   searchStrategyClient: ISearchStart;
 }

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts

@@ -145,6 +145,7 @@ export const RuleSchema = t.intersection([
     timestamp_override,
     note: t.string,
     exceptions_list: listArray,
+    uuid: t.string,
     version: t.number,
   }),
 ]);

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_with_fallback.tsx

@@ -28,8 +28,13 @@ interface AlertHit {
   _index: string;
   _source: {
     '@timestamp': string;
-    signal: {
-      rule: Rule;
+    signal?: {
+      rule?: Rule;
+    };
+    kibana?: {
+      alert?: {
+        rule?: Rule;
+      };
     };
   };
 }
@@ -77,7 +82,10 @@ export const useRuleWithFallback = (ruleId: string): UseRuleWithFallback => {
   }, [addError, error]);
 
   const rule = useMemo<Rule | undefined>(() => {
-    const result = isExistingRule ? ruleData : alertsData?.hits.hits[0]?._source.signal.rule;
+    const hit = alertsData?.hits.hits[0];
+    const result = isExistingRule
+      ? ruleData
+      : hit?._source.signal?.rule ?? hit?._source.kibana?.alert?.rule;
     if (result) {
       return transformInput(result);
     }

x-pack/plugins/security_solution/public/timelines/components/timeline/body/actions/index.tsx

@@ -169,7 +169,6 @@ const ActionsComponent: React.FC<ActionProps> = ({
             ariaLabel={i18n.SEND_ALERT_TO_TIMELINE_FOR_ROW({ ariaRowindex, columnValues })}
             key="investigate-in-timeline"
             ecsRowData={ecsData}
-            nonEcsRowData={data}
           />
         )}
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/build_alert.ts

@@ -35,6 +35,7 @@ import {
   ALERT_ANCESTORS,
   ALERT_DEPTH,
   ALERT_ORIGINAL_TIME,
+  ALERT_THRESHOLD_RESULT,
   ALERT_ORIGINAL_EVENT,
 } from '../../../../../../common/field_maps/field_names';
 
@@ -59,10 +60,10 @@ export const buildParent = (doc: SimpleHit): Ancestor => {
     id: doc._id,
     type: isSignal ? 'signal' : 'event',
     index: doc._index,
-    depth: isSignal ? getField(doc, 'signal.depth') ?? 1 : 0,
+    depth: isSignal ? getField(doc, ALERT_DEPTH) ?? 1 : 0,
   };
   if (isSignal) {
-    parent.rule = getField(doc, 'signal.rule.id');
+    parent.rule = getField(doc, ALERT_RULE_UUID);
   }
   return parent;
 };
@@ -73,9 +74,8 @@ export const buildParent = (doc: SimpleHit): Ancestor => {
  * @param doc The parent event for which to extend the ancestry.
  */
 export const buildAncestors = (doc: SimpleHit): Ancestor[] => {
-  // TODO: handle alerts-on-legacy-alerts
   const newAncestor = buildParent(doc);
-  const existingAncestors: Ancestor[] = getField(doc, 'signal.ancestors') ?? [];
+  const existingAncestors: Ancestor[] = getField(doc, ALERT_ANCESTORS) ?? [];
   return [...existingAncestors, newAncestor];
 };
 
@@ -130,7 +130,7 @@ export const additionalAlertFields = (doc: BaseSignalHit) => {
   });
   const additionalFields: Record<string, unknown> = {
     [ALERT_ORIGINAL_TIME]: originalTime != null ? originalTime.toISOString() : undefined,
-    ...(thresholdResult != null ? { threshold_result: thresholdResult } : {}),
+    ...(thresholdResult != null ? { [ALERT_THRESHOLD_RESULT]: thresholdResult } : {}),
   };
 
   for (const [key, val] of Object.entries(doc._source ?? {})) {

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/filter_source.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { ALERT_THRESHOLD_RESULT } from '../../../../../../common/field_maps/field_names';
 import { SignalSourceHit } from '../../../signals/types';
 import { RACAlert } from '../../types';
 
@@ -12,10 +13,13 @@ export const filterSource = (doc: SignalSourceHit): Partial<RACAlert> => {
   const docSource = doc._source ?? {};
   const {
     event,
-    threshold_result: thresholdResult,
+    threshold_result: siemSignalsThresholdResult,
+    [ALERT_THRESHOLD_RESULT]: alertThresholdResult,
     ...filteredSource
   } = docSource || {
+    event: null,
     threshold_result: null,
+    [ALERT_THRESHOLD_RESULT]: null,
   };
 
   return filteredSource;

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts

@@ -54,6 +54,7 @@ import {
   ALERT_ORIGINAL_EVENT,
   ALERT_ORIGINAL_EVENT_CATEGORY,
   ALERT_GROUP_ID,
+  ALERT_THRESHOLD_RESULT,
 } from '../../../../plugins/security_solution/common/field_maps/field_names';
 
 /**
@@ -728,7 +729,7 @@ export default ({ getService }: FtrProviderContext) => {
             [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID],
             [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME],
             [ALERT_DEPTH]: 1,
-            threshold_result: {
+            [ALERT_THRESHOLD_RESULT]: {
               terms: [
                 {
                   field: 'host.id',
@@ -849,7 +850,7 @@ export default ({ getService }: FtrProviderContext) => {
             [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID],
             [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME],
             [ALERT_DEPTH]: 1,
-            threshold_result: {
+            [ALERT_THRESHOLD_RESULT]: {
               terms: [
                 {
                   field: 'host.id',
@@ -916,7 +917,7 @@ export default ({ getService }: FtrProviderContext) => {
             [ALERT_RULE_UUID]: fullSignal[ALERT_RULE_UUID],
             [ALERT_ORIGINAL_TIME]: fullSignal[ALERT_ORIGINAL_TIME],
             [ALERT_DEPTH]: 1,
-            threshold_result: {
+            [ALERT_THRESHOLD_RESULT]: {
               terms: [
                 {
                   field: 'event.module',

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/const_keyword.ts

@@ -10,6 +10,7 @@ import {
   EqlCreateSchema,
   ThresholdCreateSchema,
 } from '../../../../../plugins/security_solution/common/detection_engine/schemas/request';
+import { ALERT_THRESHOLD_RESULT } from '../../../../../plugins/security_solution/common/field_maps/field_names';
 
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
 import {
@@ -131,7 +132,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, log, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, log, id);
         const hits = signalsOpen.hits.hits
-          .map((hit) => hit._source?.threshold_result ?? null)
+          .map((hit) => hit._source?.[ALERT_THRESHOLD_RESULT] ?? null)
           .sort();
         expect(hits).to.eql([
           {

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword.ts

@@ -25,6 +25,7 @@ import {
   QueryCreateSchema,
   ThresholdCreateSchema,
 } from '../../../../../plugins/security_solution/common/detection_engine/schemas/request';
+import { ALERT_THRESHOLD_RESULT } from '../../../../../plugins/security_solution/common/field_maps/field_names';
 
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
@@ -105,7 +106,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, log, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, log, id);
         const hits = signalsOpen.hits.hits
-          .map((hit) => hit._source?.threshold_result ?? null)
+          .map((hit) => hit._source?.[ALERT_THRESHOLD_RESULT] ?? null)
           .sort();
         expect(hits).to.eql([
           {

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/keyword_family/keyword_mixed_with_const.ts

@@ -10,6 +10,7 @@ import {
   EqlCreateSchema,
   ThresholdCreateSchema,
 } from '../../../../../plugins/security_solution/common/detection_engine/schemas/request';
+import { ALERT_THRESHOLD_RESULT } from '../../../../../plugins/security_solution/common/field_maps/field_names';
 
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
 import {
@@ -144,7 +145,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, log, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, log, id);
         const hits = signalsOpen.hits.hits
-          .map((hit) => hit._source?.threshold_result ?? null)
+          .map((hit) => hit._source?.[ALERT_THRESHOLD_RESULT] ?? null)
           .sort();
         expect(hits).to.eql([
           {