[SIEM][Detections Engine] - minor updates to monitoring table with unit tests (#64020)

### Summary

Minor updates to the All Rules monitoring table. Includes front end and backend changes:

- Displays dashes in the monitoring table when no values are present
- Displays `lastLookBackDate` only if the rule indexes events into the signals index, otherwise `lastLookBackDate` is set to null

x-pack/legacy/plugins/siem/public/containers/detection_engine/rules/use_rule_status.test.tsx

@@ -72,104 +72,108 @@ describe('useRuleStatus', () => {
     cleanup();
   });
 
-  test('init', async () => {
+  describe('useRuleStatus', () => {
-    await act(async () => {
+    test('init', async () => {
-      const { result, waitForNextUpdate } = renderHook<string, ReturnRuleStatus>(() =>
+      await act(async () => {
-        useRuleStatus('myOwnRuleID')
+        const { result, waitForNextUpdate } = renderHook<string, ReturnRuleStatus>(() =>
-      );
+          useRuleStatus('myOwnRuleID')
-      await waitForNextUpdate();
+        );
-      expect(result.current).toEqual([true, null, null]);
+        await waitForNextUpdate();
+        expect(result.current).toEqual([true, null, null]);
+      });
     });
-  });
 
-  test('fetch rule status', async () => {
+    test('fetch rule status', async () => {
-    await act(async () => {
+      await act(async () => {
-      const { result, waitForNextUpdate } = renderHook<string, ReturnRuleStatus>(() =>
+        const { result, waitForNextUpdate } = renderHook<string, ReturnRuleStatus>(() =>
-        useRuleStatus('myOwnRuleID')
+          useRuleStatus('myOwnRuleID')
-      );
+        );
-      await waitForNextUpdate();
+        await waitForNextUpdate();
-      await waitForNextUpdate();
+        await waitForNextUpdate();
-      expect(result.current).toEqual([
+        expect(result.current).toEqual([
-        false,
+          false,
-        {
-          current_status: {
-            alert_id: 'alertId',
-            last_failure_at: null,
-            last_failure_message: null,
-            last_success_at: 'mm/dd/yyyyTHH:MM:sssz',
-            last_success_message: 'it is a success',
-            status: 'succeeded',
-            status_date: 'mm/dd/yyyyTHH:MM:sssz',
-            gap: null,
-            bulk_create_time_durations: ['2235.01'],
-            search_after_time_durations: ['616.97'],
-            last_look_back_date: '2020-03-19T00:32:07.996Z',
-          },
-          failures: [],
-        },
-        result.current[2],
-      ]);
-    });
-  });
-
-  test('re-fetch rule status', async () => {
-    const spyOngetRuleStatusById = jest.spyOn(api, 'getRuleStatusById');
-    await act(async () => {
-      const { result, waitForNextUpdate } = renderHook<string, ReturnRuleStatus>(() =>
-        useRuleStatus('myOwnRuleID')
-      );
-      await waitForNextUpdate();
-      await waitForNextUpdate();
-      if (result.current[2]) {
-        result.current[2]('myOwnRuleID');
-      }
-      await waitForNextUpdate();
-      expect(spyOngetRuleStatusById).toHaveBeenCalledTimes(2);
-    });
-  });
-
-  test('init rules statuses', async () => {
-    const payload = [testRule];
-    await act(async () => {
-      const { result, waitForNextUpdate } = renderHook<string, ReturnRulesStatuses>(() =>
-        useRulesStatuses(payload)
-      );
-      await waitForNextUpdate();
-      expect(result.current).toEqual({ loading: false, rulesStatuses: [] });
-    });
-  });
-
-  test('fetch rules statuses', async () => {
-    const payload = [testRule];
-    await act(async () => {
-      const { result, waitForNextUpdate } = renderHook<string, ReturnRulesStatuses>(() =>
-        useRulesStatuses(payload)
-      );
-      await waitForNextUpdate();
-      await waitForNextUpdate();
-      expect(result.current).toEqual({
-        loading: false,
-        rulesStatuses: [
           {
             current_status: {
               alert_id: 'alertId',
-              bulk_create_time_durations: ['2235.01'],
-              gap: null,
               last_failure_at: null,
               last_failure_message: null,
-              last_look_back_date: '2020-03-19T00:32:07.996Z',
               last_success_at: 'mm/dd/yyyyTHH:MM:sssz',
               last_success_message: 'it is a success',
-              search_after_time_durations: ['616.97'],
               status: 'succeeded',
               status_date: 'mm/dd/yyyyTHH:MM:sssz',
+              gap: null,
+              bulk_create_time_durations: ['2235.01'],
+              search_after_time_durations: ['616.97'],
+              last_look_back_date: '2020-03-19T00:32:07.996Z',
             },
             failures: [],
-            id: '12345678987654321',
-            activate: true,
-            name: 'Test rule',
           },
-        ],
+          result.current[2],
+        ]);
+      });
+    });
+
+    test('re-fetch rule status', async () => {
+      const spyOngetRuleStatusById = jest.spyOn(api, 'getRuleStatusById');
+      await act(async () => {
+        const { result, waitForNextUpdate } = renderHook<string, ReturnRuleStatus>(() =>
+          useRuleStatus('myOwnRuleID')
+        );
+        await waitForNextUpdate();
+        await waitForNextUpdate();
+        if (result.current[2]) {
+          result.current[2]('myOwnRuleID');
+        }
+        await waitForNextUpdate();
+        expect(spyOngetRuleStatusById).toHaveBeenCalledTimes(2);
+      });
+    });
+  });
+
+  describe('useRulesStatuses', () => {
+    test('init rules statuses', async () => {
+      const payload = [testRule];
+      await act(async () => {
+        const { result, waitForNextUpdate } = renderHook<string, ReturnRulesStatuses>(() =>
+          useRulesStatuses(payload)
+        );
+        await waitForNextUpdate();
+        expect(result.current).toEqual({ loading: false, rulesStatuses: [] });
+      });
+    });
+
+    test('fetch rules statuses', async () => {
+      const payload = [testRule];
+      await act(async () => {
+        const { result, waitForNextUpdate } = renderHook<string, ReturnRulesStatuses>(() =>
+          useRulesStatuses(payload)
+        );
+        await waitForNextUpdate();
+        await waitForNextUpdate();
+        expect(result.current).toEqual({
+          loading: false,
+          rulesStatuses: [
+            {
+              current_status: {
+                alert_id: 'alertId',
+                bulk_create_time_durations: ['2235.01'],
+                gap: null,
+                last_failure_at: null,
+                last_failure_message: null,
+                last_look_back_date: '2020-03-19T00:32:07.996Z',
+                last_success_at: 'mm/dd/yyyyTHH:MM:sssz',
+                last_success_message: 'it is a success',
+                search_after_time_durations: ['616.97'],
+                status: 'succeeded',
+                status_date: 'mm/dd/yyyyTHH:MM:sssz',
+              },
+              failures: [],
+              id: '12345678987654321',
+              activate: true,
+              name: 'Test rule',
+            },
+          ],
+        });
       });
     });
   });

x-pack/legacy/plugins/siem/public/pages/detection_engine/rules/all/columns.tsx

@@ -243,7 +243,7 @@ export const getMonitoringColumns = (): RulesStatusesColumns[] => {
         <EuiText data-test-subj="bulk_create_time_durations" size="s">
           {value != null && value.length > 0
             ? Math.max(...value?.map(item => Number.parseFloat(item)))
-            : null}
+            : getEmptyTagValue()}
         </EuiText>
       ),
       truncateText: true,
@@ -256,7 +256,7 @@ export const getMonitoringColumns = (): RulesStatusesColumns[] => {
         <EuiText data-test-subj="search_after_time_durations" size="s">
           {value != null && value.length > 0
             ? Math.max(...value?.map(item => Number.parseFloat(item)))
-            : null}
+            : getEmptyTagValue()}
         </EuiText>
       ),
       truncateText: true,
@@ -267,7 +267,7 @@ export const getMonitoringColumns = (): RulesStatusesColumns[] => {
       name: i18n.COLUMN_GAP,
       render: (value: RuleStatus['current_status']['gap']) => (
         <EuiText data-test-subj="gap" size="s">
-          {value}
+          {value ?? getEmptyTagValue()}
         </EuiText>
       ),
       truncateText: true,

x-pack/plugins/siem/server/lib/detection_engine/signals/__mocks__/es_results.ts

@@ -86,7 +86,7 @@ export const sampleDocNoSortId = (someUuid: string = sampleIdGuid): SignalSource
   _id: someUuid,
   _source: {
     someKey: 'someValue',
-    '@timestamp': 'someTimeStamp',
+    '@timestamp': '2020-04-20T21:27:45+0000',
   },
 });
 
@@ -97,7 +97,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
   _id: someUuid,
   _source: {
     someKey: 'someValue',
-    '@timestamp': 'someTimeStamp',
+    '@timestamp': '2020-04-20T21:27:45+0000',
   },
 });
 
@@ -109,7 +109,7 @@ export const sampleDocWithSortId = (someUuid: string = sampleIdGuid): SignalSour
   _id: someUuid,
   _source: {
     someKey: 'someValue',
-    '@timestamp': 'someTimeStamp',
+    '@timestamp': '2020-04-20T21:27:45+0000',
   },
   sort: ['1234567891111'],
 });

x-pack/plugins/siem/server/lib/detection_engine/signals/build_bulk_body.test.ts

@@ -59,7 +59,7 @@ describe('buildBulkBody', () => {
             depth: 1,
           },
         ],
-        original_time: 'someTimeStamp',
+        original_time: '2020-04-20T21:27:45+0000',
         status: 'open',
         rule: {
           actions: [],
@@ -185,7 +185,7 @@ describe('buildBulkBody', () => {
             depth: 1,
           },
         ],
-        original_time: 'someTimeStamp',
+        original_time: '2020-04-20T21:27:45+0000',
         status: 'open',
         rule: {
           actions: [],
@@ -309,7 +309,7 @@ describe('buildBulkBody', () => {
             depth: 1,
           },
         ],
-        original_time: 'someTimeStamp',
+        original_time: '2020-04-20T21:27:45+0000',
         status: 'open',
         rule: {
           actions: [],
@@ -426,7 +426,7 @@ describe('buildBulkBody', () => {
             depth: 1,
           },
         ],
-        original_time: 'someTimeStamp',
+        original_time: '2020-04-20T21:27:45+0000',
         status: 'open',
         rule: {
           actions: [],

x-pack/plugins/siem/server/lib/detection_engine/signals/build_signal.test.ts

@@ -41,7 +41,7 @@ describe('buildSignal', () => {
           depth: 1,
         },
       ],
-      original_time: 'someTimeStamp',
+      original_time: '2020-04-20T21:27:45+0000',
       status: 'open',
       rule: {
         created_by: 'elastic',
@@ -101,7 +101,7 @@ describe('buildSignal', () => {
           depth: 1,
         },
       ],
-      original_time: 'someTimeStamp',
+      original_time: '2020-04-20T21:27:45+0000',
       original_event: {
         action: 'socket_opened',
         dataset: 'socket',
@@ -173,7 +173,7 @@ describe('buildSignal', () => {
           depth: 1,
         },
       ],
-      original_time: 'someTimeStamp',
+      original_time: '2020-04-20T21:27:45+0000',
       original_event: {
         action: 'socket_opened',
         dataset: 'socket',

x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -30,7 +30,7 @@ describe('searchAfterAndBulkCreate', () => {
 
   test('if successful with empty search results', async () => {
     const sampleParams = sampleRuleAlertParams();
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: sampleEmptyDocSearchResults(),
       ruleParams: sampleParams,
       services: mockService,
@@ -55,6 +55,7 @@ describe('searchAfterAndBulkCreate', () => {
     expect(mockService.callCluster).toHaveBeenCalledTimes(0);
     expect(success).toEqual(true);
     expect(createdSignalsCount).toEqual(0);
+    expect(lastLookBackDate).toBeNull();
   });
 
   test('if successful iteration of while loop with maxDocs', async () => {
@@ -105,7 +106,7 @@ describe('searchAfterAndBulkCreate', () => {
           },
         ],
       });
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: repeatedSearchResultsWithSortId(3, 1, someGuids.slice(6, 9)),
       ruleParams: sampleParams,
       services: mockService,
@@ -130,13 +131,14 @@ describe('searchAfterAndBulkCreate', () => {
     expect(mockService.callCluster).toHaveBeenCalledTimes(5);
     expect(success).toEqual(true);
     expect(createdSignalsCount).toEqual(3);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 
   test('if unsuccessful first bulk create', async () => {
     const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
     const sampleParams = sampleRuleAlertParams(10);
     mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult);
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
       ruleParams: sampleParams,
       services: mockService,
@@ -161,6 +163,7 @@ describe('searchAfterAndBulkCreate', () => {
     expect(mockLogger.error).toHaveBeenCalled();
     expect(success).toEqual(false);
     expect(createdSignalsCount).toEqual(1);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 
   test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => {
@@ -179,7 +182,7 @@ describe('searchAfterAndBulkCreate', () => {
         },
       ],
     });
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: sampleDocSearchResultsNoSortId(),
       ruleParams: sampleParams,
       services: mockService,
@@ -204,6 +207,7 @@ describe('searchAfterAndBulkCreate', () => {
     expect(mockLogger.error).toHaveBeenCalled();
     expect(success).toEqual(false);
     expect(createdSignalsCount).toEqual(1);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 
   test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => {
@@ -222,7 +226,7 @@ describe('searchAfterAndBulkCreate', () => {
         },
       ],
     });
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: sampleDocSearchResultsNoSortIdNoHits(),
       ruleParams: sampleParams,
       services: mockService,
@@ -246,6 +250,7 @@ describe('searchAfterAndBulkCreate', () => {
     });
     expect(success).toEqual(true);
     expect(createdSignalsCount).toEqual(1);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 
   test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => {
@@ -267,7 +272,7 @@ describe('searchAfterAndBulkCreate', () => {
         ],
       })
       .mockResolvedValueOnce(sampleDocSearchResultsNoSortId());
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
       ruleParams: sampleParams,
       services: mockService,
@@ -291,6 +296,7 @@ describe('searchAfterAndBulkCreate', () => {
     });
     expect(success).toEqual(true);
     expect(createdSignalsCount).toEqual(1);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 
   test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => {
@@ -312,7 +318,7 @@ describe('searchAfterAndBulkCreate', () => {
         ],
       })
       .mockResolvedValueOnce(sampleEmptyDocSearchResults());
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
       ruleParams: sampleParams,
       services: mockService,
@@ -336,6 +342,7 @@ describe('searchAfterAndBulkCreate', () => {
     });
     expect(success).toEqual(true);
     expect(createdSignalsCount).toEqual(1);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 
   test('if returns false when singleSearchAfter throws an exception', async () => {
@@ -359,7 +366,7 @@ describe('searchAfterAndBulkCreate', () => {
       .mockImplementation(() => {
         throw Error('Fake Error');
       });
-    const { success, createdSignalsCount } = await searchAfterAndBulkCreate({
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
       someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
       ruleParams: sampleParams,
       services: mockService,
@@ -383,5 +390,6 @@ describe('searchAfterAndBulkCreate', () => {
     });
     expect(success).toEqual(false);
     expect(createdSignalsCount).toEqual(1);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
 });

x-pack/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts

@@ -98,13 +98,15 @@ export const searchAfterAndBulkCreate = async ({
     tags,
     throttle,
   });
-  toReturn.lastLookBackDate =
+
-    someResult.hits.hits.length > 0
+  if (createdItemsCount > 0) {
-      ? new Date(someResult.hits.hits[someResult.hits.hits.length - 1]?._source['@timestamp'])
-      : null;
-  if (createdItemsCount) {
     toReturn.createdSignalsCount = createdItemsCount;
+    toReturn.lastLookBackDate =
+      someResult.hits.hits.length > 0
+        ? new Date(someResult.hits.hits[someResult.hits.hits.length - 1]?._source['@timestamp'])
+        : null;
   }
+
   if (bulkCreateDuration) {
     toReturn.bulkCreateTimes.push(bulkCreateDuration);
   }

x-pack/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts

@@ -300,7 +300,7 @@ describe('singleBulkCreate', () => {
         _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a',
         _source: {
           someKey: 'someValue',
-          '@timestamp': 'someTimeStamp',
+          '@timestamp': '2020-04-20T21:27:45+0000',
           signal: {
             parent: {
               rule: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
@@ -334,7 +334,7 @@ describe('singleBulkCreate', () => {
 
   test('filter duplicate rules will return back search responses if they do not have a signal and will NOT filter the source out', () => {
     const ancestors = sampleDocWithAncestors();
-    ancestors.hits.hits[0]._source = { '@timestamp': 'some timestamp' };
+    ancestors.hits.hits[0]._source = { '@timestamp': '2020-04-20T21:27:45+0000' };
     const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', ancestors);
     expect(filtered).toEqual([
       {
@@ -343,7 +343,7 @@ describe('singleBulkCreate', () => {
         _score: 100,
         _version: 1,
         _id: 'e1e08ddc-5e37-49ff-a258-5393aa44435a',
-        _source: { '@timestamp': 'some timestamp' },
+        _source: { '@timestamp': '2020-04-20T21:27:45+0000' },
       },
     ]);
   });