[Security Solutions] Fixes 11 different flakey FTR/e2e tests and scenarios (#115688)

## Summary

Fixes flakes across tests that have either been skipped or have been a source of flake in the categories of:
* Sorting fixes because Elasticsearch can return hits/arrays back in different orders
* Flat array fixes because Elasticsearch can sometimes return `[]` or `[[]]` in-deterministically in some cases 🤷 , so we just flatten the array out completely and test for `[]` within those tests.
* `waitForSignalsToBePresent` was missing in a test and sometimes we would get an empty array response which would fail CI.

Also I audited other tests for `[[]]` and `waitForSignalsToBePresent` and fixed them where they were present or if the `waitForSignalsToBePresent` count was incorrect. This should give us more stability when the CI is under pressure.

Sorting fixes:
https://github.com/elastic/kibana/issues/115554
https://github.com/elastic/kibana/issues/115321
https://github.com/elastic/kibana/issues/115319
https://github.com/elastic/kibana/issues/114581


Flat array fixes:
https://github.com/elastic/kibana/issues/89052
https://github.com/elastic/kibana/issues/115315
https://github.com/elastic/kibana/issues/115308
https://github.com/elastic/kibana/issues/115304
https://github.com/elastic/kibana/issues/115313
https://github.com/elastic/kibana/issues/113418

Missing additional check for "waitForSignalsToBePresent" or incorrect number of signals to wait for fixes:
https://github.com/elastic/kibana/issues/115310


### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts

@@ -51,9 +51,9 @@ export default ({ getService }: FtrProviderContext) => {
       await waitForRuleSuccessOrStatus(supertest, id);
       await waitForSignalsToBePresent(supertest, 4, [id]);
       const signalsOpen = await getSignalsById(supertest, id);
-      const hits = signalsOpen.hits.hits.map(
-        (signal) => (signal._source?.host_alias as HostAlias).name
-      );
+      const hits = signalsOpen.hits.hits
+        .map((signal) => (signal._source?.host_alias as HostAlias).name)
+        .sort();
       expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']);
     });
 
@@ -63,7 +63,9 @@ export default ({ getService }: FtrProviderContext) => {
       await waitForRuleSuccessOrStatus(supertest, id);
       await waitForSignalsToBePresent(supertest, 4, [id]);
       const signalsOpen = await getSignalsById(supertest, id);
-      const hits = signalsOpen.hits.hits.map((signal) => (signal._source?.host as HostAlias).name);
+      const hits = signalsOpen.hits.hits
+        .map((signal) => (signal._source?.host as HostAlias).name)
+        .sort();
       expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']);
     });
   });

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts

@@ -7,6 +7,7 @@
 
 import expect from '@kbn/expect';
 
+import type SuperTest from 'supertest';
 import {
   createListsIndex,
   deleteAllExceptions,
@@ -25,6 +26,45 @@ import {
   waitForSignalsToBePresent,
 } from '../../utils';
 
+interface Host {
+  os: {
+    type?: string;
+    name?: string;
+  };
+}
+
+/**
+ * Convenience method to get signals by host and sort them for better deterministic testing
+ * since Elastic can return the hits back in any order we want to sort them on return for testing.
+ * @param supertest Super test for testing.
+ * @param id The signals id
+ * @returns The array of hosts sorted
+ */
+export const getHostHits = async (
+  supertest: SuperTest.SuperTest<SuperTest.Test>,
+  id: string
+): Promise<Host[]> => {
+  const signalsOpen = await getSignalsById(supertest, id);
+  return signalsOpen.hits.hits
+    .map<Host>((hit) => hit._source?.host as Host)
+    .sort((a, b) => {
+      let sortOrder = 0;
+      if (a.os.name != null && b.os.name != null) {
+        sortOrder += a.os.name.localeCompare(b.os.name);
+      }
+      if (a.os.type != null && b.os.type != null) {
+        sortOrder += a.os.type.localeCompare(b.os.type);
+      }
+      if (a.os.type != null && b.os.name != null) {
+        sortOrder += a.os.type.localeCompare(b.os.name);
+      }
+      if (a.os.name != null && b.os.type != null) {
+        sortOrder += a.os.name.localeCompare(b.os.type);
+      }
+      return sortOrder;
+    });
+};
+
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
@@ -64,20 +104,19 @@ export default ({ getService }: FtrProviderContext) => {
         const { id } = await createRule(supertest, rule);
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 4, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host).sort();
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
             os: { type: 'linux' },
           },
           {
-            os: { type: 'windows' },
+            os: { type: 'linux' },
           },
           {
             os: { type: 'macos' },
           },
           {
-            os: { type: 'linux' },
+            os: { type: 'windows' },
           },
         ]);
       });
@@ -87,20 +126,19 @@ export default ({ getService }: FtrProviderContext) => {
         const { id } = await createRule(supertest, rule);
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 4, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host).sort();
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
             os: { name: 'Linux' },
           },
           {
-            os: { name: 'Windows' },
+            os: { name: 'Linux' },
           },
           {
             os: { name: 'Macos' },
           },
           {
-            os: { name: 'Linux' },
+            os: { name: 'Windows' },
           },
         ]);
       });
@@ -130,17 +168,16 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 3, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { name: 'Windows' },
+              os: { name: 'Linux' },
             },
             {
               os: { name: 'Macos' },
             },
             {
-              os: { name: 'Linux' },
+              os: { name: 'Windows' },
             },
           ]);
         });
@@ -167,17 +204,16 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 3, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { name: 'Windows' },
+              os: { name: 'Linux' },
             },
             {
               os: { name: 'Macos' },
             },
             {
-              os: { name: 'Linux' },
+              os: { name: 'Windows' },
             },
           ]);
         });
@@ -215,14 +251,13 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 2, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { name: 'Macos' },
+              os: { name: 'Linux' },
             },
             {
-              os: { name: 'Linux' },
+              os: { name: 'Macos' },
             },
           ]);
         });
@@ -260,14 +295,13 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 2, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { name: 'Macos' },
+              os: { name: 'Linux' },
             },
             {
-              os: { name: 'Linux' },
+              os: { name: 'Macos' },
             },
           ]);
         });
@@ -296,17 +330,16 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 3, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { type: 'windows' },
+              os: { type: 'linux' },
             },
             {
               os: { type: 'macos' },
             },
             {
-              os: { type: 'linux' },
+              os: { type: 'windows' },
             },
           ]);
         });
@@ -333,17 +366,16 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 3, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { type: 'windows' },
+              os: { type: 'linux' },
             },
             {
               os: { type: 'macos' },
             },
             {
-              os: { type: 'linux' },
+              os: { type: 'windows' },
             },
           ]);
         });
@@ -381,14 +413,13 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 2, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { type: 'macos' },
+              os: { type: 'linux' },
             },
             {
-              os: { type: 'linux' },
+              os: { type: 'macos' },
             },
           ]);
         });
@@ -426,14 +457,13 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 2, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { type: 'macos' },
+              os: { type: 'linux' },
             },
             {
-              os: { type: 'linux' },
+              os: { type: 'macos' },
             },
           ]);
         });
@@ -462,14 +492,13 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 6, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { type: 'windows' },
+              os: { type: 'linux' },
             },
             {
-              os: { name: 'Windows' },
+              os: { name: 'Linux' },
             },
             {
               os: { type: 'macos' },
@@ -478,10 +507,10 @@ export default ({ getService }: FtrProviderContext) => {
               os: { name: 'Macos' },
             },
             {
-              os: { type: 'linux' },
+              os: { type: 'windows' },
             },
             {
-              os: { name: 'Linux' },
+              os: { name: 'Windows' },
             },
           ]);
         });
@@ -508,14 +537,13 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 6, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
             {
-              os: { type: 'windows' },
+              os: { type: 'linux' },
             },
             {
-              os: { name: 'Windows' },
+              os: { name: 'Linux' },
             },
             {
               os: { type: 'macos' },
@@ -524,10 +552,10 @@ export default ({ getService }: FtrProviderContext) => {
               os: { name: 'Macos' },
             },
             {
-              os: { type: 'linux' },
+              os: { type: 'windows' },
             },
             {
-              os: { name: 'Linux' },
+              os: { name: 'Windows' },
             },
           ]);
         });
@@ -565,21 +593,20 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 4, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
-            {
-              os: { type: 'macos' },
-            },
-            {
-              os: { name: 'Macos' },
-            },
             {
               os: { type: 'linux' },
             },
             {
               os: { name: 'Linux' },
             },
+            {
+              os: { type: 'macos' },
+            },
+            {
+              os: { name: 'Macos' },
+            },
           ]);
         });
 
@@ -616,21 +643,20 @@ export default ({ getService }: FtrProviderContext) => {
           );
           await waitForRuleSuccessOrStatus(supertest, id);
           await waitForSignalsToBePresent(supertest, 4, [id]);
-          const signalsOpen = await getSignalsById(supertest, id);
-          const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+          const hits = await getHostHits(supertest, id);
           expect(hits).to.eql([
-            {
-              os: { type: 'macos' },
-            },
-            {
-              os: { name: 'Macos' },
-            },
             {
               os: { type: 'linux' },
             },
             {
               os: { name: 'Linux' },
             },
+            {
+              os: { type: 'macos' },
+            },
+            {
+              os: { name: 'Macos' },
+            },
           ]);
         });
       });
@@ -668,8 +694,7 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 1, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
             os: { type: 'macos' },
@@ -708,8 +733,7 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 1, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
             os: { type: 'macos' },
@@ -741,17 +765,16 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 3, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
             os: { type: 'linux' },
           },
           {
-            os: { type: 'macos' },
+            os: { type: 'linux' },
           },
           {
-            os: { type: 'linux' },
+            os: { type: 'macos' },
           },
         ]);
       });
@@ -778,14 +801,13 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 2, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
-            os: { type: 'macos' },
+            os: { type: 'linux' },
           },
           {
-            os: { type: 'linux' },
+            os: { type: 'macos' },
           },
         ]);
       });
@@ -812,14 +834,13 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 2, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
-            os: { type: 'macos' },
+            os: { type: 'linux' },
           },
           {
-            os: { type: 'linux' },
+            os: { type: 'macos' },
           },
         ]);
       });
@@ -846,20 +867,19 @@ export default ({ getService }: FtrProviderContext) => {
         );
         await waitForRuleSuccessOrStatus(supertest, id);
         await waitForSignalsToBePresent(supertest, 4, [id]);
-        const signalsOpen = await getSignalsById(supertest, id);
-        const hits = signalsOpen.hits.hits.map((hit) => hit._source?.host);
+        const hits = await getHostHits(supertest, id);
         expect(hits).to.eql([
           {
             os: { type: 'linux' },
           },
           {
-            os: { type: 'windows' },
+            os: { type: 'linux' },
           },
           {
             os: { type: 'macos' },
           },
           {
-            os: { type: 'linux' },
+            os: { type: 'windows' },
           },
         ]);
       });

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts

@@ -499,7 +499,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.float).sort();
           expect(hits).to.eql(['1.1', '1.2', '1.3']);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts

@@ -501,7 +501,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.integer).sort();
           expect(hits).to.eql(['2', '3', '4']);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts

@@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
-        expect(ips).to.eql([[]]);
+        expect(ips.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
 
       it('should filter a CIDR range of "127.0.0.1/30"', async () => {
@@ -167,7 +167,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
         expect(ips).to.eql([
@@ -190,7 +190,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
         expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
@@ -346,7 +346,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
-        expect(ips).to.eql([[]]);
+        expect(ips.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -392,8 +392,7 @@ export default ({ getService }: FtrProviderContext) => {
       });
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/115315
-    describe.skip('"exists" operator', () => {
+    describe('"exists" operator', () => {
       it('will return 1 empty result if matching against ip', async () => {
         const rule = getRuleForSignalTesting(['ip_as_array']);
         const { id } = await createRuleWithExceptionEntries(supertest, rule, [
@@ -408,7 +407,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, id);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
-        expect(ips).to.eql([[]]);
+        expect(ips.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -487,8 +486,7 @@ export default ({ getService }: FtrProviderContext) => {
         expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
       });
 
-      // FLAKY https://github.com/elastic/kibana/issues/89052
-      it.skip('will return 1 result if we have a list that includes all ips', async () => {
+      it('will return 1 result if we have a list that includes all ips', async () => {
         await importFile(
           supertest,
           'ip',
@@ -512,7 +510,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, id);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
-        expect(ips).to.eql([[]]);
+        expect(ips.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
 
       it('will return 2 results if we have a list which contains the CIDR ranges of "127.0.0.1/32, 127.0.0.2/31, 127.0.0.4/30"', async () => {
@@ -546,7 +544,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
         expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);
@@ -577,7 +575,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
+        await waitForSignalsToBePresent(supertest, 2, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const ips = signalsOpen.hits.hits.map((hit) => hit._source?.ip).sort();
         expect(ips).to.eql([[], ['127.0.0.8', '127.0.0.9', '127.0.0.10']]);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts

@@ -60,7 +60,7 @@ export default ({ getService }: FtrProviderContext) => {
         const rule = getRuleForSignalTesting(['keyword_as_array']);
         const { id } = await createRule(supertest, rule);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 3, [id]);
+        await waitForSignalsToBePresent(supertest, 4, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
         expect(hits).to.eql([
@@ -84,7 +84,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 2, [id]);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
         expect(hits).to.eql([
@@ -153,7 +153,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -281,7 +281,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -328,8 +328,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"exists" operator', () => {
-      // FLAKY https://github.com/elastic/kibana/issues/115308
-      it.skip('will return 1 results if matching against keyword for the empty array', async () => {
+      it('will return 1 results if matching against keyword for the empty array', async () => {
         const rule = getRuleForSignalTesting(['keyword_as_array']);
         const { id } = await createRuleWithExceptionEntries(supertest, rule, [
           [
@@ -343,7 +342,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, id);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -399,7 +398,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 3, [id]);
+        await waitForSignalsToBePresent(supertest, 4, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
         expect(hits).to.eql([
@@ -437,7 +436,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 2, [id]);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
         expect(hits).to.eql([
@@ -497,8 +496,7 @@ export default ({ getService }: FtrProviderContext) => {
         expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
       });
 
-      // FLAKY https://github.com/elastic/kibana/issues/115304
-      it.skip('will return only the empty array for results if we have a list that includes all keyword', async () => {
+      it('will return only the empty array for results if we have a list that includes all keyword', async () => {
         await importFile(
           supertest,
           'keyword',
@@ -522,7 +520,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, id);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.keyword).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts

@@ -499,7 +499,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.long).sort();
           expect(hits).to.eql(['2', '3', '4']);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts

@@ -56,8 +56,7 @@ export default ({ getService }: FtrProviderContext) => {
       await deleteListsIndex(supertest);
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/115310
-    describe.skip('"is" operator', () => {
+    describe('"is" operator', () => {
       it('should find all the text from the data set when no exceptions are set on the rule', async () => {
         const rule = getRuleForSignalTesting(['text']);
         const { id } = await createRule(supertest, rule);
@@ -241,7 +240,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 1, [id]);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
         expect(hits).to.eql(['word four', 'word three', 'word two']);
@@ -344,6 +343,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
+        await waitForSignalsToBePresent(supertest, 4, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
         expect(hits).to.eql(['word four', 'word one', 'word three', 'word two']);
@@ -618,7 +618,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
           expect(hits).to.eql(['word four', 'word three', 'word two']);
@@ -646,7 +646,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 3, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
           expect(hits).to.eql(['word four', 'word three', 'word two']);
@@ -669,7 +669,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 2, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
           expect(hits).to.eql(['word four', 'word two']);
@@ -850,7 +850,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 2, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
           expect(hits).to.eql(['word one', 'word three']);
@@ -878,7 +878,7 @@ export default ({ getService }: FtrProviderContext) => {
             ],
           ]);
           await waitForRuleSuccessOrStatus(supertest, id);
-          await waitForSignalsToBePresent(supertest, 1, [id]);
+          await waitForSignalsToBePresent(supertest, 4, [id]);
           const signalsOpen = await getSignalsById(supertest, id);
           const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
           expect(hits).to.eql(['word four', 'word one', 'word three', 'word two']);

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts

@@ -58,7 +58,7 @@ export default ({ getService }: FtrProviderContext) => {
         const rule = getRuleForSignalTesting(['text_as_array']);
         const { id } = await createRule(supertest, rule);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 3, [id]);
+        await waitForSignalsToBePresent(supertest, 4, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
         expect(hits).to.eql([
@@ -82,7 +82,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 2, [id]);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
         expect(hits).to.eql([
@@ -151,7 +151,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -279,7 +279,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForSignalsToBePresent(supertest, 1, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -326,8 +326,7 @@ export default ({ getService }: FtrProviderContext) => {
     });
 
     describe('"exists" operator', () => {
-      // FLAKY https://github.com/elastic/kibana/issues/115313
-      it.skip('will return 1 results if matching against text for the empty array', async () => {
+      it('will return 1 results if matching against text for the empty array', async () => {
         const rule = getRuleForSignalTesting(['text_as_array']);
         const { id } = await createRuleWithExceptionEntries(supertest, rule, [
           [
@@ -341,7 +340,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, id);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });
 
@@ -435,7 +434,7 @@ export default ({ getService }: FtrProviderContext) => {
           ],
         ]);
         await waitForRuleSuccessOrStatus(supertest, id);
-        await waitForSignalsToBePresent(supertest, 2, [id]);
+        await waitForSignalsToBePresent(supertest, 3, [id]);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
         expect(hits).to.eql([
@@ -495,8 +494,7 @@ export default ({ getService }: FtrProviderContext) => {
         expect(hits).to.eql([[], ['word eight', 'word nine', 'word ten']]);
       });
 
-      // FLAKY https://github.com/elastic/kibana/issues/113418
-      it.skip('will return only the empty array for results if we have a list that includes all text', async () => {
+      it('will return only the empty array for results if we have a list that includes all text', async () => {
         await importFile(
           supertest,
           'text',
@@ -520,7 +518,7 @@ export default ({ getService }: FtrProviderContext) => {
         await waitForRuleSuccessOrStatus(supertest, id);
         const signalsOpen = await getSignalsById(supertest, id);
         const hits = signalsOpen.hits.hits.map((hit) => hit._source?.text).sort();
-        expect(hits).to.eql([[]]);
+        expect(hits.flat(Number.MAX_SAFE_INTEGER)).to.eql([]);
       });
     });