[Security Solution][Entity Store][Test] Make Entity Store initialization more resilient (#223528)

## Summary

Add retries and error-check logic to Entity Store set-up. Also clean up
running Enrichment Policies, which have been preventing proper tear down
in the past.

- Closes https://github.com/elastic/kibana/issues/222090
- Closes https://github.com/elastic/kibana/issues/223166
- Closes https://github.com/elastic/kibana/issues/224196

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/test/security_solution_api_integration/test_suites/entity_analytics/entity_store/trial_license_complete_tier/host_transform.ts

@@ -19,14 +19,13 @@ import { EntityStoreUtils } from '../../utils';
 const DATASTREAM_NAME: string = 'logs-elastic_agent.cloudbeat-test';
 const HOST_TRANSFORM_ID: string = 'entities-v1-latest-security_host_default';
 const INDEX_NAME: string = '.entities.v1.latest.security_host_default';
-const TIMEOUT_MS: number = 300000; // 5 minutes
+const TIMEOUT_MS: number = 600000; // 10 minutes
 
 export default function (providerContext: FtrProviderContext) {
   const supertest = providerContext.getService('supertest');
   const retry = providerContext.getService('retry');
   const es = providerContext.getService('es');
   const dataView = dataViewRouteHelpersFactory(supertest);
-  const utils = EntityStoreUtils(providerContext.getService);
 
   describe('@ess Host transform logic', () => {
     describe('Entity Store is not installed by default', () => {
@@ -40,7 +39,7 @@ export default function (providerContext: FtrProviderContext) {
 
     describe('Install Entity Store and test Host transform', () => {
       before(async () => {
-        await utils.cleanEngines();
+        await cleanUpEntityStore(providerContext);
         // Initialize security solution by creating a prerequisite index pattern.
         // Helps avoid "Error initializing entity store: Data view not found 'security-solution-default'"
         await dataView.create('security-solution');
@@ -55,26 +54,11 @@ export default function (providerContext: FtrProviderContext) {
 
       beforeEach(async () => {
         // Now we can enable the Entity Store...
-        const response = await supertest
-          .post('/api/entity_store/enable')
-          .set('kbn-xsrf', 'xxxx')
-          .send({});
-        expect(response.statusCode).to.eql(200);
-        expect(response.body.succeeded).to.eql(true);
-
-        // and wait for it to start up
-        await retry.waitForWithTimeout('Entity Store to initialize', TIMEOUT_MS, async () => {
-          const { body } = await supertest
-            .get('/api/entity_store/status')
-            .query({ include_components: true })
-            .expect(200);
-          expect(body.status).to.eql('running');
-          return true;
-        });
+        await enableEntityStore(providerContext);
       });
 
       afterEach(async () => {
-        await utils.cleanEngines();
+        await cleanUpEntityStore(providerContext);
       });
 
       it("Should return 200 and status 'running' for all engines", async () => {
@@ -264,6 +248,69 @@ async function createDocumentsAndTriggerTransform(
   });
 }
 
+async function enableEntityStore(providerContext: FtrProviderContext): Promise<void> {
+  const log = providerContext.getService('log');
+  const supertest = providerContext.getService('supertest');
+  const retry = providerContext.getService('retry');
+
+  const RETRIES = 5;
+  let success: boolean = false;
+  for (let attempt = 0; attempt < RETRIES; attempt++) {
+    const response = await supertest
+      .post('/api/entity_store/enable')
+      .set('kbn-xsrf', 'xxxx')
+      .send({});
+    expect(response.statusCode).to.eql(200);
+    expect(response.body.succeeded).to.eql(true);
+
+    // and wait for it to start up
+    await retry.waitForWithTimeout('Entity Store to initialize', TIMEOUT_MS, async () => {
+      const { body } = await supertest
+        .get('/api/entity_store/status')
+        .query({ include_components: true })
+        .expect(200);
+      if (body.status === 'error') {
+        log.error(`Expected body.status to be 'running', got 'error': ${JSON.stringify(body)}`);
+        success = false;
+        return true;
+      }
+      expect(body.status).to.eql('running');
+      success = true;
+      return true;
+    });
+
+    if (success) {
+      break;
+    } else {
+      log.info(`Retrying Entity Store setup...`);
+      await cleanUpEntityStore(providerContext);
+    }
+  }
+  expect(success).ok();
+}
+
+async function cleanUpEntityStore(providerContext: FtrProviderContext): Promise<void> {
+  const log = providerContext.getService('log');
+  const es = providerContext.getService('es');
+  const utils = EntityStoreUtils(providerContext.getService);
+  const attempts = 5;
+  const delayMs = 60000;
+
+  await utils.cleanEngines();
+  for (const kind of ['host', 'user', 'service', 'generic']) {
+    const name: string = `entity_store_field_retention_${kind}_default_v1.0.0`;
+    for (let currentAttempt = 0; currentAttempt < attempts; currentAttempt++) {
+      try {
+        await es.enrich.deletePolicy({ name }, { ignore: [404] });
+        break;
+      } catch (e) {
+        log.error(`Error deleting policy ${name}: ${e.message} after ${currentAttempt} tries`);
+      }
+      await new Promise((resolve) => setTimeout(resolve, delayMs));
+    }
+  }
+}
+
 interface HostTransformResult {
   host: HostTransformResultHost;
 }