[Defend Workflows] Unskip endpoint FTR tests (#161646)

## Summary

Unskips tests that were skipped in
bb824d3c10

Attempt to fix `no_shard_available_action_exception` type errors that
often happen during install of transforms (part of Endpoint package
install).

closes #72874


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Paul Tavares <paul.tavares@elastic.co>
Co-authored-by: Paul Tavares <56442535+paul-tavares@users.noreply.github.com>

x-pack/plugins/security_solution/common/endpoint/data_loaders/setup_fleet_for_endpoint.ts

@@ -14,7 +14,11 @@ import type {
   PostFleetSetupResponse,
 } from '@kbn/fleet-plugin/common';
 import { AGENTS_SETUP_API_ROUTES, EPM_API_ROUTES, SETUP_API_ROUTE } from '@kbn/fleet-plugin/common';
-import { EndpointDataLoadingError, wrapErrorAndRejectPromise } from './utils';
+import { ToolingLog } from '@kbn/tooling-log';
+import { UsageTracker } from './usage_tracker';
+import { EndpointDataLoadingError, retryOnError, wrapErrorAndRejectPromise } from './utils';
+
+const usageTracker = new UsageTracker({ dumpOnProcessExit: true });
 
 export interface SetupFleetForEndpointResponse {
   endpointPackage: BulkInstallPackageInfo;
@@ -23,11 +27,16 @@ export interface SetupFleetForEndpointResponse {
 /**
  * Calls the fleet setup APIs and then installs the latest Endpoint package
  * @param kbnClient
+ * @param logger
  */
-export const setupFleetForEndpoint = async (kbnClient: KbnClient): Promise<void> => {
-  // We try to use the kbnClient **private** logger, bug if unable to access it, then just use console
-  // @ts-expect-error TS2341
-  const log = kbnClient.log ? kbnClient.log : console;
+export const setupFleetForEndpoint = async (
+  kbnClient: KbnClient,
+  logger?: ToolingLog
+): Promise<void> => {
+  const log = logger ?? new ToolingLog();
+  const usageRecord = usageTracker.create('setupFleetForEndpoint()');
+
+  log.info(`setupFleetForEndpoint(): Setting up fleet for endpoint`);
 
   // Setup Fleet
   try {
@@ -39,11 +48,13 @@ export const setupFleetForEndpoint = async (kbnClient: KbnClient): Promise<void>
       .catch(wrapErrorAndRejectPromise)) as AxiosResponse<PostFleetSetupResponse>;
 
     if (!setupResponse.data.isInitialized) {
-      log.error(setupResponse.data);
+      log.error(new Error(JSON.stringify(setupResponse.data, null, 2)));
       throw new Error('Initializing the ingest manager failed, existing');
     }
   } catch (error) {
     log.error(error);
+    usageRecord.set('failure', error.message);
+
     throw error;
   }
 
@@ -57,70 +68,102 @@ export const setupFleetForEndpoint = async (kbnClient: KbnClient): Promise<void>
       .catch(wrapErrorAndRejectPromise)) as AxiosResponse<PostFleetSetupResponse>;
 
     if (!setupResponse.data.isInitialized) {
-      log.error(setupResponse.data);
-      throw new Error('Initializing Fleet failed, existing');
+      log.error(new Error(JSON.stringify(setupResponse, null, 2)));
+      throw new Error('Initializing Fleet failed');
     }
   } catch (error) {
     log.error(error);
+
+    usageRecord.set('failure', error.message);
+
     throw error;
   }
 
   // Install/upgrade the endpoint package
   try {
-    await installOrUpgradeEndpointFleetPackage(kbnClient);
+    await installOrUpgradeEndpointFleetPackage(kbnClient, log);
   } catch (error) {
     log.error(error);
+
+    usageRecord.set('failure', error.message);
+
     throw error;
   }
+
+  usageRecord.set('success');
 };
 
 /**
  * Installs the Endpoint package (or upgrades it) in Fleet to the latest available in the registry
  *
  * @param kbnClient
+ * @param logger
  */
 export const installOrUpgradeEndpointFleetPackage = async (
-  kbnClient: KbnClient
+  kbnClient: KbnClient,
+  logger: ToolingLog
 ): Promise<BulkInstallPackageInfo> => {
-  const installEndpointPackageResp = (await kbnClient
-    .request({
-      path: EPM_API_ROUTES.BULK_INSTALL_PATTERN,
-      method: 'POST',
-      body: {
-        packages: ['endpoint'],
-      },
-      query: {
-        prerelease: true,
-      },
-    })
-    .catch(wrapErrorAndRejectPromise)) as AxiosResponse<BulkInstallPackagesResponse>;
+  logger.info(`installOrUpgradeEndpointFleetPackage(): starting`);
 
-  const bulkResp = installEndpointPackageResp.data.items;
+  const usageRecord = usageTracker.create('installOrUpgradeEndpointFleetPackage()');
 
-  if (bulkResp.length <= 0) {
-    throw new EndpointDataLoadingError(
-      'Installing the Endpoint package failed, response was empty, existing',
-      bulkResp
-    );
-  }
+  const updatePackages = async () => {
+    const installEndpointPackageResp = (await kbnClient
+      .request({
+        path: EPM_API_ROUTES.BULK_INSTALL_PATTERN,
+        method: 'POST',
+        body: {
+          packages: ['endpoint'],
+        },
+        query: {
+          prerelease: true,
+        },
+      })
+      .catch(wrapErrorAndRejectPromise)) as AxiosResponse<BulkInstallPackagesResponse>;
 
-  const firstError = bulkResp[0];
+    logger.debug(`Fleet bulk install response:`, installEndpointPackageResp.data);
 
-  if (isFleetBulkInstallError(firstError)) {
-    if (firstError.error instanceof Error) {
+    const bulkResp = installEndpointPackageResp.data.items;
+
+    if (bulkResp.length <= 0) {
       throw new EndpointDataLoadingError(
-        `Installing the Endpoint package failed: ${firstError.error.message}, exiting`,
+        'Installing the Endpoint package failed, response was empty, existing',
         bulkResp
       );
     }
 
-    // Ignore `409` (conflicts due to Concurrent install or upgrades of package) errors
-    if (firstError.statusCode !== 409) {
-      throw new EndpointDataLoadingError(firstError.error, bulkResp);
-    }
-  }
+    const installResponse = bulkResp[0];
 
-  return bulkResp[0] as BulkInstallPackageInfo;
+    logger.debug('package install response:', installResponse);
+
+    if (isFleetBulkInstallError(installResponse)) {
+      if (installResponse.error instanceof Error) {
+        throw new EndpointDataLoadingError(
+          `Installing the Endpoint package failed: ${installResponse.error.message}`,
+          bulkResp
+        );
+      }
+
+      // Ignore `409` (conflicts due to Concurrent install or upgrades of package) errors
+      if (installResponse.statusCode !== 409) {
+        throw new EndpointDataLoadingError(installResponse.error, bulkResp);
+      }
+    }
+
+    return bulkResp[0] as BulkInstallPackageInfo;
+  };
+
+  return retryOnError(updatePackages, ['no_shard_available_action_exception'], logger, 5, 10000)
+    .then((result) => {
+      usageRecord.set('success');
+
+      return result;
+    })
+    .catch((err) => {
+      usageRecord.set('failure', err.message);
+
+      throw err;
+    });
 };
 
 function isFleetBulkInstallError(

x-pack/plugins/security_solution/common/endpoint/data_loaders/usage_tracker.ts

@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable max-classes-per-file */
+
+import { ToolingLog } from '@kbn/tooling-log';
+
+interface UsageRecordJson {
+  id: string;
+  start: string;
+  finish: string;
+  status: 'success' | 'failure' | 'pending';
+  error?: string;
+}
+
+interface UsageTrackerOptions {
+  logger?: ToolingLog;
+  dumpOnProcessExit?: boolean;
+  maxRecordsPerType?: number;
+}
+
+/**
+ * Keep track of usage of stuff. Example: can track how many time a given utility/function was called.
+ *
+ * ** Should not be used for production code **
+ */
+export class UsageTracker {
+  private readonly records: Record<
+    string,
+    {
+      count: number;
+      records: UsageRecord[];
+    }
+  > = {};
+  private readonly options: Required<UsageTrackerOptions>;
+
+  constructor({
+    logger = new ToolingLog(),
+    dumpOnProcessExit = false,
+    maxRecordsPerType = 25,
+  }: UsageTrackerOptions = {}) {
+    this.options = {
+      logger,
+      dumpOnProcessExit,
+      maxRecordsPerType,
+    };
+
+    try {
+      if (dumpOnProcessExit && process && process.once) {
+        ['SIGINT', 'exit', 'uncaughtException', 'unhandledRejection'].forEach((event) => {
+          process.once(event, () => {
+            //
+          });
+        });
+      }
+    } catch (err) {
+      logger.debug(`Unable to setup 'dumpOnProcessExit': ${err.message}`);
+    }
+  }
+
+  create(id: string): UsageRecord {
+    this.records[id] = this.records[id] ?? { count: 0, records: [] };
+
+    const maxRecords = this.options.maxRecordsPerType;
+    const usageRecord = new UsageRecord(id);
+    const usageForId = this.records[id];
+
+    usageForId.count++;
+    usageForId.records.push(usageRecord);
+
+    if (usageForId.records.length > maxRecords) {
+      usageForId.records.splice(0, usageForId.records.length - maxRecords);
+    }
+
+    return usageRecord;
+  }
+
+  toJSON(): UsageRecordJson[] {
+    return Object.values(this.records)
+      .map(({ records }) => records)
+      .flat()
+      .map((record) => record.toJSON());
+  }
+
+  toString(): string {
+    return JSON.stringify(this.toJSON());
+  }
+
+  public dump(logger?: ToolingLog) {
+    (logger ?? this.options.logger).info(
+      Object.entries(this.records)
+        .map(([key, { count, records: usageRecords }]) => {
+          return `
+  [${key}] Invoked ${count} times. Last ${this.options.maxRecordsPerType}:
+      ${usageRecords
+        .map((record) => {
+          return record.toString();
+        })
+        .join('\n      ')}
+`;
+        })
+        .join('\n')
+    );
+  }
+}
+
+class UsageRecord {
+  private start: UsageRecordJson['start'] = new Date().toISOString();
+  private finish: UsageRecordJson['finish'] = '';
+  private status: UsageRecordJson['status'] = 'pending';
+  private error: UsageRecordJson['error'];
+
+  constructor(private readonly id: string) {}
+
+  set(status: Exclude<UsageRecordJson['status'], 'pending'>, error?: string) {
+    this.finish = new Date().toISOString();
+    this.error = error;
+  }
+
+  public toJSON(): UsageRecordJson {
+    const { id, start, finish, status, error } = this;
+
+    return {
+      id,
+      start,
+      finish,
+      status,
+      ...(error ? { error } : {}),
+    };
+  }
+
+  public toString(): string {
+    return JSON.stringify(this.toJSON());
+  }
+}

x-pack/plugins/security_solution/common/endpoint/data_loaders/utils.ts

@@ -6,6 +6,7 @@
  */
 
 import { mergeWith } from 'lodash';
+import { ToolingLog } from '@kbn/tooling-log';
 
 export class EndpointDataLoadingError extends Error {
   constructor(message: string, public meta?: unknown) {
@@ -30,3 +31,60 @@ export const mergeAndAppendArrays = <T, S>(destinationObj: T, srcObj: S): T => {
 
   return mergeWith(destinationObj, srcObj, customizer);
 };
+
+/**
+ * Call the provided `callback` and retry that call if it fails and the error in the failure
+ * contains one of the `errors` provided.
+ * @param callback
+ * @param errors
+ * @param tryCount
+ * @param interval
+ * @param logger
+ */
+export const retryOnError = async <T>(
+  callback: () => Promise<T>,
+  errors: Array<string | RegExp>,
+  logger?: ToolingLog,
+  tryCount: number = 5,
+  interval: number = 10000
+): Promise<T> => {
+  const log = logger ?? new ToolingLog({ writeTo: { write(_: string) {} }, level: 'silent' });
+  const msg = (message: string): string => `retryOnError(): ${message}`;
+  const isRetryableError = (err: Error): boolean => {
+    return errors.some((retryMessage) => {
+      if (typeof retryMessage === 'string') {
+        return err.message.includes(retryMessage);
+      } else {
+        return retryMessage.test(err.message);
+      }
+    });
+  };
+
+  let attempt = 1;
+  let responsePromise: Promise<T>;
+
+  while (attempt <= tryCount) {
+    const thisAttempt = attempt;
+    attempt++;
+
+    log.info(msg(`attempt ${thisAttempt} started at: ${new Date().toISOString()}`));
+
+    try {
+      responsePromise = callback(); // store promise so that if it fails and no more attempts, we return the last failure
+      return await responsePromise;
+    } catch (err) {
+      log.info(msg(`attempt ${thisAttempt} failed with: ${err.message}`), err);
+
+      // If not an error that is retryable, then end loop here and return that error;
+      if (!isRetryableError(err)) {
+        log.error(err);
+        return Promise.reject(err);
+      }
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, interval));
+  }
+
+  // @ts-expect-error TS2454: Variable 'responsePromise' is used before being assigned.
+  return responsePromise;
+};

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -9,6 +9,7 @@ import type { Client } from '@elastic/elasticsearch';
 import seedrandom from 'seedrandom';
 import type { KbnClient } from '@kbn/test';
 import type { CreatePackagePolicyResponse } from '@kbn/fleet-plugin/common';
+import { ToolingLog } from '@kbn/tooling-log';
 import type { TreeOptions } from './generate_data';
 import { EndpointDocGenerator } from './generate_data';
 import type {
@@ -49,6 +50,9 @@ export type IndexedHostsAndAlertsResponse = IndexedHostsResponse;
  * @param options
  * @param DocGenerator
  * @param withResponseActions
+ * @param numResponseActions
+ * @param alertIds
+ * @param logger_
  */
 export async function indexHostsAndAlerts(
   client: Client,
@@ -66,9 +70,11 @@ export async function indexHostsAndAlerts(
   DocGenerator: typeof EndpointDocGenerator = EndpointDocGenerator,
   withResponseActions = true,
   numResponseActions?: number,
-  alertIds?: string[]
+  alertIds?: string[],
+  logger_?: ToolingLog
 ): Promise<IndexedHostsAndAlertsResponse> {
   const random = seedrandom(seed);
+  const logger = logger_ ?? new ToolingLog({ level: 'info', writeTo: process.stdout });
   const epmEndpointPackage = await getEndpointPackageInfo(kbnClient);
   const response: IndexedHostsAndAlertsResponse = {
     hosts: [],
@@ -90,7 +96,7 @@ export async function indexHostsAndAlerts(
   };
 
   // Ensure fleet is setup and endpoint package installed
-  await setupFleetForEndpoint(kbnClient);
+  await setupFleetForEndpoint(kbnClient, logger);
 
   // If `fleet` integration is true, then ensure a (fake) fleet-server is connected
   if (fleet) {

x-pack/plugins/security_solution/scripts/endpoint/common/random_policy_id_generator.ts

@@ -36,7 +36,7 @@ export const randomPolicyIdGenerator: (
   log: ToolingLog
 ) => Promise<() => string> = async (kbn, log) => {
   log.info('Setting up fleet');
-  await setupFleetForEndpoint(kbn);
+  await setupFleetForEndpoint(kbn, log);
   const endpointPackage = await getEndpointPackageInfo(kbn);
 
   log.info('Generarting test policies...');

x-pack/plugins/security_solution/scripts/endpoint/endpoint_policies/index.ts

@@ -31,7 +31,7 @@ export const cli = () => {
       log.info(`Creating ${count} endpoint policies...`);
 
       try {
-        await setupFleetForEndpoint(kbn);
+        await setupFleetForEndpoint(kbn, log);
         const endpointPackage = await getEndpointPackageInfo(kbn);
 
         while (created < max) {

x-pack/test/security_solution_endpoint/apps/endpoint/index.ts

@@ -8,15 +8,14 @@
 import { getRegistryUrl as getRegistryUrlFromIngest } from '@kbn/fleet-plugin/server';
 import { FtrProviderContext } from '../../ftr_provider_context';
 import {
-  isRegistryEnabled,
   getRegistryUrlFromTestEnv,
+  isRegistryEnabled,
 } from '../../../security_solution_endpoint_api_int/registry';
 
 export default function (providerContext: FtrProviderContext) {
   const { loadTestFile, getService } = providerContext;
 
-  // FLAKY: https://github.com/elastic/kibana/issues/72874
-  describe.skip('endpoint', function () {
+  describe('endpoint', function () {
     const ingestManager = getService('ingestManager');
     const log = getService('log');
     const endpointTestResources = getService('endpointTestResources');

x-pack/test/security_solution_endpoint/services/endpoint.ts

@@ -139,7 +139,11 @@ export class EndpointTestResources extends FtrService {
           alertsPerHost,
           enableFleetIntegration,
           undefined,
-          CurrentKibanaVersionDocGenerator
+          CurrentKibanaVersionDocGenerator,
+          undefined,
+          undefined,
+          undefined,
+          this.log
         );
 
     if (waitUntilTransformed && customIndexFn) {
@@ -272,7 +276,7 @@ export class EndpointTestResources extends FtrService {
   async installOrUpgradeEndpointFleetPackage(): ReturnType<
     typeof installOrUpgradeEndpointFleetPackage
   > {
-    return installOrUpgradeEndpointFleetPackage(this.kbnClient);
+    return installOrUpgradeEndpointFleetPackage(this.kbnClient, this.log);
   }
 
   /**