[9.0] [Synthetics] Retries journeys on failures !! (#215264) (#215311)

# Backport

This will backport the following commits from `main` to `9.0`:
- [[Synthetics] Retries journeys on failures !!
(#215264)](https://github.com/elastic/kibana/pull/215264)

<!--- Backport version: 9.6.6 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT
[{"author":{"name":"Shahzad","email":"shahzad31comp@gmail.com"},"sourceCommit":{"committedDate":"2025-03-20T10:53:55Z","message":"[Synthetics]
Retries journeys on failures !! (#215264)\n\n## Summary\n\nRetries
journeys on failures
!!","sha":"0dad68dce78a21b74e5a946e494f8d2bf81c6822","branchLabelMapping":{"^v9.1.0$":"main","^v8.19.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:version","v8.18.0","ci:synthetics-runner-suites","v9.1.0","v8.19.0"],"title":"[Synthetics]
Retries journeys on failures
!!","number":215264,"url":"https://github.com/elastic/kibana/pull/215264","mergeCommit":{"message":"[Synthetics]
Retries journeys on failures !! (#215264)\n\n## Summary\n\nRetries
journeys on failures
!!","sha":"0dad68dce78a21b74e5a946e494f8d2bf81c6822"}},"sourceBranch":"main","suggestedTargetBranches":["9.0","8.18","8.x"],"targetPullRequestStates":[{"branch":"9.0","label":"v9.0.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"8.18","label":"v8.18.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.1.0","branchLabelMappingKey":"^v9.1.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/215264","number":215264,"mergeCommit":{"message":"[Synthetics]
Retries journeys on failures !! (#215264)\n\n## Summary\n\nRetries
journeys on failures
!!","sha":"0dad68dce78a21b74e5a946e494f8d2bf81c6822"}},{"branch":"8.x","label":"v8.19.0","branchLabelMappingKey":"^v8.19.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Shahzad <shahzad31comp@gmail.com>

x-pack/solutions/observability/packages/synthetics-test-data/src/e2e/helpers/synthetics_runner.ts

@@ -8,20 +8,28 @@
 /* eslint-disable no-console */
 
 import Url from 'url';
-import { run as syntheticsRun } from '@elastic/synthetics';
-import { PromiseType } from 'utility-types';
+import { JourneyResult, run as syntheticsRun, Runner } from '@elastic/synthetics';
 import { createApmUsers } from '@kbn/apm-plugin/server/test_helpers/create_apm_users/create_apm_users';
 
 import { EsArchiver } from '@kbn/es-archiver';
 import { esArchiverUnload } from '../tasks/es_archiver';
 import { TestReporter } from './test_reporter';
 
+const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
+
+// @ts-ignore
+export const runner: Runner = global[SYNTHETICS_RUNNER];
+
 export interface ArgParams {
   headless: boolean;
   match?: string;
   pauseOnError: boolean;
 }
 
+interface JourneyResults {
+  [x: string]: JourneyResult;
+}
+
 export class SyntheticsRunner {
   public getService: any;
   public kibanaUrl: string;
@@ -59,6 +67,34 @@ export class SyntheticsRunner {
     console.log('Successfully loaded test files');
   }
 
+  async reloadTestFiles(results: JourneyResults, onlyOnFailure = false) {
+    if (!this.loadTestFilesCallback) {
+      throw new Error('Test files not loaded');
+    }
+    console.log('Reloading test files');
+    // Clear require cache
+    const fileKeys = Object.keys(require.cache);
+    Object.entries(results).forEach(([_journey, result]) => {
+      const fileName = result.location?.file;
+      if (fileName) {
+        const key = fileKeys.find((k) => k.includes(fileName));
+        if (key && (onlyOnFailure ? result.status !== 'succeeded' : true)) {
+          delete require.cache[key];
+          // make sure key is deleted
+          if (require.cache[key]) {
+            console.log(`Failed to delete ${key} from require cache`);
+          } else {
+            // reload the file
+            console.log(`Reloading ${key}...`);
+            require(key);
+          }
+        }
+      }
+    });
+
+    await this.loadTestFiles(this.loadTestFilesCallback, true);
+  }
+
   async loadTestData(e2eDir: string, dataArchives: string[]) {
     try {
       console.log('Loading esArchiver...');
@@ -106,12 +142,12 @@ export class SyntheticsRunner {
       throw new Error('Test files not loaded');
     }
     const { headless, match, pauseOnError } = this.params;
-    const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : 1;
     const CI = process.env.CI === 'true';
+    const noOfRuns = process.env.NO_OF_RUNS ? Number(process.env.NO_OF_RUNS) : CI ? 2 : 1;
     console.log(`Running ${noOfRuns} times`);
-    let results: PromiseType<ReturnType<typeof syntheticsRun>> = {};
+    let results: JourneyResults = {};
     for (let i = 0; i < noOfRuns; i++) {
-      results = await syntheticsRun({
+      const currResults = await syntheticsRun({
         params: {
           kibanaUrl: this.kibanaUrl,
           getService: this.getService,
@@ -135,16 +171,25 @@ export class SyntheticsRunner {
         screenshots: 'only-on-failure',
         reporter: TestReporter,
       });
-      if (noOfRuns > 1) {
+      results = { ...results, ...currResults };
+      if (i < noOfRuns - 1) {
+        console.log(`Run ${i + 1} completed, reloading test files...`);
         // need to reload again since runner resets the journeys
-        await this.loadTestFiles(this.loadTestFilesCallback!, true);
+        const failedJourneys = Object.entries(currResults).filter(
+          ([_journey, result]) => result.status !== 'succeeded'
+        );
+        if (CI && failedJourneys.length === 0) {
+          console.log('All journeys succeeded, skipping reload');
+          break;
+        }
+        await this.reloadTestFiles(results, CI);
       }
     }
 
-    await this.assertResults(results);
+    this.assertResults(results);
   }
 
-  assertResults(results: PromiseType<ReturnType<typeof syntheticsRun>>) {
+  assertResults(results: JourneyResults) {
     console.log('Asserting results...');
     Object.entries(results).forEach(([_journey, result]) => {
       console.log(`Journey: ${_journey}, Status: ${result.status}`);

x-pack/solutions/observability/plugins/uptime/e2e/uptime/synthetics_run.ts

@@ -30,15 +30,7 @@ async function runE2ETests({ readConfigFile }: FtrConfigProviderContext) {
         'browser',
       ]);
 
-      await syntheticsRunner.loadTestFiles(async (reload) => {
-        if (reload) {
-          const dirPath = require.resolve('./journeys').replace('index.ts', '');
-          Object.keys(require.cache).forEach(function (key) {
-            if (key.startsWith(dirPath)) {
-              delete require.cache[key];
-            }
-          });
-        }
+      await syntheticsRunner.loadTestFiles(async () => {
         require(path.join(__dirname, './journeys'));
       });