[ftr] remove testMetadata and maintain a unique lifecycle instance per run

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -98,11 +98,7 @@ export function runFtrCli() {
           });
         }
 
-        try {
-          await functionalTestRunner.close();
-        } finally {
-          process.exit();
-        }
+        process.exit();
       };
 
       process.on('unhandledRejection', (err) =>

packages/kbn-test/src/functional_test_runner/functional_test_runner.ts

@@ -14,8 +14,6 @@ import { REPO_ROOT } from '@kbn/utils';
 import { Suite, Test } from './fake_mocha_types';
 import {
   Lifecycle,
-  LifecyclePhase,
-  TestMetadata,
   readConfigFile,
   ProviderCollection,
   Providers,
@@ -30,10 +28,6 @@ import {
 import { createEsClientForFtrConfig } from '../es';
 
 export class FunctionalTestRunner {
-  public readonly lifecycle = new Lifecycle();
-  public readonly testMetadata = new TestMetadata(this.lifecycle);
-  private closed = false;
-
   private readonly esVersion: EsVersion;
   constructor(
     private readonly log: ToolingLog,
@@ -41,12 +35,6 @@ export class FunctionalTestRunner {
     private readonly configOverrides: any,
     esVersion?: string | EsVersion
   ) {
-    for (const [key, value] of Object.entries(this.lifecycle)) {
-      if (value instanceof LifecyclePhase) {
-        value.before$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
-        value.after$.subscribe(() => log.verbose('starting %j lifecycle phase', key));
-      }
-    }
     this.esVersion =
       esVersion === undefined
         ? EsVersion.getDefault()
@@ -58,8 +46,8 @@ export class FunctionalTestRunner {
   async run() {
     const testStats = await this.getTestStats();
 
-    return await this._run(async (config, coreProviders) => {
-      SuiteTracker.startTracking(this.lifecycle, this.configFile);
+    return await this.runHarness(async (config, lifecycle, coreProviders) => {
+      SuiteTracker.startTracking(lifecycle, this.configFile);
 
       const realServices =
         !!config.get('testRunner') ||
@@ -101,7 +89,7 @@ export class FunctionalTestRunner {
       }
 
       const mocha = await setupMocha(
-        this.lifecycle,
+        lifecycle,
         this.log,
         config,
         providers,
@@ -119,10 +107,10 @@ export class FunctionalTestRunner {
         return this.simulateMochaDryRun(mocha);
       }
 
-      await this.lifecycle.beforeTests.trigger(mocha.suite);
+      await lifecycle.beforeTests.trigger(mocha.suite);
       this.log.info('Starting tests');
 
-      return await runTests(this.lifecycle, mocha);
+      return await runTests(lifecycle, mocha);
     });
   }
 
@@ -154,13 +142,13 @@ export class FunctionalTestRunner {
   }
 
   async getTestStats() {
-    return await this._run(async (config, coreProviders) => {
+    return await this.runHarness(async (config, lifecycle, coreProviders) => {
       if (config.get('testRunner')) {
         throw new Error('Unable to get test stats for config that uses a custom test runner');
       }
 
       const providers = this.getStubProviderCollection(config, coreProviders);
-      const mocha = await setupMocha(this.lifecycle, this.log, config, providers, this.esVersion);
+      const mocha = await setupMocha(lifecycle, this.log, config, providers, this.esVersion);
 
       const queue = new Set([mocha.suite]);
       const allTests: Test[] = [];
@@ -216,10 +204,11 @@ export class FunctionalTestRunner {
     ]);
   }
 
-  async _run<T = any>(
-    handler: (config: Config, coreProviders: Providers) => Promise<T>
+  private async runHarness<T = any>(
+    handler: (config: Config, lifecycle: Lifecycle, coreProviders: Providers) => Promise<T>
   ): Promise<T> {
     let runErrorOccurred = false;
+    const lifecycle = new Lifecycle(this.log);
 
     try {
       const config = await readConfigFile(
@@ -240,26 +229,25 @@ export class FunctionalTestRunner {
       const dockerServers = new DockerServersService(
         config.get('dockerServers'),
         this.log,
-        this.lifecycle
+        lifecycle
       );
 
       // base level services that functional_test_runner exposes
       const coreProviders = readProviderSpec('Service', {
-        lifecycle: () => this.lifecycle,
+        lifecycle: () => lifecycle,
         log: () => this.log,
-        testMetadata: () => this.testMetadata,
         config: () => config,
         dockerServers: () => dockerServers,
         esVersion: () => this.esVersion,
       });
 
-      return await handler(config, coreProviders);
+      return await handler(config, lifecycle, coreProviders);
     } catch (runError) {
       runErrorOccurred = true;
       throw runError;
     } finally {
       try {
-        await this.close();
+        await lifecycle.cleanup.trigger();
       } catch (closeError) {
         if (runErrorOccurred) {
           this.log.error('failed to close functional_test_runner');
@@ -272,13 +260,6 @@ export class FunctionalTestRunner {
     }
   }
 
-  async close() {
-    if (this.closed) return;
-
-    this.closed = true;
-    await this.lifecycle.cleanup.trigger();
-  }
-
   simulateMochaDryRun(mocha: any) {
     interface TestEntry {
       file: string;

packages/kbn-test/src/functional_test_runner/index.ts

@@ -15,7 +15,6 @@ export {
   Lifecycle,
   LifecyclePhase,
 } from './lib';
-export type { ScreenshotRecord } from './lib';
 export { runFtrCli } from './cli';
 export * from './lib/docker_servers';
 export * from './public_types';

packages/kbn-test/src/functional_test_runner/lib/index.ts

@@ -12,7 +12,6 @@ export { readConfigFile, Config } from './config';
 export * from './providers';
 // @internal
 export { runTests, setupMocha } from './mocha';
-export * from './test_metadata';
 export * from './docker_servers';
 export { SuiteTracker } from './suite_tracker';
 

packages/kbn-test/src/functional_test_runner/lib/lifecycle.ts

@@ -6,29 +6,48 @@
  * Side Public License, v 1.
  */
 
+import * as Rx from 'rxjs';
+import { ToolingLog } from '@kbn/tooling-log';
+
 import { LifecyclePhase } from './lifecycle_phase';
 
 import { Suite, Test } from '../fake_mocha_types';
 
 export class Lifecycle {
+  /** root subscription to cleanup lifecycle phases when lifecycle completes */
+  private readonly sub = new Rx.Subscription();
+
   /** lifecycle phase that will run handlers once before tests execute */
-  public readonly beforeTests = new LifecyclePhase<[Suite]>({
+  public readonly beforeTests = new LifecyclePhase<[Suite]>(this.sub, {
     singular: true,
   });
   /** lifecycle phase that runs handlers before each runnable (test and hooks) */
-  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>();
+  public readonly beforeEachRunnable = new LifecyclePhase<[Test]>(this.sub);
   /** lifecycle phase that runs handlers before each suite */
-  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>();
+  public readonly beforeTestSuite = new LifecyclePhase<[Suite]>(this.sub);
   /** lifecycle phase that runs handlers before each test */
-  public readonly beforeEachTest = new LifecyclePhase<[Test]>();
+  public readonly beforeEachTest = new LifecyclePhase<[Test]>(this.sub);
   /** lifecycle phase that runs handlers after each suite */
-  public readonly afterTestSuite = new LifecyclePhase<[Suite]>();
+  public readonly afterTestSuite = new LifecyclePhase<[Suite]>(this.sub);
   /** lifecycle phase that runs handlers after a test fails */
-  public readonly testFailure = new LifecyclePhase<[Error, Test]>();
+  public readonly testFailure = new LifecyclePhase<[Error, Test]>(this.sub);
   /** lifecycle phase that runs handlers after a hook fails */
-  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>();
+  public readonly testHookFailure = new LifecyclePhase<[Error, Test]>(this.sub);
   /** lifecycle phase that runs handlers at the very end of execution */
-  public readonly cleanup = new LifecyclePhase<[]>({
+  public readonly cleanup = new LifecyclePhase<[]>(this.sub, {
     singular: true,
   });
+
+  constructor(log: ToolingLog) {
+    for (const [name, phase] of Object.entries(this)) {
+      if (phase instanceof LifecyclePhase) {
+        phase.before$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
+        phase.after$.subscribe(() => log.verbose('starting %j lifecycle phase', name));
+      }
+    }
+
+    this.cleanup.after$.subscribe(() => {
+      this.sub.unsubscribe();
+    });
+  }
 }

packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.test.ts

@@ -26,7 +26,7 @@ describe('with randomness', () => {
   });
 
   it('calls handlers in random order', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
     const order: string[] = [];
 
     phase.add(
@@ -69,7 +69,7 @@ describe('without randomness', () => {
   afterEach(() => jest.restoreAllMocks());
 
   it('calls all handlers and throws first error', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
     const fn1 = jest.fn();
     phase.add(fn1);
 
@@ -88,7 +88,7 @@ describe('without randomness', () => {
   });
 
   it('triggers before$ just before calling handler and after$ once it resolves', async () => {
-    const phase = new LifecyclePhase();
+    const phase = new LifecyclePhase(new Rx.Subscription());
     const order: string[] = [];
 
     const beforeSub = jest.fn(() => order.push('before'));
@@ -116,7 +116,7 @@ describe('without randomness', () => {
   });
 
   it('completes before$ and after$ if phase is singular', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
 
     const beforeNotifs: Array<Rx.Notification<unknown>> = [];
     phase.before$.pipe(materialize()).subscribe((n) => beforeNotifs.push(n));
@@ -160,7 +160,7 @@ describe('without randomness', () => {
   });
 
   it('completes before$ subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
     await phase.trigger();
 
     await expect(phase.before$.pipe(materialize(), toArray()).toPromise()).resolves
@@ -177,7 +177,7 @@ describe('without randomness', () => {
   });
 
   it('replays after$ event subscribers after trigger of singular phase', async () => {
-    const phase = new LifecyclePhase({ singular: true });
+    const phase = new LifecyclePhase(new Rx.Subscription(), { singular: true });
     await phase.trigger();
 
     await expect(phase.after$.pipe(materialize(), toArray()).toPromise()).resolves

packages/kbn-test/src/functional_test_runner/lib/lifecycle_phase.ts

@@ -26,6 +26,7 @@ export class LifecyclePhase<Args extends readonly any[]> {
   public readonly after$: Rx.Observable<void>;
 
   constructor(
+    sub: Rx.Subscription,
     private readonly options: {
       singular?: boolean;
     } = {}
@@ -35,6 +36,12 @@ export class LifecyclePhase<Args extends readonly any[]> {
 
     this.afterSubj = this.options.singular ? new Rx.ReplaySubject<void>(1) : new Rx.Subject<void>();
     this.after$ = this.afterSubj.asObservable();
+
+    sub.add(() => {
+      this.beforeSubj.complete();
+      this.afterSubj.complete();
+      this.handlers.length = 0;
+    });
   }
 
   public add(fn: (...args: Args) => Promise<void> | void) {

packages/kbn-test/src/functional_test_runner/lib/mocha/filter_suites.test.js

@@ -69,6 +69,9 @@ function setup({ include, exclude, esVersion }) {
         info(...args) {
           history.push(`info: ${format(...args)}`);
         },
+        debug(...args) {
+          history.push(`debg: ${format(...args)}`);
+        },
       },
       mocha,
       include,
@@ -221,7 +224,7 @@ it(`excludes tests which don't meet the esVersionRequirement`, async () => {
 
   expect(history).toMatchInlineSnapshot(`
     Array [
-      "info: Only running suites which are compatible with ES version 9.0.0",
+      "debg: Only running suites which are compatible with ES version 9.0.0",
       "suite: ",
       "suite: level 1",
       "suite: level 1 level 1a",

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/ci_stats_ftr_reporter.ts

@@ -17,7 +17,6 @@ import {
 
 import { Config } from '../../config';
 import { Runner } from '../../../fake_mocha_types';
-import { TestMetadata, ScreenshotRecord } from '../../test_metadata';
 import { Lifecycle } from '../../lifecycle';
 import { getSnapshotOfRunnableLogs } from '../../../../mocha';
 
@@ -36,7 +35,6 @@ interface Runnable {
   file: string;
   title: string;
   parent: Suite;
-  _screenshots?: ScreenshotRecord[];
 }
 
 function getHookType(hook: Runnable): CiStatsTestType {
@@ -60,13 +58,11 @@ export function setupCiStatsFtrTestGroupReporter({
   config,
   lifecycle,
   runner,
-  testMetadata,
   reporter,
 }: {
   config: Config;
   lifecycle: Lifecycle;
   runner: Runner;
-  testMetadata: TestMetadata;
   reporter: CiStatsReporter;
 }) {
   const testGroupType = process.env.TEST_GROUP_TYPE_FUNCTIONAL;
@@ -111,10 +107,6 @@ export function setupCiStatsFtrTestGroupReporter({
       type,
       error: error?.stack,
       stdout: getSnapshotOfRunnableLogs(runnable),
-      screenshots: testMetadata.getScreenshots(runnable).map((s) => ({
-        base64Png: s.base64Png,
-        name: s.name,
-      })),
     });
   }
 

packages/kbn-test/src/functional_test_runner/lib/mocha/reporter/reporter.js

@@ -24,7 +24,6 @@ export function MochaReporterProvider({ getService }) {
   const log = getService('log');
   const config = getService('config');
   const lifecycle = getService('lifecycle');
-  const testMetadata = getService('testMetadata');
   let originalLogWriters;
   let reporterCaptureStartTime;
 
@@ -61,7 +60,6 @@ export function MochaReporterProvider({ getService }) {
             config,
             lifecycle,
             runner,
-            testMetadata,
           });
         }
       }

packages/kbn-test/src/functional_test_runner/lib/snapshots/decorate_snapshot_ui.test.ts

@@ -6,11 +6,14 @@
  * Side Public License, v 1.
  */
 
+import path from 'path';
+import fs from 'fs';
+
+import { ToolingLog } from '@kbn/tooling-log';
+
 import { Suite, Test } from '../../fake_mocha_types';
 import { Lifecycle } from '../lifecycle';
 import { decorateSnapshotUi, expectSnapshot } from './decorate_snapshot_ui';
-import path from 'path';
-import fs from 'fs';
 
 const createRootSuite = () => {
   const suite = {
@@ -65,7 +68,7 @@ describe('decorateSnapshotUi', () => {
     let lifecycle: Lifecycle;
     let rootSuite: Suite;
     beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
       rootSuite = createRootSuite();
       decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: false });
 
@@ -116,7 +119,7 @@ describe('decorateSnapshotUi', () => {
     let lifecycle: Lifecycle;
     let rootSuite: Suite;
     beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
       rootSuite = createRootSuite();
       decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: false });
 
@@ -162,7 +165,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
     let lifecycle: Lifecycle;
     let rootSuite: Suite;
     beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
       rootSuite = createRootSuite();
       decorateSnapshotUi({ lifecycle, updateSnapshots: true, isCi: false });
 
@@ -185,7 +188,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
         fs.writeFileSync(
           snapshotFile,
           `// Jest Snapshot v1, https://goo.gl/fbAQLP
-  
+
   exports[\`Test 1\`] = \`"foo"\`;
         `,
           { encoding: 'utf-8' }
@@ -219,7 +222,7 @@ exports[\`Test2 1\`] = \`"bar"\`;
     let lifecycle: Lifecycle;
     let rootSuite: Suite;
     beforeEach(async () => {
-      lifecycle = new Lifecycle();
+      lifecycle = new Lifecycle(new ToolingLog());
       rootSuite = createRootSuite();
       decorateSnapshotUi({ lifecycle, updateSnapshots: false, isCi: true });
 

packages/kbn-test/src/functional_test_runner/lib/suite_tracker.test.ts

@@ -9,6 +9,8 @@
 import fs from 'fs';
 import { join, resolve } from 'path';
 
+import { ToolingLog } from '@kbn/tooling-log';
+
 jest.mock('fs');
 jest.mock('@kbn/utils', () => {
   return { REPO_ROOT: '/dev/null/root' };
@@ -60,7 +62,7 @@ describe('SuiteTracker', () => {
   };
 
   const runLifecycleWithMocks = async (mocks: Suite[], fn: (objs: any) => any = () => {}) => {
-    const lifecycle = new Lifecycle();
+    const lifecycle = new Lifecycle(new ToolingLog());
     const suiteTracker = SuiteTracker.startTracking(
       lifecycle,
       resolve(REPO_ROOT, MOCK_CONFIG_PATH)

packages/kbn-test/src/functional_test_runner/lib/test_metadata.ts

@@ -1,41 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { Lifecycle } from './lifecycle';
-
-export interface ScreenshotRecord {
-  name: string;
-  base64Png: string;
-  baselinePath?: string;
-  failurePath?: string;
-}
-
-export class TestMetadata {
-  // mocha's global types mean we can't import Mocha or it will override the global jest types..............
-  private currentRunnable?: any;
-
-  constructor(lifecycle: Lifecycle) {
-    lifecycle.beforeEachRunnable.add((runnable) => {
-      this.currentRunnable = runnable;
-    });
-  }
-
-  addScreenshot(screenshot: ScreenshotRecord) {
-    this.currentRunnable._screenshots = (this.currentRunnable._screenshots || []).concat(
-      screenshot
-    );
-  }
-
-  getScreenshots(test: any): ScreenshotRecord[] {
-    if (!test || typeof test !== 'object' || !test._screenshots) {
-      return [];
-    }
-
-    return test._screenshots.slice();
-  }
-}

packages/kbn-test/src/functional_test_runner/public_types.ts

@@ -8,10 +8,10 @@
 
 import type { ToolingLog } from '@kbn/tooling-log';
 
-import type { Config, Lifecycle, TestMetadata, DockerServersService, EsVersion } from './lib';
+import type { Config, Lifecycle, DockerServersService, EsVersion } from './lib';
 import type { Test, Suite } from './fake_mocha_types';
 
-export { Lifecycle, Config, TestMetadata };
+export { Lifecycle, Config };
 
 export interface AsyncInstance<T> {
   /**
@@ -56,9 +56,7 @@ export interface GenericFtrProviderContext<
    * Determine if a service is avaliable
    * @param serviceName
    */
-  hasService(
-    serviceName: 'config' | 'log' | 'lifecycle' | 'testMetadata' | 'dockerServers' | 'esVersion'
-  ): true;
+  hasService(serviceName: 'config' | 'log' | 'lifecycle' | 'dockerServers' | 'esVersion'): true;
   hasService<K extends keyof ServiceMap>(serviceName: K): serviceName is K;
   hasService(serviceName: string): serviceName is Extract<keyof ServiceMap, string>;
 
@@ -71,7 +69,6 @@ export interface GenericFtrProviderContext<
   getService(serviceName: 'log'): ToolingLog;
   getService(serviceName: 'lifecycle'): Lifecycle;
   getService(serviceName: 'dockerServers'): DockerServersService;
-  getService(serviceName: 'testMetadata'): TestMetadata;
   getService(serviceName: 'esVersion'): EsVersion;
   getService<T extends keyof ServiceMap>(serviceName: T): ServiceMap[T];
 

test/functional/services/common/screenshots.ts

@@ -22,7 +22,6 @@ const writeFileAsync = promisify(writeFile);
 export class ScreenshotsService extends FtrService {
   private readonly log = this.ctx.getService('log');
   private readonly config = this.ctx.getService('config');
-  private readonly testMetadata = this.ctx.getService('testMetadata');
   private readonly browser = this.ctx.getService('browser');
 
   private readonly SESSION_DIRECTORY = resolve(this.config.get('screenshots.directory'), 'session');
@@ -54,13 +53,7 @@ export class ScreenshotsService extends FtrService {
     const baselinePath = resolve(this.BASELINE_DIRECTORY, `${name}.png`);
     const failurePath = resolve(this.FAILURE_DIRECTORY, `${name}.png`);
 
-    await this.capture({
-      path: sessionPath,
-      name,
-      el,
-      baselinePath,
-      failurePath,
-    });
+    await this.capture(sessionPath, el);
 
     if (updateBaselines) {
       this.log.debug('Updating baseline snapshot');
@@ -82,42 +75,20 @@ export class ScreenshotsService extends FtrService {
 
   async take(name: string, el?: WebElementWrapper, subDirectories: string[] = []) {
     const path = resolve(this.SESSION_DIRECTORY, ...subDirectories, `${name}.png`);
-    await this.capture({ path, name, el });
+    await this.capture(path, el);
   }
 
   async takeForFailure(name: string, el?: WebElementWrapper) {
     const path = resolve(this.FAILURE_DIRECTORY, `${name}.png`);
-    await this.capture({
-      path,
-      name: `failure[${name}]`,
-      el,
-    });
+    await this.capture(path, el);
   }
 
-  private async capture({
-    path,
-    el,
-    name,
-    baselinePath,
-    failurePath,
-  }: {
-    path: string;
-    name: string;
-    el?: WebElementWrapper;
-    baselinePath?: string;
-    failurePath?: string;
-  }) {
+  private async capture(path: string, el?: WebElementWrapper) {
     try {
       this.log.info(`Taking screenshot "${path}"`);
       const screenshot = await (el ? el.takeScreenshot() : this.browser.takeScreenshot());
       await mkdirAsync(dirname(path), { recursive: true });
       await writeFileAsync(path, screenshot, 'base64');
-      this.testMetadata.addScreenshot({
-        name,
-        base64Png: Buffer.isBuffer(screenshot) ? screenshot.toString('base64') : screenshot,
-        baselinePath,
-        failurePath,
-      });
     } catch (err) {
       this.log.error('SCREENSHOT FAILED');
       this.log.error(err);