[6.x] [dev/mocha/junit] report real skipped test count and errors from hooks (#15465) (#15475)

* [dev/mocha/junit] report real skipped test count and errors from hooks

* typo

package.json

@@ -295,6 +295,7 @@
     "supertest-as-promised": "2.0.2",
     "tree-kill": "1.1.0",
     "webpack-dev-server": "2.9.1",
+    "xml2js": "0.4.19",
     "xmlbuilder": "9.0.4",
     "yeoman-generator": "1.1.1",
     "yo": "2.0.0"

src/dev/mocha/__tests__/fixtures/project/test.js

@@ -0,0 +1,15 @@
+describe('SUITE', () => {
+  it('works', () => {});
+  it('fails', () => {
+    throw new Error('FORCE_TEST_FAIL');
+  });
+
+  describe('SUB_SUITE', () => {
+    beforeEach('success hook', () => {});
+    beforeEach('fail hook', () => {
+      throw new Error('FORCE_HOOK_FAIL');
+    });
+
+    it('never runs', () => {});
+  });
+});

src/dev/mocha/__tests__/junit_report_generation.js

@@ -0,0 +1,115 @@
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+
+import { fromNode as fcb } from 'bluebird';
+import { parseString } from 'xml2js';
+import del from 'del';
+import Mocha from 'mocha';
+import expect from 'expect.js';
+
+import { setupJunitReportGeneration } from '../junit_report_generation';
+
+const PROJECT_DIR = resolve(__dirname, 'fixtures/project');
+const DURATION_REGEX = /^\d+\.\d{3}$/;
+const ISO_DATE_SEC_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}$/;
+
+describe('dev/mocha/junit report generation', () => {
+  afterEach(() => {
+    del.sync(resolve(PROJECT_DIR, 'target'));
+  });
+
+  it('reports on failed setup hooks', async () => {
+    const mocha = new Mocha({
+      reporter: function Runner(runner) {
+        setupJunitReportGeneration(runner, {
+          reportName: 'test',
+          rootDirectory: PROJECT_DIR
+        });
+      }
+    });
+
+    mocha.addFile(resolve(PROJECT_DIR, 'test.js'));
+    await new Promise(resolve => mocha.run(resolve));
+    const report = await fcb(cb => parseString(readFileSync(resolve(PROJECT_DIR, 'target/junit/test.xml')), cb));
+
+    // test case results are wrapped in <testsuites></testsuites>
+    expect(report).to.eql({
+      testsuites: {
+        testsuite: [
+          report.testsuites.testsuite[0]
+        ]
+      }
+    });
+
+    // the single <testsuite> element at the root contains summary data for all tests results
+    const [ testsuite ] = report.testsuites.testsuite;
+    expect(testsuite.$.time).to.match(DURATION_REGEX);
+    expect(testsuite.$.timestamp).to.match(ISO_DATE_SEC_REGEX);
+    expect(testsuite).to.eql({
+      $: {
+        failures: '2',
+        skipped: '1',
+        tests: '4',
+        time: testsuite.$.time,
+        timestamp: testsuite.$.timestamp,
+      },
+      testcase: testsuite.testcase,
+    });
+
+    // there are actually only three tests, but since the hook failed
+    // it is reported as a test failure
+    expect(testsuite.testcase).to.have.length(4);
+    const [
+      testPass,
+      testFail,
+      beforeEachFail,
+      testSkipped,
+    ] = testsuite.testcase;
+
+    const sharedClassname = testPass.$.classname;
+    expect(sharedClassname).to.match(/^test\.test[^\.]js$/);
+    expect(testPass.$.time).to.match(DURATION_REGEX);
+    expect(testPass).to.eql({
+      $: {
+        classname: sharedClassname,
+        name: 'SUITE works',
+        time: testPass.$.time,
+      }
+    });
+
+    expect(testFail.$.time).to.match(DURATION_REGEX);
+    expect(testFail.failure[0]).to.match(/Error: FORCE_TEST_FAIL\n.+fixtures.project.test.js/);
+    expect(testFail).to.eql({
+      $: {
+        classname: sharedClassname,
+        name: 'SUITE fails',
+        time: testFail.$.time,
+      },
+      failure: [
+        testFail.failure[0]
+      ]
+    });
+
+    expect(beforeEachFail.$.time).to.match(DURATION_REGEX);
+    expect(beforeEachFail.failure).to.have.length(1);
+    expect(beforeEachFail.failure[0]).to.match(/Error: FORCE_HOOK_FAIL\n.+fixtures.project.test.js/);
+    expect(beforeEachFail).to.eql({
+      $: {
+        classname: sharedClassname,
+        name: 'SUITE SUB_SUITE "before each" hook: fail hook for "never runs"',
+        time: beforeEachFail.$.time,
+      },
+      failure: [
+        beforeEachFail.failure[0]
+      ]
+    });
+
+    expect(testSkipped).to.eql({
+      $: {
+        classname: sharedClassname,
+        name: 'SUITE SUB_SUITE never runs',
+      },
+      skipped: ['']
+    });
+  });
+});

src/dev/mocha/junit_report_generation.js

@@ -11,10 +11,8 @@ export function setupJunitReportGeneration(runner, options = {}) {
     rootDirectory = dirname(require.resolve('../../../package.json')),
   } = options;
 
-  const rootSuite = runner.suite;
-  const isTestFailed = test => test.state === 'failed';
-  const isTestPending = test => !!test.pending;
-  const returnTrue = () => true;
+  const stats = {};
+  const results = [];
 
   const getDuration = (node) => (
     node.startTime && node.endTime
@@ -22,17 +20,17 @@ export function setupJunitReportGeneration(runner, options = {}) {
       : null
   );
 
-  const getTimestamp = (node) => (
-    node.startTime
-      ? new Date(node.startTime).toISOString().slice(0, -5)
-      : null
+  const findAllTests = (suite) => (
+    suite.suites.reduce((acc, suite) => acc.concat(findAllTests(suite)), suite.tests)
   );
 
-  const countTests = (suite, filter = returnTrue) => (
-    suite.suites.reduce((sum, suite) => (
-      sum + countTests(suite, filter)
-    ), suite.tests.filter(filter).length)
-  );
+  const setStartTime = (node) => {
+    node.startTime = Date.now();
+  };
+
+  const setEndTime = node => {
+    node.endTime = Date.now();
+  };
 
   const getFullTitle = node => {
     const parentTitle = node.parent && getFullTitle(node.parent);
@@ -51,28 +49,33 @@ export function setupJunitReportGeneration(runner, options = {}) {
     return 'unknown';
   };
 
-  runner.on('start', () => {
-    rootSuite.startTime = Date.now();
-  });
-
-  runner.on('suite', (suite) => {
-    suite.startTime = Date.now();
-  });
-
-  runner.on('test', (test) => {
-    test.startTime = Date.now();
-  });
-
-  runner.on('test end', (test) => {
-    test.endTime = Date.now();
-  });
-
-  runner.on('suite end', (suite) => {
-    suite.endTime = Date.now();
-  });
+  runner.on('start', () => setStartTime(stats));
+  runner.on('suite', setStartTime);
+  runner.on('hook', setStartTime);
+  runner.on('hook end', setEndTime);
+  runner.on('test', setStartTime);
+  runner.on('pass', (node) => results.push({ node }));
+  runner.on('pass', setEndTime);
+  runner.on('fail', (node, error) => results.push({ failed: true, error, node }));
+  runner.on('fail', setEndTime);
+  runner.on('suite end', () => setEndTime(stats));
 
   runner.on('end', () => {
-    rootSuite.endTime = Date.now();
+    // crawl the test graph to collect all defined tests
+    const allTests = findAllTests(runner.suite);
+
+    // filter out just the failures
+    const failures = results.filter(result => result.failed);
+
+    // any failure that isn't for a test is for a hook
+    const failedHooks = failures.filter(result => !allTests.includes(result.node));
+
+    // mocha doesn't emit 'pass' or 'fail' when it skips a test
+    // or a test is pending, so we find them ourselves
+    const skippedResults = allTests
+      .filter(node => node.pending || !results.find(result => result.node === node))
+      .map(node => ({ skipped: true, node }));
+
     const builder = xmlBuilder.create(
       'testsuites',
       { encoding: 'utf-8' },
@@ -80,47 +83,34 @@ export function setupJunitReportGeneration(runner, options = {}) {
       { skipNullAttributes: true }
     );
 
-    function addSuite(parent, suite) {
-      const attributes = {
-        name: suite.title,
-        timestamp: getTimestamp(suite),
-        time: getDuration(suite),
-        tests: countTests(suite),
-        failures: countTests(suite, isTestFailed),
-        skipped: countTests(suite, isTestPending),
-        file: suite.file
-      };
+    const testsuitesEl = builder.ele('testsuite', {
+      timestamp: new Date(stats.startTime).toISOString().slice(0, -5),
+      time: getDuration(stats),
+      tests: allTests.length + failedHooks.length,
+      failures: failures.length,
+      skipped: skippedResults.length,
+    });
 
-      const el = suite === rootSuite
-        ? parent.att(attributes)
-        : parent.ele('testsuite', attributes);
-
-      suite.suites.forEach(childSuite => {
-        addSuite(el, childSuite);
-      });
-
-      suite.tests.forEach(test => {
-        addTest(el, test);
+    function addTestcaseEl(node) {
+      return testsuitesEl.ele('testcase', {
+        name: getFullTitle(node),
+        classname: `${reportName}.${getPath(node).replace(/\./g, '·')}`,
+        time: getDuration(node),
       });
     }
 
-    function addTest(parent, test) {
-      const el = parent.ele('testcase', {
-        name: getFullTitle(test),
-        classname: `${reportName}.${getPath(test).replace(/\./g, '·')}`,
-        time: getDuration(test),
-      });
+    [...results, ...skippedResults].forEach(result => {
+      const el = addTestcaseEl(result.node);
 
-      if (isTestFailed(test)) {
-        el
-          .ele('failure')
-          .dat(inspect(test.err));
-      } else if (isTestPending(test)) {
+      if (result.failed) {
+        el.ele('failure').dat(inspect(result.error));
+        return;
+      }
+
+      if (result.skipped) {
         el.ele('skipped');
       }
-    }
-
-    addSuite(builder, rootSuite);
+    });
 
     const reportPath = resolve(rootDirectory, `target/junit/${reportName}.xml`);
     const reportXML = builder.end({