[7.2] backport failed-test related fixes (#39133)

* [7.2] backport failed-test related fixes * restore trailing spaces in snapshots
2025-04-24 17:59:23 -04:00 · 2019-06-20 10:17:11 -07:00 · 2019-06-20 10:17:11 -07:00 · fbe9ecfab1
commit fbe9ecfab1
parent 5ea8947911
6 changed files with 274 additions and 8 deletions
--- a/src/dev/failed_tests/fixtures/ftr_report.xml
+++ b/src/dev/failed_tests/fixtures/ftr_report.xml
@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+  <testsuite timestamp="2019-06-05T23:37:10" time="903.670" tests="129" failures="5" skipped="71">
+    <testcase name="maps app  maps loaded from sample data ecommerce &quot;before all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js" time="154.378">
+      <system-out>
+        <![CDATA[[00:00:00]       │
+[00:07:04]         └-: maps app
+...
+[00:15:02]                   │
+]]>
+      </system-out>
+      <failure>
+        <![CDATA[Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~="layerTocActionsPanelToggleButtonRoad_Map_-_Bright"])
+Wait timed out after 10055ms
+    at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17
+    at process._tickCallback (internal/process/next_tick.js:68:7)
+    at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
+    at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)]]>
+      </failure>
+    </testcase>
+    <testcase name="maps app &quot;after all&quot; hook" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps" time="0.179">
+      <system-out>
+        <![CDATA[[00:00:00]       │
+[00:07:04]         └-: maps app
+...
+]]>
+      </system-out>
+      <failure>
+        <![CDATA[{ NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
+    at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
+    at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12)
+    at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }]]>
+      </failure>
+    </testcase>
+    <testcase name="InfraOps app feature controls infrastructure security global infrastructure all privileges shows infrastructure navlink" classname="Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/infra/feature_controls/infrastructure_security·ts">
+      <system-out>
+        <![CDATA[[00:00:00]       │
+[00:05:13]         └-: InfraOps app
+...
+]]>
+      </system-out>
+      <skipped/>
+    </testcase>
+  </testsuite>
+</testsuites>
--- a/src/dev/failed_tests/fixtures/jest_report.xml
+++ b/src/dev/failed_tests/fixtures/jest_report.xml
@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites name="jest" timestamp="2019-06-07T03:36:23" time="781.292" tests="5487" skipped="9">
+  <testsuite name="x-pack/plugins/code/server/lsp/abstract_launcher.test.ts" timestamp="2019-06-07T03:42:21" time="14.504" tests="5" failures="1" skipped="0" file="/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/plugins/code/server/lsp/abstract_launcher.test.ts">
+    <testcase classname="X-Pack Jest Tests.x-pack/plugins/code/server/lsp" name="launcher can start and end a process" time="1.316"/>
+    <testcase classname="X-Pack Jest Tests.x-pack/plugins/code/server/lsp" name="launcher can force kill the process if langServer can not exit" time="3.182"/>
+    <testcase classname="X-Pack Jest Tests.x-pack/plugins/code/server/lsp" name="launcher can reconnect if process died" time="7.060">
+      <failure>
+        <![CDATA[TypeError: Cannot read property '0' of undefined
+    at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)]]>
+      </failure>
+    </testcase>
+    <testcase classname="X-Pack Jest Tests.x-pack/plugins/code/server/lsp" name="passive launcher can start and end a process" time="0.435"/>
+    <testcase classname="X-Pack Jest Tests.x-pack/plugins/code/server/lsp" name="passive launcher should restart a process if a process died before connected" time="1.502"/>
+  </testsuite>
+</testsuites>
--- a/src/dev/failed_tests/fixtures/mocha_report.xml
+++ b/src/dev/failed_tests/fixtures/mocha_report.xml
@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+  <testsuite timestamp="2019-06-13T23:29:36" time="30.739" tests="1444" failures="2" skipped="3">
+    <testcase name="code in multiple nodes &quot;before all&quot; hook" classname="X-Pack Mocha Tests.x-pack/plugins/code/server/__tests__/multi_node·ts" time="0.121">
+      <system-out>
+        <![CDATA[]]>
+      </system-out>
+      <failure>
+        <![CDATA[Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
+  <html>
+<head><title>503 Service Temporarily Unavailable</title></head>
+<body bgcolor="white">
+<center><h1>503 Service Temporarily Unavailable</h1></center>
+<hr><center>nginx/1.13.7</center>
+</body>
+</html>
+
+    at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13)
+    at process._tickCallback (internal/process/next_tick.js:68:7)]]>
+      </failure>
+    </testcase>
+    <testcase name="code in multiple nodes &quot;after all&quot; hook" classname="X-Pack Mocha Tests.x-pack/plugins/code/server/__tests__/multi_node·ts" time="0.003">
+      <system-out>
+        <![CDATA[]]>
+      </system-out>
+      <failure>
+        <![CDATA[TypeError: Cannot read property 'shutdown' of undefined
+    at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
+    at process.topLevelDomainCallback (domain.js:120:23)]]>
+      </failure>
+    </testcase>
+    <testcase name="repository service test can not clone a repo by ssh without a key" classname="X-Pack Mocha Tests.x-pack/plugins/code/server/__tests__/repository_service·ts" time="0.005">
+      <system-out>
+        <![CDATA[]]>
+      </system-out>
+    </testcase>
+  </testsuite>
+</testsuites>
--- a/src/dev/failed_tests/cli.js
+++ b/src/dev/failed_tests/cli.js
@ -17,15 +17,21 @@
 * under the License.
 */

+const { resolve } = require('path');
+
+// force cwd
+process.chdir(resolve(__dirname, '../../..'));
+
 if (!process.env.JOB_NAME) {
  console.log('Unable to determine job name');
  process.exit(1);
 }

-const [org, proj, branch] = process.env.JOB_NAME.split('+');
+// JOB_NAME is formatted as `elastic+kibana+7.x` in some places and `elastic+kibana+7.x/JOB=kibana-intake,node=immutable` in others
+const [org, proj, branch] = process.env.JOB_NAME.split(/\+|\//);
 const masterOrVersion = branch === 'master' || branch.match(/^\d+\.(x|\d+)$/);
 if (!(org === 'elastic' && proj === 'kibana' && masterOrVersion)) {
-  console.log('Failure issues only created on master/version branch jobs');
+  console.log(`Failure issues only created on master/version branch jobs [JOB_NAME=${process.env.JOB_NAME}] [${org}/${proj}/${branch}]`);
  process.exit(0);
 }

--- a/src/dev/failed_tests/report.js
+++ b/src/dev/failed_tests/report.js
@ -33,8 +33,8 @@ const indent = text => (
  `  ${text.split('\n').map(l => `  ${l}`).join('\n')}`
 );

-const isLikelyIrrelevant = ({ failure }) => {
-  if (failure.include('NoSuchSessionError: This driver instance does not have a valid session ID')) {
+const isLikelyIrrelevant = ({ name, failure }) => {
+  if (failure.includes('NoSuchSessionError: This driver instance does not have a valid session ID')) {
    return true;
  }

@ -42,6 +42,14 @@ const isLikelyIrrelevant = ({ failure }) => {
    return true;
  }

+  if (name.includes('"after all" hook') && failure.includes(`Cannot read property 'shutdown' of undefined`)) {
+    return true;
+  }
+
+  if (failure.includes('Unable to read artifact info') && failure.includes('Service Temporarily Unavailable')) {
+    return true;
+  }
+
  if (failure.includes('Unable to fetch Kibana status API response from Kibana')) {
    return true;
  }
@ -50,7 +58,7 @@ const isLikelyIrrelevant = ({ failure }) => {
 /**
 * Parses junit XML files into JSON
 */
-const mapXml = createMapStream((file) => new Promise((resolve, reject) => {
+export const mapXml = () => createMapStream((file) => new Promise((resolve, reject) => {
  xml2js.parseString(file.contents.toString(), (err, result) => {
    if (err) {
      return reject(err);
@ -62,7 +70,7 @@ const mapXml = createMapStream((file) => new Promise((resolve, reject) => {
 /**
 * Filters all testsuites to find failed testcases
 */
-const filterFailures = createMapStream((testSuite) => {
+export const filterFailures = () => createMapStream((testSuite) => {
  // Grab the failures. Reporters may report multiple testsuites in a single file.
  const testFiles = testSuite.testsuites
    ? testSuite.testsuites.testsuite
@ -172,8 +180,8 @@ export async function reportFailedTests() {

  vfs
    .src(['./target/junit/**/*.xml'])
-    .pipe(mapXml)
-    .pipe(filterFailures)
+    .pipe(mapXml())
+    .pipe(filterFailures())
    .pipe(updateGithubIssues(githubClient, issues))
    .on('done', () => console.log(`Finished reporting test failures.`));
 }
--- a/src/dev/failed_tests/report.test.js
+++ b/src/dev/failed_tests/report.test.js
@ -0,0 +1,154 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* eslint-disable max-len */
+
+import { resolve } from 'path';
+
+import vfs from 'vinyl-fs';
+
+import { mapXml, filterFailures } from './report';
+import { createPromiseFromStreams } from '../../legacy/utils/streams/promise_from_streams';
+import { createConcatStream } from '../../legacy/utils/streams/concat_stream';
+
+console.log = jest.fn();
+afterEach(() => jest.resetAllMocks());
+
+describe('irrelevant failure filtering', () => {
+  describe('jest report', () => {
+    it('allows relevant tests', async () => {
+      const failures = await createPromiseFromStreams([
+        vfs.src([resolve(__dirname, '__fixtures__/jest_report.xml')]),
+        mapXml(),
+        filterFailures(),
+        createConcatStream(),
+      ]);
+
+      expect(console.log.mock.calls).toMatchInlineSnapshot(`
+Array [
+  Array [
+    "Found 1 test failures",
+  ],
+]
+`);
+      expect(failures).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "classname": "X-Pack Jest Tests.x-pack/plugins/code/server/lsp",
+    "failure": "
+        TypeError: Cannot read property '0' of undefined
+    at Object.<anonymous>.test (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/x-pack/plugins/code/server/lsp/abstract_launcher.test.ts:166:10)
+      ",
+    "name": "launcher can reconnect if process died",
+    "time": "7.060",
+  },
+]
+`);
+    });
+  });
+
+  describe('ftr report', () => {
+    it('allows relevant tests', async () => {
+      const failures = await createPromiseFromStreams([
+        vfs.src([resolve(__dirname, '__fixtures__/ftr_report.xml')]),
+        mapXml(),
+        filterFailures(),
+        createConcatStream(),
+      ]);
+
+      expect(console.log.mock.calls).toMatchInlineSnapshot(`
+Array [
+  Array [
+    "Ignoring likely irrelevant failure: Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps - maps app \\"after all\\" hook
+    
+          { NoSuchSessionError: This driver instance does not have a valid session ID (did you call WebDriver.quit()?) and may no longer be used.
+      at promise.finally (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:726:38)
+      at Object.thenFinally [as finally] (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/promise.js:124:12)
+      at process._tickCallback (internal/process/next_tick.js:68:7) name: 'NoSuchSessionError', remoteStacktrace: '' }
+        ",
+  ],
+  Array [
+    "Found 1 test failures",
+  ],
+]
+`);
+      expect(failures).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "classname": "Chrome X-Pack UI Functional Tests.x-pack/test/functional/apps/maps/sample_data·js",
+    "failure": "
+        Error: retry.try timeout: TimeoutError: Waiting for element to be located By(css selector, [data-test-subj~=\\"layerTocActionsPanelToggleButtonRoad_Map_-_Bright\\"])
+Wait timed out after 10055ms
+    at /var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/node_modules/selenium-webdriver/lib/webdriver.js:834:17
+    at process._tickCallback (internal/process/next_tick.js:68:7)
+    at lastError (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:28:9)
+    at onFailure (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-ciGroup7/node/immutable/kibana/test/common/services/retry/retry_for_success.ts:68:13)
+      ",
+    "name": "maps app  maps loaded from sample data ecommerce \\"before all\\" hook",
+    "time": "154.378",
+  },
+]
+`);
+    });
+  });
+
+  describe('mocha report', () => {
+    it('allows relevant tests', async () => {
+      const failures = await createPromiseFromStreams([
+        vfs.src([resolve(__dirname, '__fixtures__/mocha_report.xml')]),
+        mapXml(),
+        filterFailures(),
+        createConcatStream(),
+      ]);
+
+      expect(console.log.mock.calls).toMatchInlineSnapshot(`
+Array [
+  Array [
+    "Ignoring likely irrelevant failure: X-Pack Mocha Tests.x-pack/plugins/code/server/__tests__/multi_node·ts - code in multiple nodes \\"before all\\" hook
+    
+          Error: Unable to read artifact info from https://artifacts-api.elastic.co/v1/versions/8.0.0-SNAPSHOT/builds/latest/projects/elasticsearch: Service Temporarily Unavailable
+    <html>
+  <head><title>503 Service Temporarily Unavailable</title></head>
+  <body bgcolor=\\"white\\">
+  <center><h1>503 Service Temporarily Unavailable</h1></center>
+  <hr><center>nginx/1.13.7</center>
+  </body>
+  </html>
+  
+      at Function.getSnapshot (/var/lib/jenkins/workspace/elastic+kibana+master/JOB/x-pack-intake/node/immutable/kibana/packages/kbn-es/src/artifact.js:95:13)
+      at process._tickCallback (internal/process/next_tick.js:68:7)
+        ",
+  ],
+  Array [
+    "Ignoring likely irrelevant failure: X-Pack Mocha Tests.x-pack/plugins/code/server/__tests__/multi_node·ts - code in multiple nodes \\"after all\\" hook
+    
+          TypeError: Cannot read property 'shutdown' of undefined
+      at Context.shutdown (plugins/code/server/__tests__/multi_node.ts:125:23)
+      at process.topLevelDomainCallback (domain.js:120:23)
+        ",
+  ],
+  Array [
+    "Found 0 test failures",
+  ],
+]
+`);
+      expect(failures).toMatchInlineSnapshot(`Array []`);
+    });
+  });
+});