[6.x] Add jenkins:report task for test failures (#22682) (#23659)

* Add jenkins:report task for test failures (#22682) * Add jenkins:report task * PR comments * Handle different junit XML formats (#23617)
2025-04-23 17:28:26 -04:00 · 2018-10-01 14:52:48 -05:00 · 2018-10-01 14:52:48 -05:00 · 78b2159f33
commit 78b2159f33
parent 632c63ab21
7 changed files with 353 additions and 1 deletions
--- a/package.json
+++ b/package.json
@ -228,6 +228,7 @@
    "@kbn/eslint-plugin-license-header": "link:packages/kbn-eslint-plugin-license-header",
    "@kbn/plugin-generator": "link:packages/kbn-plugin-generator",
    "@kbn/test": "link:packages/kbn-test",
+    "@octokit/rest": "^15.10.0",
    "@types/angular": "^1.6.45",
    "@types/babel-core": "^6.25.5",
    "@types/bluebird": "^3.1.1",
--- a/src/dev/failed_tests/report.js
+++ b/src/dev/failed_tests/report.js
@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import xml2js from 'xml2js';
+import vfs from 'vinyl-fs';
+import es from 'event-stream';
+import { getGithubClient, markdownMetadata, paginate } from '../github_utils';
+import { find } from 'lodash';
+import stripAnsi from 'strip-ansi';
+
+const GITHUB_FLAKY_TEST_LABEL = 'failed-test';
+const GITHUB_OWNER = 'elastic';
+const GITHUB_REPO = 'kibana';
+const BUILD_URL = process.env.BUILD_URL;
+
+/**
+ * Parses junit XML files into JSON
+ */
+const mapXml = es.map((file, cb) => {
+  xml2js.parseString(file.contents.toString(), (err, result) => {
+    cb(null, result);
+  });
+});
+
+/**
+ * Filters all testsuites to find failed testcases
+ */
+const filterFailures = es.map((testSuite, cb) => {
+  // Grab the failures. Reporters may report multiple testsuites in a single file.
+  const testFiles = testSuite.testsuites
+    ? testSuite.testsuites.testsuite
+    : [testSuite.testsuite];
+
+  const failures = testFiles.reduce((failures, testFile) => {
+    for (const testCase of testFile.testcase) {
+      if (testCase.failure) {
+        // unwrap xml weirdness
+        failures.push({
+          ...testCase.$,
+          // Strip ANSI color characters
+          failure: stripAnsi(testCase.failure[0])
+        });
+      }
+    }
+
+    return failures;
+  }, []);
+
+  console.log(`Found ${failures.length} test failures`);
+
+  cb(null, failures);
+});
+
+/**
+ * Creates and updates github issues for the given testcase failures.
+ */
+const updateGithubIssues = (githubClient, issues) => {
+  return es.map(async (failureCases, cb) => {
+
+    const issueOps = failureCases.map(async (failureCase) => {
+      const existingIssue = find(issues, (issue) => {
+        return markdownMetadata.get(issue.body, 'test.class') === failureCase.classname &&
+          markdownMetadata.get(issue.body, 'test.name') === failureCase.name;
+      });
+
+      if (existingIssue) {
+        // Increment failCount
+        const newCount = (markdownMetadata.get(existingIssue.body, 'test.failCount') || 0) + 1;
+        const newBody = markdownMetadata.set(existingIssue.body, 'test.failCount', newCount);
+
+        await githubClient.issues.edit({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          number: existingIssue.number,
+          state: 'open',  // Reopen issue if it was closed.
+          body: newBody
+        });
+
+        // Append a new comment
+        await githubClient.issues.createComment({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          number: existingIssue.number,
+          body: `New failure: [Jenkins Build](${BUILD_URL})`
+        });
+
+        console.log(`Updated issue ${existingIssue.html_url}, failCount: ${newCount}`);
+      } else {
+        let body = 'A test failed on a tracked branch\n' +
+          '```\n' + failureCase.failure + '\n```\n' +
+          `First failure: [Jenkins Build](${BUILD_URL})`;
+        body = markdownMetadata.set(body, {
+          'test.class': failureCase.classname,
+          'test.name': failureCase.name,
+          'test.failCount': 1
+        });
+
+        const newIssue = await githubClient.issues.create({
+          owner: GITHUB_OWNER,
+          repo: GITHUB_REPO,
+          title: `Failing test: ${failureCase.classname} - ${failureCase.name}`,
+          body: body,
+          labels: [GITHUB_FLAKY_TEST_LABEL]
+        });
+
+        console.log(`Created issue ${newIssue.data.html_url}`);
+      }
+    });
+
+    Promise
+      .all(issueOps)
+      .then(() => cb(null, failureCases))
+      .catch(e => cb(e));
+  });
+};
+
+/**
+ * Scans all junit XML files in ./target/junit/ and reports any found test failures to Github Issues.
+ */
+export async function reportFailedTests(done) {
+  const githubClient = getGithubClient();
+  const issues = await paginate(githubClient, githubClient.issues.getForRepo({
+    owner: GITHUB_OWNER,
+    repo: GITHUB_REPO,
+    labels: GITHUB_FLAKY_TEST_LABEL,
+    state: 'all',
+    per_page: 100
+  }));
+
+  vfs
+    .src(['./target/junit/**/*.xml'])
+    .pipe(mapXml)
+    .pipe(filterFailures)
+    .pipe(updateGithubIssues(githubClient, issues))
+    .on('done', done);
+}
--- a/src/dev/github_utils/index.js
+++ b/src/dev/github_utils/index.js
@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Octokit from '@octokit/rest';
+import { markdownMetadata } from './metadata';
+
+export { markdownMetadata };
+
+export function getGithubClient() {
+  const client = new Octokit();
+  client.authenticate({
+    type: 'token',
+    token: process.env.GITHUB_TOKEN
+  });
+
+  return client;
+}
+
+export async function paginate(client, promise) {
+  let response = await promise;
+  let { data } = response;
+  while (client.hasNextPage(response)) {
+    response = await client.getNextPage(response);
+    data = data.concat(response.data);
+  }
+  return data;
+}
--- a/src/dev/github_utils/metadata.js
+++ b/src/dev/github_utils/metadata.js
@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const REGEX = /\n\n<!-- kibanaCiData = (.*) -->/;
+
+/**
+ * Allows retrieving and setting key/value pairs on a Github Issue. Keys and values must be JSON-serializable.
+ * Derived from https://github.com/probot/metadata/blob/6ae1523d5035ba727d09c0e7f77a6a154d9a4777/index.js
+ *
+ * `body` is a string that contains markdown and any existing metadata (eg. an issue or comment body)
+ * `prefix` is a string that can be used to namespace the metadata, defaults to `ci`.
+ *
+ *
+ * @notice
+ * This product bundles code based on probot-metadata@1.0.0 which is
+ * available under a "MIT" license.
+ *
+ * ISC License
+ *
+ * Copyright (c) 2017 Brandon Keepers
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+export const markdownMetadata = {
+  get(body, key = null, prefix = 'failed-test') {
+    const match = body.match(REGEX);
+
+    if (match) {
+      const data = JSON.parse(match[1])[prefix];
+      return key ? data && data[key] : data;
+    } else {
+      return null;
+    }
+  },
+
+  /**
+   * Set data on the body. Can either be set individually with `key` and `value` OR
+   */
+  set(body, key, value, prefix = 'failed-test') {
+    let newData = {};
+    // If second arg is an object, use all supplied values.
+    if (typeof key === 'object') {
+      newData = key;
+      prefix = value || prefix;  // need to move third arg to prefix.
+    } else {
+      newData[key] = value;
+    }
+
+    let data = {};
+
+    body = body.replace(REGEX, (_, json) => {
+      data = JSON.parse(json);
+      return '';
+    });
+
+    if (!data[prefix]) data[prefix] = {};
+
+    Object.assign(data[prefix], newData);
+
+    return `${body}\n\n<!-- kibanaCiData = ${JSON.stringify(data)} -->`;
+  }
+};
--- a/tasks/jenkins.js
+++ b/tasks/jenkins.js
@ -17,6 +17,8 @@
 * under the License.
 */

+import { reportFailedTests } from '../src/dev/failed_tests/report';
+
 module.exports = function (grunt) {
  grunt.registerTask('jenkins:docs', [
    'docker:docs'
@ -44,4 +46,12 @@ module.exports = function (grunt) {
    'run:functionalTestsRelease',
    'run:pluginFunctionalTestsRelease',
  ]);
+
+  grunt.registerTask(
+    'jenkins:report',
+    'Reports failed tests found in junit xml files to Github issues',
+    function () {
+      reportFailedTests(this.async());
+    }
+  );
 };
--- a/test/scripts/jenkins_report_failed_tests.sh
+++ b/test/scripts/jenkins_report_failed_tests.sh
@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+xvfb-run "$(FORCE_COLOR=0 yarn bin)/grunt" jenkins:report;
--- a/yarn.lock
+++ b/yarn.lock
@ -215,6 +215,20 @@
    call-me-maybe "^1.0.1"
    glob-to-regexp "^0.3.0"

+"@octokit/rest@^15.10.0":
+  version "15.12.1"
+  resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-15.12.1.tgz#f0dc169f336f8fd05ae8d8c83371b2f0c0dd8de1"
+  dependencies:
+    before-after-hook "^1.1.0"
+    btoa-lite "^1.0.0"
+    debug "^3.1.0"
+    http-proxy-agent "^2.1.0"
+    https-proxy-agent "^2.2.0"
+    lodash "^4.17.4"
+    node-fetch "^2.1.1"
+    universal-user-agent "^2.0.0"
+    url-template "^2.0.8"
+
 "@samverschueren/stream-to-observable@^0.3.0":
  version "0.3.0"
  resolved "https://registry.yarnpkg.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.0.tgz#ecdf48d532c58ea477acfcab80348424f8d0662f"
@ -1997,6 +2011,10 @@ bcrypt-pbkdf@^1.0.0:
  dependencies:
    tweetnacl "^0.14.3"

+before-after-hook@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-1.1.0.tgz#83165e15a59460d13702cb8febd6a1807896db5a"
+
 better-assert@~1.0.0:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/better-assert/-/better-assert-1.0.2.tgz#40866b9e1b9e0b55b481894311e68faffaebc522"
@ -2314,6 +2332,10 @@ bser@^2.0.0:
  dependencies:
    node-int64 "^0.4.0"

+btoa-lite@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337"
+
 buffer-crc32@~0.2.3:
  version "0.2.13"
  resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
@ -6528,7 +6550,7 @@ https-browserify@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73"

-https-proxy-agent@2.2.1, https-proxy-agent@^2.2.1:
+https-proxy-agent@2.2.1, https-proxy-agent@^2.2.0, https-proxy-agent@^2.2.1:
  version "2.2.1"
  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0"
  dependencies:
@ -8835,6 +8857,10 @@ macaddress@^0.2.8:
  version "0.2.9"
  resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.9.tgz#3579b8b9acd5b96b4553abf0f394185a86813cb3"

+macos-release@^1.0.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/macos-release/-/macos-release-1.1.0.tgz#831945e29365b470aa8724b0ab36c8f8959d10fb"
+
 magic-string@^0.22.4:
  version "0.22.5"
  resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.22.5.tgz#8e9cf5afddf44385c1da5bc2a6a0dbd10b03657e"
@ -9399,6 +9425,10 @@ node-fetch@^2.0.0:
  version "2.1.1"
  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.1.1.tgz#369ca70b82f50c86496104a6c776d274f4e4a2d4"

+node-fetch@^2.1.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.2.0.tgz#4ee79bde909262f9775f731e3656d0db55ced5b5"
+
 node-gyp@^3.3.1:
  version "3.6.2"
  resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.6.2.tgz#9bfbe54562286284838e750eac05295853fa1c60"
@ -9830,6 +9860,13 @@ os-locale@^2.0.0:
    lcid "^1.0.0"
    mem "^1.1.0"

+os-name@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/os-name/-/os-name-2.0.1.tgz#b9a386361c17ae3a21736ef0599405c9a8c5dc5e"
+  dependencies:
+    macos-release "^1.0.0"
+    win-release "^1.0.0"
+
 os-tmpdir@^1.0.0, os-tmpdir@^1.0.1, os-tmpdir@~1.0.1, os-tmpdir@~1.0.2:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274"
@ -13896,6 +13933,12 @@ unist-util-visit@^1.1.0, unist-util-visit@^1.3.0:
  dependencies:
    unist-util-is "^2.1.1"

+universal-user-agent@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-2.0.1.tgz#18e591ca52b1cb804f6b9cbc4c336cf8191f80e1"
+  dependencies:
+    os-name "^2.0.1"
+
 universalify@^0.1.0:
  version "0.1.1"
  resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7"
@ -13992,6 +14035,10 @@ url-regex@^3.0.0:
  dependencies:
    ip-regex "^1.0.1"

+url-template@^2.0.8:
+  version "2.0.8"
+  resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21"
+
 url-to-options@^1.0.1:
  version "1.0.1"
  resolved "https://registry.yarnpkg.com/url-to-options/-/url-to-options-1.0.1.tgz#1505a03a289a48cbd7a434efbaeec5055f5633a9"
@ -14627,6 +14674,12 @@ widest-line@^2.0.0:
  dependencies:
    string-width "^2.1.1"

+win-release@^1.0.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/win-release/-/win-release-1.1.1.tgz#5fa55e02be7ca934edfc12665632e849b72e5209"
+  dependencies:
+    semver "^5.0.1"
+
 window-size@0.1.0:
  version "0.1.0"
  resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"