[7.17] Backport comment flaky results (#184336)

# Summary 
Backport of #183043

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/package-lock.json

@@ -8,7 +8,7 @@
       "name": "kibana-buildkite",
       "version": "1.0.0",
       "dependencies": {
-        "kibana-buildkite-library": "elastic/kibana-buildkite-library#1f0381a052a62da9af4e341db8eab9548fc8a4ea"
+        "kibana-buildkite-library": "elastic/kibana-buildkite-library#c469f23cf38b735ac93cbdcb6e2202d5b448aab1"
       }
     },
     "node_modules/@nodelib/fs.scandir": {
@@ -407,8 +407,8 @@
     },
     "node_modules/kibana-buildkite-library": {
       "version": "1.1.1",
-      "resolved": "git+ssh://git@github.com/elastic/kibana-buildkite-library.git#1f0381a052a62da9af4e341db8eab9548fc8a4ea",
-      "integrity": "sha512-wOwMxLa8i1S5A17MoJWnGJPTncNNyEi5F52QFT56JO9vEz1495dKr04NcS2brd1eIIHx1b+1Jeua8PYNswRvTQ==",
+      "resolved": "git+ssh://git@github.com/elastic/kibana-buildkite-library.git#c469f23cf38b735ac93cbdcb6e2202d5b448aab1",
+      "integrity": "sha512-VxEByq2JzH8qEk6pvWy0FDL5pNGIBquHA/r/U73aatT8gykS33aSruICbiGPmEb8fLUy9ogq/iYYTWBOYau6qA==",
       "dependencies": {
         "@octokit/rest": "^18.10.0",
         "axios": "^1.6.3",
@@ -931,9 +931,9 @@
       }
     },
     "kibana-buildkite-library": {
-      "version": "git+ssh://git@github.com/elastic/kibana-buildkite-library.git#1f0381a052a62da9af4e341db8eab9548fc8a4ea",
-      "integrity": "sha512-wOwMxLa8i1S5A17MoJWnGJPTncNNyEi5F52QFT56JO9vEz1495dKr04NcS2brd1eIIHx1b+1Jeua8PYNswRvTQ==",
-      "from": "kibana-buildkite-library@elastic/kibana-buildkite-library#1f0381a052a62da9af4e341db8eab9548fc8a4ea",
+      "version": "git+ssh://git@github.com/elastic/kibana-buildkite-library.git#c469f23cf38b735ac93cbdcb6e2202d5b448aab1",
+      "integrity": "sha512-VxEByq2JzH8qEk6pvWy0FDL5pNGIBquHA/r/U73aatT8gykS33aSruICbiGPmEb8fLUy9ogq/iYYTWBOYau6qA==",
+      "from": "kibana-buildkite-library@elastic/kibana-buildkite-library#c469f23cf38b735ac93cbdcb6e2202d5b448aab1",
       "requires": {
         "@octokit/rest": "^18.10.0",
         "axios": "^1.6.3",

.buildkite/package.json

@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "elastic/kibana-buildkite-library#1f0381a052a62da9af4e341db8eab9548fc8a4ea"
+    "kibana-buildkite-library": "elastic/kibana-buildkite-library#c469f23cf38b735ac93cbdcb6e2202d5b448aab1"
   }
 }

.buildkite/pipelines/flaky_tests/post_stats_on_pr.js

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { BuildkiteClient, getGithubClient } = require('kibana-buildkite-library');
+
+async function main() {
+  // Get buildkite build
+  const buildkite = new BuildkiteClient();
+  const buildkiteBuild = await buildkite.getBuild(
+    process.env.BUILDKITE_PIPELINE_SLUG,
+    process.env.BUILDKITE_BUILD_NUMBER
+  );
+  const buildLink = `[${buildkiteBuild.pipeline.slug}#${buildkiteBuild.number}](${buildkiteBuild.web_url})`;
+
+  // Calculate success metrics
+  const jobs = buildkiteBuild.jobs;
+  const testSuiteRuns = jobs.filter((step) => {
+    return step.step_key?.includes('test-group');
+  });
+
+  const testSuiteGroups = groupBy('name', testSuiteRuns);
+  const success = testSuiteRuns.every((job) => job.state === 'passed');
+  const testGroupResults = Object.entries(testSuiteGroups).map(([name, group]) => {
+    const passingTests = group.filter((job) => job.state === 'passed');
+    return {
+      name,
+      success: passingTests.length === group.length,
+      successCount: passingTests.length,
+      groupSize: group.length,
+    };
+  });
+
+  // Comment results on the PR
+  const prNumber = Number(extractPRNumberFromBranch(buildkiteBuild.branch));
+  if (isNaN(prNumber)) {
+    throw new Error(`Couldn't find PR number for build ${buildkiteBuild.web_url}.`);
+  }
+  const flakyRunHistoryLink = `https://buildkite.com/elastic/${
+    buildkiteBuild.pipeline.slug
+  }/builds?branch=${encodeURIComponent(buildkiteBuild.branch)}`;
+
+  const prComment = `
+## Flaky Test Runner Stats
+### ${success ? '🎉 All tests passed!' : '🟠 Some tests failed.'} - ${buildLink}
+${testGroupResults.map(formatTestGroupResult).join('\n')}
+
+[see run history](${flakyRunHistoryLink})
+`;
+
+  const githubClient = getGithubClient();
+  const commentResult = await githubClient.issues.createComment({
+    owner: 'elastic',
+    repo: 'kibana',
+    body: prComment,
+    issue_number: prNumber,
+  });
+
+  console.log(`Comment added: ${commentResult.data.html_url}`);
+}
+
+function formatTestGroupResult(result) {
+  const statusIcon = result.success ? '✅' : '❌';
+  const testName = result.name;
+  const successCount = result.successCount;
+  const groupSize = result.groupSize;
+
+  return `[${statusIcon}] ${testName}: ${successCount}/${groupSize} tests passed.`;
+}
+
+function groupBy(field, values) {
+  return values.reduce((acc, value) => {
+    const key = value[field];
+    if (typeof key !== 'string') {
+      throw new Error('Cannot group by non-string value field');
+    }
+
+    if (!acc[key]) {
+      acc[key] = [];
+    }
+    acc[key].push(value);
+    return acc;
+  }, {});
+}
+
+function extractPRNumberFromBranch(branch) {
+  if (!branch) {
+    return null;
+  } else {
+    return branch.match(/refs\/pull\/(\d+)\/head/)?.[1];
+  }
+}
+
+main()
+  .then(() => {
+    console.log('Flaky runner stats comment added to PR!');
+  })
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });

.buildkite/pipelines/flaky_tests/runner.js

@@ -142,6 +142,7 @@ steps.push({
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
 
+let suiteIndex = 0;
 for (const testSuite of testSuites) {
   const TEST_SUITE = testSuite.key;
   const RUN_COUNT = testSuite.count;
@@ -162,6 +163,7 @@ for (const testSuite of testSuites) {
         steps.push({
           command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/xpack_cigroup.sh`,
           label: `Default CI Group ${CI_GROUP}`,
+          key: `test-group-${suiteIndex++}`,
           agents: getAgentRule('n2-4'),
           depends_on: 'build',
           parallelism: RUN_COUNT,
@@ -173,6 +175,7 @@ for (const testSuite of testSuites) {
         steps.push({
           command: `CI_GROUP=${CI_GROUP} .buildkite/scripts/steps/functional/oss_cigroup.sh`,
           label: `OSS CI Group ${CI_GROUP}`,
+          key: `test-group-${suiteIndex++}`,
           agents: getAgentRule('n2-4-spot'),
           depends_on: 'build',
           parallelism: RUN_COUNT,
@@ -187,6 +190,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${IS_XPACK ? 'xpack' : 'oss'}_firefox.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Firefox`,
+        key: `test-group-${suiteIndex++}`,
         agents: getAgentRule(IS_XPACK ? 'n2-4' : 'n2-4-spot'),
         depends_on: 'build',
         parallelism: RUN_COUNT,
@@ -202,6 +206,7 @@ for (const testSuite of testSuites) {
           IS_XPACK ? 'xpack' : 'oss'
         }_accessibility.sh`,
         label: `${IS_XPACK ? 'Default' : 'OSS'} Accessibility`,
+        key: `test-group-${suiteIndex++}`,
         agents: getAgentRule(IS_XPACK ? 'n2-4' : 'n2-4-spot'),
         depends_on: 'build',
         parallelism: RUN_COUNT,
@@ -213,4 +218,20 @@ for (const testSuite of testSuites) {
   }
 }
 
+pipeline.steps.push({
+  wait: '~',
+  continue_on_failure: true,
+});
+
+pipeline.steps.push({
+  command: 'node .buildkite/pipelines/flaky_tests/post_stats_on_pr.js',
+  label: 'Post results on Github pull request',
+  agents: getAgentRule('n2-4-spot'),
+  timeout_in_minutes: 15,
+  retry: {
+    automatic: [{ exit_status: '-1', limit: 3 }],
+  },
+  soft_fail: true,
+});
+
 console.log(JSON.stringify(pipeline, null, 2));