[CI] Comment flaky test results on tested PR (#183043)

## Summary
Extends the flaky-test-runner with the capability to comment on the
flaky test runs on the PR that's being tested.

Closes: https://github.com/elastic/kibana/issues/173129

- chore(flaky-test-runner): Add a step to collect results and comment on
the tested PR

.buildkite/pipeline-utils/buildkite/client.ts

@@ -29,12 +29,19 @@ export interface BuildkiteGroup {
   steps: BuildkiteStep[];
 }
 
-export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep | BuildkiteTriggerStep;
+export type BuildkiteStep =
+  | BuildkiteCommandStep
+  | BuildkiteInputStep
+  | BuildkiteTriggerStep
+  | BuildkiteWaitStep;
 
 export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
+  concurrency?: number;
+  concurrency_group?: string;
+  concurrency_method?: 'eager' | 'ordered';
   agents:
     | {
         queue: string;
@@ -49,6 +56,7 @@ export interface BuildkiteCommandStep {
       };
   timeout_in_minutes?: number;
   key?: string;
+  cancel_on_build_failing?: boolean;
   depends_on?: string | string[];
   retry?: {
     automatic: Array<{
@@ -56,7 +64,7 @@ export interface BuildkiteCommandStep {
       limit: number;
     }>;
   };
-  env?: { [key: string]: string };
+  env?: { [key: string]: string | number };
 }
 
 interface BuildkiteInputTextField {
@@ -100,7 +108,7 @@ export interface BuildkiteInputStep {
       limit: number;
     }>;
   };
-  env?: { [key: string]: string };
+  env?: { [key: string]: string | number };
 }
 
 export interface BuildkiteTriggerStep {
@@ -138,6 +146,14 @@ export interface BuildkiteTriggerBuildParams {
   pull_request_repository?: string;
 }
 
+export interface BuildkiteWaitStep {
+  wait: string;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  continue_on_failure?: boolean;
+  branches?: string;
+}
+
 export class BuildkiteClient {
   http: AxiosInstance;
   exec: ExecType;

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -7,6 +7,7 @@
  */
 
 import { groups } from './groups.json';
+import { BuildkiteStep } from '#pipeline-utils';
 
 const configJson = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG;
 if (!configJson) {
@@ -138,7 +139,7 @@ if (totalJobs > MAX_JOBS) {
   process.exit(1);
 }
 
-const steps: any[] = [];
+const steps: BuildkiteStep[] = [];
 const pipeline = {
   env: {
     IGNORE_SHIP_CI_STATS_ERROR: 'true',
@@ -154,6 +155,7 @@ steps.push({
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
 
+let suiteIndex = 0;
 for (const testSuite of testSuites) {
   if (testSuite.count <= 0) {
     continue;
@@ -165,6 +167,7 @@ for (const testSuite of testSuites) {
       env: {
         FTR_CONFIG: testSuite.ftrConfig,
       },
+      key: `ftr-suite-${suiteIndex++}`,
       label: `${testSuite.ftrConfig}`,
       parallelism: testSuite.count,
       concurrency,
@@ -195,6 +198,7 @@ for (const testSuite of testSuites) {
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
         agents: getAgentRule(agentQueue),
+        key: `cypress-suite-${suiteIndex++}`,
         depends_on: 'build',
         timeout_in_minutes: 150,
         parallelism: testSuite.count,
@@ -221,4 +225,20 @@ for (const testSuite of testSuites) {
   }
 }
 
+pipeline.steps.push({
+  wait: '~',
+  continue_on_failure: true,
+});
+
+pipeline.steps.push({
+  command: 'ts-node .buildkite/pipelines/flaky_tests/post_stats_on_pr.ts',
+  label: 'Post results on Github pull request',
+  agents: getAgentRule('n2-4-spot'),
+  timeout_in_minutes: 15,
+  retry: {
+    automatic: [{ exit_status: '-1', limit: 3 }],
+  },
+  soft_fail: true,
+});
+
 console.log(JSON.stringify(pipeline, null, 2));

.buildkite/pipelines/flaky_tests/post_stats_on_pr.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { BuildkiteClient, getGithubClient } from '#pipeline-utils';
+
+interface TestSuiteResult {
+  name: string;
+  success: boolean;
+  successCount: number;
+  groupSize: number;
+}
+
+async function main() {
+  // Get buildkite build
+  const buildkite = new BuildkiteClient();
+  const buildkiteBuild = await buildkite.getBuild(
+    process.env.BUILDKITE_PIPELINE_SLUG!,
+    process.env.BUILDKITE_BUILD_NUMBER!
+  );
+  const buildLink = `[${buildkiteBuild.pipeline.slug}#${buildkiteBuild.number}](${buildkiteBuild.web_url})`;
+
+  // Calculate success metrics
+  const jobs = buildkiteBuild.jobs;
+  const testSuiteRuns = jobs.filter((step) => {
+    return step.step_key?.includes('ftr-suite') || step.step_key?.includes('cypress-suite');
+  });
+  const testSuiteGroups = groupBy('name', testSuiteRuns);
+
+  const success = testSuiteRuns.every((job) => job.state === 'passed');
+  const testGroupResults = Object.entries(testSuiteGroups).map(([name, group]) => {
+    const passingTests = group.filter((job) => job.state === 'passed');
+    return {
+      name,
+      success: passingTests.length === group.length,
+      successCount: passingTests.length,
+      groupSize: group.length,
+    };
+  });
+
+  // Comment results on the PR
+  const prNumber = Number(extractPRNumberFromBranch(buildkiteBuild.branch));
+  if (isNaN(prNumber)) {
+    throw new Error(`Couldn't find PR number for build ${buildkiteBuild.web_url}.`);
+  }
+  const flakyRunHistoryLink = `https://buildkite.com/elastic/${
+    buildkiteBuild.pipeline.slug
+  }/builds?branch=${encodeURIComponent(buildkiteBuild.branch)}`;
+
+  const prComment = `
+## Flaky Test Runner Stats
+### ${success ? '🎉 All tests passed!' : '🟠 Some tests failed.'} - ${buildLink}
+${testGroupResults.map(formatTestGroupResult).join('\n')}
+
+[see run history](${flakyRunHistoryLink})
+`;
+
+  const githubClient = getGithubClient();
+  const commentResult = await githubClient.issues.createComment({
+    owner: 'elastic',
+    repo: 'kibana',
+    body: prComment,
+    issue_number: prNumber,
+  });
+
+  console.log(`Comment added: ${commentResult.data.html_url}`);
+}
+
+function formatTestGroupResult(result: TestSuiteResult) {
+  const statusIcon = result.success ? '✅' : '❌';
+  const testName = result.name;
+  const successCount = result.successCount;
+  const groupSize = result.groupSize;
+
+  return `[${statusIcon}] ${testName}: ${successCount}/${groupSize} tests passed.`;
+}
+
+function groupBy<T>(field: keyof T, values: T[]): Record<string, T[]> {
+  return values.reduce((acc, value) => {
+    const key = value[field];
+    if (typeof key !== 'string') {
+      throw new Error('Cannot group by non-string value field');
+    }
+
+    if (!acc[key]) {
+      acc[key] = [];
+    }
+    acc[key].push(value);
+    return acc;
+  }, {} as Record<string, T[]>);
+}
+
+function extractPRNumberFromBranch(branch: string | undefined) {
+  if (!branch) {
+    return null;
+  } else {
+    return branch.match(/refs\/pull\/(\d+)\/head/)?.[1];
+  }
+}
+
+main()
+  .then(() => {
+    console.log('Flaky runner stats comment added to PR!');
+  })
+  .catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });