[ci/flaky] support triggering a with config via the env (#121215)

.buildkite/pipelines/flaky_tests/groups.json

@@ -0,0 +1,34 @@
+{
+  "groups": [
+    {
+      "key": "oss/cigroup",
+      "name": "OSS CI Group",
+      "ciGroups": 12
+    },
+    {
+      "key": "oss/firefox",
+      "name": "OSS Firefox"
+    },
+    {
+      "key": "oss/accessibility",
+      "name": "OSS Accessibility"
+    },
+    {
+      "key": "xpack/cigroup",
+      "name": "Default CI Group",
+      "ciGroups": 27
+    },
+    {
+      "key": "xpack/cigroup/Docker",
+      "name": "Default CI Group Docker"
+    },
+    {
+      "key": "xpack/firefox",
+      "name": "Default Firefox"
+    },
+    {
+      "key": "xpack/accessibility",
+      "name": "Default Accessibility"
+    }
+  ]
+}

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -1,3 +1,7 @@
+const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>} */(
+  require('./groups.json').groups
+)
+
 const stepInput = (key, nameOfSuite) => {
   return {
     key: `ftsr-suite/${key}`,
@@ -7,38 +11,31 @@ const stepInput = (key, nameOfSuite) => {
   };
 };
 
-const OSS_CI_GROUPS = 12;
-const XPACK_CI_GROUPS = 27;
-
 const inputs = [
   {
     key: 'ftsr-override-count',
     text: 'Override for all suites',
-    default: 0,
+    default: '0',
     required: true,
   },
 ];
 
-for (let i = 1; i <= OSS_CI_GROUPS; i++) {
-  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+for (const group of groups) {
+  if (!group.ciGroups) {
+    inputs.push(stepInput(group.key, group.name))
+  } else {
+    for (let i = 1; i <= group.ciGroups; i++) {
+      inputs.push(stepInput(`${group.key}/${i}`, `${group.name} ${i}`))
+    }
+  }
 }
 
-inputs.push(stepInput(`oss/firefox`, 'OSS Firefox'));
-inputs.push(stepInput(`oss/accessibility`, 'OSS Accessibility'));
-
-for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
-  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
-}
-
-inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
-inputs.push(stepInput(`xpack/firefox`, 'Default Firefox'));
-inputs.push(stepInput(`xpack/accessibility`, 'Default Accessibility'));
-
 const pipeline = {
   steps: [
     {
       input: 'Number of Runs - Click Me',
       fields: inputs,
+      if: `build.env('KIBANA_FLAKY_TEST_RUNNER_CONFIG') == null`
     },
     {
       wait: '~',

.buildkite/pipelines/flaky_tests/runner.js

@@ -1,37 +1,85 @@
 const { execSync } = require('child_process');
 
-const keys = execSync('buildkite-agent meta-data keys')
-  .toString()
-  .split('\n')
-  .filter((k) => k.startsWith('ftsr-suite/'));
-
-const overrideCount = parseInt(
-  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
-);
-
 const concurrency = 25;
+const defaultCount = concurrency * 2;
 const initialJobs = 3;
 
-let totalJobs = initialJobs;
+function getTestSuitesFromMetadata() {
+  const keys = execSync('buildkite-agent meta-data keys')
+    .toString()
+    .split('\n')
+    .filter((k) => k.startsWith('ftsr-suite/'));
 
-const testSuites = [];
-for (const key of keys) {
-  if (!key) {
-    continue;
+  const overrideCount = execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim();
+
+  const testSuites = [];
+  for (const key of keys) {
+    if (!key) {
+      continue;
+    }
+
+    const value =
+      overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+    const count = value === '' ? defaultCount : parseInt(value);
+    totalJobs += count;
+
+    testSuites.push({
+      key: key.replace('ftsr-suite/', ''),
+      count: count,
+    });
   }
 
-  const value =
-    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
-
-  const count = value === '' ? defaultCount : parseInt(value);
-  totalJobs += count;
-
-  testSuites.push({
-    key: key.replace('ftsr-suite/', ''),
-    count: count,
-  });
+  return testSuites
 }
 
+function getTestSuitesFromJson(json) {
+  const fail = (errorMsg) => {
+    console.error('+++ Invalid test config provided')
+    console.error(`${errorMsg}: ${json}`);
+    process.exit(1);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(json)
+  } catch (error) {
+    fail(`JSON test config did not parse correctly`)
+  }
+
+  if (!Array.isArray(parsed)) {
+    fail(`JSON test config must be an array`)
+  }
+
+  /** @type {Array<{ key: string, count: number }>} */
+  const testSuites = []
+  for (const item of parsed) {
+    if (typeof item !== 'object' || item === null) {
+      fail(`testSuites must be objects`)
+    }
+    const key = item.key
+    if (typeof key !== 'string') {
+      fail(`testSuite.key must be a string`)
+    }
+    const count = item.count;
+    if (typeof count !== 'number') {
+      fail(`testSuite.count must be a number`)
+    }
+    testSuites.push({
+      key,
+      count,
+    })
+  }
+
+  return testSuites
+}
+
+const testSuites = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG
+  ? getTestSuitesFromJson(process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG)
+  : getTestSuitesFromMetadata();
+
+let totalJobs = testSuites.reduce((acc, t) => acc + t.count, initialJobs);
+
 if (totalJobs > 500) {
   console.error('+++ Too many tests');
   console.error(