[ci/flaky] support triggering with config via the env (#122041)

.buildkite/pipelines/flaky_tests/groups.json

@@ -0,0 +1,34 @@
+{
+  "groups": [
+    {
+      "key": "oss/cigroup",
+      "name": "OSS CI Group",
+      "ciGroups": 12
+    },
+    {
+      "key": "oss/firefox",
+      "name": "OSS Firefox"
+    },
+    {
+      "key": "oss/accessibility",
+      "name": "OSS Accessibility"
+    },
+    {
+      "key": "xpack/cigroup",
+      "name": "Default CI Group",
+      "ciGroups": 27
+    },
+    {
+      "key": "xpack/cigroup/Docker",
+      "name": "Default CI Group Docker"
+    },
+    {
+      "key": "xpack/firefox",
+      "name": "Default Firefox"
+    },
+    {
+      "key": "xpack/accessibility",
+      "name": "Default Accessibility"
+    }
+  ]
+}

.buildkite/pipelines/flaky_tests/pipeline.js

@@ -1,3 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>} */ (
+  require('./groups.json').groups
+);
+
 const stepInput = (key, nameOfSuite) => {
   return {
     key: `ftsr-suite/${key}`,
@@ -7,38 +19,31 @@ const stepInput = (key, nameOfSuite) => {
   };
 };
 
-const OSS_CI_GROUPS = 12;
-const XPACK_CI_GROUPS = 27;
-
 const inputs = [
   {
     key: 'ftsr-override-count',
     text: 'Override for all suites',
-    default: 0,
+    default: '0',
     required: true,
   },
 ];
 
-for (let i = 1; i <= OSS_CI_GROUPS; i++) {
-  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
+for (const group of groups) {
+  if (!group.ciGroups) {
+    inputs.push(stepInput(group.key, group.name));
+  } else {
+    for (let i = 1; i <= group.ciGroups; i++) {
+      inputs.push(stepInput(`${group.key}/${i}`, `${group.name} ${i}`));
+    }
+  }
 }
 
-inputs.push(stepInput(`oss/firefox`, 'OSS Firefox'));
-inputs.push(stepInput(`oss/accessibility`, 'OSS Accessibility'));
-
-for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
-  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
-}
-
-inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
-inputs.push(stepInput(`xpack/firefox`, 'Default Firefox'));
-inputs.push(stepInput(`xpack/accessibility`, 'Default Accessibility'));
-
 const pipeline = {
   steps: [
     {
       input: 'Number of Runs - Click Me',
       fields: inputs,
+      if: `build.env('KIBANA_FLAKY_TEST_RUNNER_CONFIG') == null`,
     },
     {
       wait: '~',

.buildkite/pipelines/flaky_tests/runner.js

@@ -1,37 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
 const { execSync } = require('child_process');
 
-const keys = execSync('buildkite-agent meta-data keys')
-  .toString()
-  .split('\n')
-  .filter((k) => k.startsWith('ftsr-suite/'));
-
-const overrideCount = parseInt(
-  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
-);
-
 const concurrency = 25;
+const defaultCount = concurrency * 2;
 const initialJobs = 3;
 
-let totalJobs = initialJobs;
+function getTestSuitesFromMetadata() {
+  const keys = execSync('buildkite-agent meta-data keys')
+    .toString()
+    .split('\n')
+    .filter((k) => k.startsWith('ftsr-suite/'));
 
-const testSuites = [];
-for (const key of keys) {
-  if (!key) {
-    continue;
+  const overrideCount = execSync(`buildkite-agent meta-data get 'ftsr-override-count'`)
+    .toString()
+    .trim();
+
+  const testSuites = [];
+  for (const key of keys) {
+    if (!key) {
+      continue;
+    }
+
+    const value =
+      overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+
+    const count = value === '' ? defaultCount : parseInt(value);
+    testSuites.push({
+      key: key.replace('ftsr-suite/', ''),
+      count: count,
+    });
   }
 
-  const value =
-    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
-
-  const count = value === '' ? defaultCount : parseInt(value);
-  totalJobs += count;
-
-  testSuites.push({
-    key: key.replace('ftsr-suite/', ''),
-    count: count,
-  });
+  return testSuites;
 }
 
+function getTestSuitesFromJson(json) {
+  const fail = (errorMsg) => {
+    console.error('+++ Invalid test config provided');
+    console.error(`${errorMsg}: ${json}`);
+    process.exit(1);
+  };
+
+  let parsed;
+  try {
+    parsed = JSON.parse(json);
+  } catch (error) {
+    fail(`JSON test config did not parse correctly`);
+  }
+
+  if (!Array.isArray(parsed)) {
+    fail(`JSON test config must be an array`);
+  }
+
+  /** @type {Array<{ key: string, count: number }>} */
+  const testSuites = [];
+  for (const item of parsed) {
+    if (typeof item !== 'object' || item === null) {
+      fail(`testSuites must be objects`);
+    }
+    const key = item.key;
+    if (typeof key !== 'string') {
+      fail(`testSuite.key must be a string`);
+    }
+    const count = item.count;
+    if (typeof count !== 'number') {
+      fail(`testSuite.count must be a number`);
+    }
+    testSuites.push({
+      key,
+      count,
+    });
+  }
+
+  return testSuites;
+}
+
+const testSuites = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG
+  ? getTestSuitesFromJson(process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG)
+  : getTestSuitesFromMetadata();
+
+const totalJobs = testSuites.reduce((acc, t) => acc + t.count, initialJobs);
+
 if (totalJobs > 500) {
   console.error('+++ Too many tests');
   console.error(

.buildkite/scripts/lifecycle/annotate_test_failures.js

@@ -1,3 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// eslint-disable-next-line import/no-unresolved
 const { TestFailures } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/build_status.js

@@ -1,3 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// eslint-disable-next-line import/no-unresolved
 const { BuildkiteClient } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/ci_stats_complete.js

@@ -1,3 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// eslint-disable-next-line import/no-unresolved
 const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/ci_stats_start.js

@@ -1,3 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// eslint-disable-next-line import/no-unresolved
 const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/lifecycle/print_agent_links.js

@@ -1,3 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+// eslint-disable-next-line import/no-unresolved
 const { BuildkiteClient } = require('kibana-buildkite-library');
 
 (async () => {

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -1,5 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
 const execSync = require('child_process').execSync;
 const fs = require('fs');
+// eslint-disable-next-line import/no-unresolved
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
 
 const SKIPPABLE_PATHS = [
@@ -77,20 +86,14 @@ const uploadPipeline = (pipelineContent) => {
     }
 
     if (
-      (await doAnyChangesMatch([
-        /^x-pack\/plugins\/fleet/,
-        /^x-pack\/test\/fleet_cypress/,
-      ])) ||
+      (await doAnyChangesMatch([/^x-pack\/plugins\/fleet/, /^x-pack\/test\/fleet_cypress/])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fleet_cypress.yml'));
     }
 
     if (
-      (await doAnyChangesMatch([
-        /^x-pack\/plugins\/osquery/,
-        /^x-pack\/test\/osquery_cypress/,
-      ])) ||
+      (await doAnyChangesMatch([/^x-pack\/plugins\/osquery/, /^x-pack\/test\/osquery_cypress/])) ||
       process.env.GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/osquery_cypress.yml'));

.buildkite/scripts/steps/es_snapshots/bucket_config.js

@@ -1,3 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
 module.exports = {
   BASE_BUCKET_DAILY: 'kibana-ci-es-snapshots-daily',
   BASE_BUCKET_PERMANENT: 'kibana-ci-es-snapshots-permanent',

.buildkite/scripts/steps/es_snapshots/create_manifest.js

@@ -1,3 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
 const fs = require('fs');
 const { execSync } = require('child_process');
 const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
@@ -47,7 +55,7 @@ const { BASE_BUCKET_DAILY } = require('./bucket_config.js');
           version: parts[1],
           platform: parts[3],
           architecture: parts[4].split('.')[0],
-          license: parts[0] == 'oss' ? 'oss' : 'default',
+          license: parts[0] === 'oss' ? 'oss' : 'default',
         };
       });
 

.buildkite/scripts/steps/es_snapshots/promote_manifest.js

@@ -1,3 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
 const fs = require('fs');
 const { execSync } = require('child_process');
 const { BASE_BUCKET_DAILY, BASE_BUCKET_PERMANENT } = require('./bucket_config.js');

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -1,3 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const path = require('path');
@@ -73,7 +81,7 @@ const upload = () => {
       .trim()
       .split('\n')
       .map((path) => path.replace('/', ''))
-      .filter((path) => path != 'composite');
+      .filter((path) => path !== 'composite');
 
     const listHtml = storybooks
       .map((storybook) => `<li><a href="${STORYBOOK_BASE_URL}/${storybook}">${storybook}</a></li>`)

.eslintignore

@@ -17,6 +17,7 @@ snapshots.js
 !/.ci
 !/.eslintrc.js
 !.storybook
+!.buildkite
 
 # plugin overrides
 /src/core/lib/kbn_internal_native_observable