[FTSR] Convert to tasks and add jest/api integration suites (#91770)

.ci/Jenkinsfile_flaky

@@ -3,47 +3,39 @@
 library 'kibana-pipeline-library'
 kibanaLibrary.load()
 
-def CI_GROUP_PARAM = params.CI_GROUP
+def TASK_PARAM = params.TASK ?: params.CI_GROUP
 
 // Looks like 'oss:ciGroup:1', 'oss:firefoxSmoke'
-def JOB_PARTS = CI_GROUP_PARAM.split(':')
+def JOB_PARTS = TASK_PARAM.split(':')
 def IS_XPACK = JOB_PARTS[0] == 'xpack'
-def JOB = JOB_PARTS[1]
+def JOB = JOB_PARTS.size() > 1 ? JOB_PARTS[1] : JOB_PARTS[0]
 def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : ''
 def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger()
 def AGENT_COUNT = getAgentCount(EXECUTIONS)
-
-def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)
-
-def workerFailures = []
+def NEED_BUILD = JOB != 'jestIntegration' && JOB != 'apiIntegration'
 
 currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
 currentBuild.description = "${params.CI_GROUP}<br />Agents: ${AGENT_COUNT}<br />Executions: ${params.NUMBER_EXECUTIONS}"
 
 kibanaPipeline(timeoutMinutes: 180) {
   def agents = [:]
-  for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
-    def agentNumberInside = agentNumber
-    def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
-    agents["agent-${agentNumber}"] = {
-      catchErrors {
-        print "Agent ${agentNumberInside} - ${agentExecutions} executions"
+  def workerFailures = []
 
-        withEnv([
-          'IGNORE_SHIP_CI_STATS_ERROR=true',
-        ]) {
-          workers.functional('flaky-test-runner', {
-            if (!IS_XPACK) {
-              kibanaPipeline.buildOss()
-              if (CI_GROUP == '1') {
-                runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
-              }
-            } else {
-              kibanaPipeline.buildXpack()
-            }
-          }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
-        }
-      }
+  def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)
+
+  for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
+    def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
+
+    agents["agent-${agentNumber}"] = {
+      agentProcess(
+        agentNumber: agentNumber,
+        agentExecutions: agentExecutions,
+        worker: worker,
+        workerFailures: workerFailures,
+        needBuild: NEED_BUILD,
+        isXpack: IS_XPACK,
+        ciGroup: CI_GROUP
+      )
     }
   }
 
@@ -59,14 +51,70 @@ kibanaPipeline(timeoutMinutes: 180) {
   }
 }
 
+def agentProcess(Map params = [:]) {
+  def config = [
+    agentNumber: 1,
+    agentExecutions: 0,
+    worker: {},
+    workerFailures: [],
+    needBuild: false,
+    isXpack: false,
+    ciGroup: null,
+  ] + params
+
+  catchErrors {
+    print "Agent ${config.agentNumber} - ${config.agentExecutions} executions"
+
+    withEnv([
+      'IGNORE_SHIP_CI_STATS_ERROR=true',
+    ]) {
+      kibanaPipeline.withTasks([
+        parallel: 20,
+      ]) {
+        task {
+          if (config.needBuild) {
+            if (!config.isXpack) {
+              kibanaPipeline.buildOss()
+            } else {
+              kibanaPipeline.buildXpack()
+            }
+          }
+
+          for(def i = 0; i < config.agentExecutions; i++) {
+            def taskNumber = i
+            task({
+              withEnv([
+                "REMOVE_KIBANA_INSTALL_DIR=1",
+              ]) {
+                catchErrors {
+                  try {
+                    config.worker()
+                  } catch (ex) {
+                    config.workerFailures << "agent-${config.agentNumber}-${taskNumber}"
+                    throw ex
+                  }
+                }
+              }
+            })
+          }
+        }
+      }
+    }
+  }
+}
+
 def getWorkerFromParams(isXpack, job, ciGroup) {
   if (!isXpack) {
     if (job == 'accessibility') {
       return kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh')
     } else if (job == 'firefoxSmoke') {
       return kibanaPipeline.functionalTestProcess('firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh')
-    } else if(job == 'visualRegression') {
+    } else if (job == 'visualRegression') {
       return kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh')
+    } else if (job == 'jestIntegration') {
+      return kibanaPipeline.scriptTaskDocker('Jest Integration Tests', 'test/scripts/test/jest_integration.sh')
+    } else if (job == 'apiIntegration') {
+      return kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh')
     } else {
       return kibanaPipeline.ossCiGroupProcess(ciGroup)
     }
@@ -76,45 +124,16 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
     return kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh')
   } else if (job == 'firefoxSmoke') {
     return kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh')
-  } else if(job == 'visualRegression') {
+  } else if (job == 'visualRegression') {
     return kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')
   } else {
     return kibanaPipeline.xpackCiGroupProcess(ciGroup)
   }
 }
 
-def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) {
-  def workerMap = [:]
-  def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses)
-
-  for(def i = 1; i <= numberOfWorkers; i++) {
-    def workerExecutions = floor(numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0))
-
-    workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber ->
-      for(def j = 0; j < workerExecutions; j++) {
-        print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
-        withEnv([
-          "REMOVE_KIBANA_INSTALL_DIR=1",
-        ]) {
-          catchErrors {
-            try {
-              worker(workerNumber)
-            } catch (ex) {
-              workerFailures << "agent-${agentNumber} worker-${workerNumber}-${j}"
-              throw ex
-            }
-          }
-        }
-      }
-    }
-  }
-
-  return workerMap
-}
-
 def getAgentCount(executions) {
-  // Increase agent count every 24 worker processess, up to 3 agents maximum
-  return Math.min(3, 1 + floor(executions/24))
+  // Increase agent count every 20 worker processess, up to 3 agents maximum
+  return Math.min(3, 1 + floor(executions/20))
 }
 
 def trunc(str, length) {

vars/kibanaPipeline.groovy

@@ -425,12 +425,13 @@ def buildXpackPlugins() {
   runbld('./test/scripts/jenkins_xpack_build_plugins.sh', 'Build X-Pack Plugins')
 }
 
-def withTasks(Map params = [worker: [:]], Closure closure) {
+def withTasks(Map params = [:], Closure closure) {
   catchErrors {
-    def config = [name: 'ci-worker', size: 'xxl', ramDisk: true] + (params.worker ?: [:])
+    def config = [setupWork: {}, worker: [:], parallel: 24] + params
+    def workerConfig = [name: 'ci-worker', size: 'xxl', ramDisk: true] + config.worker
 
-    workers.ci(config) {
-      withCiTaskQueue(parallel: 24) {
+    workers.ci(workerConfig) {
+      withCiTaskQueue([parallel: config.parallel]) {
         parallel([
           docker: {
             retry(2) {
@@ -443,6 +444,8 @@ def withTasks(Map params = [worker: [:]], Closure closure) {
           xpackPlugins: { buildXpackPlugins() },
         ])
 
+        config.setupWork()
+
         catchErrors {
           closure()
         }