[7.17][BK] Migrate kibana-pull-request job (#183707)

## Summary
Manual backport of #183158 

Success: https://buildkite.com/elastic/kibana-kme-test/builds/339

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

.buildkite/pipelines/artifacts.yml

@@ -3,7 +3,7 @@ steps:
     label: Build Kibana Artifacts
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: c2-standard-16
     timeout_in_minutes: 75
@@ -18,7 +18,7 @@ steps:
     label: Artifact Testing
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
@@ -34,7 +34,7 @@ steps:
     label: Artifact Testing
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
@@ -50,7 +50,7 @@ steps:
     label: Artifact Testing
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1
@@ -66,7 +66,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -81,7 +81,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -96,7 +96,7 @@ steps:
     label: 'Docker Context Verification'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme
@@ -113,7 +113,7 @@ steps:
     label: 'Publish Kibana Artifacts'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme

.buildkite/pipelines/pull_request/apm_cypress.yml

@@ -2,7 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/apm_cypress.sh
     label: 'APM Cypress Tests'
     agents:
-      queue: ci-group-6
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-custom-6-16384
     depends_on: build
     timeout_in_minutes: 120
     retry:

.buildkite/pipelines/pull_request/base.yml

@@ -3,14 +3,20 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: c2-standard-16
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
@@ -19,7 +25,12 @@ steps:
     label: 'Default CI Group'
     parallelism: 27
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
@@ -31,7 +42,12 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -44,7 +60,11 @@ steps:
     label: 'OSS CI Group'
     parallelism: 11
     agents:
-      queue: ci-group-4d
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     key: oss-cigroup
@@ -56,7 +76,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_accessibility.sh
     label: 'OSS Accessibility Tests'
     agents:
-      queue: ci-group-4d
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -67,7 +91,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -78,7 +107,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_firefox.sh
     label: 'OSS Firefox Tests'
     agents:
-      queue: ci-group-4d
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -89,7 +122,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -100,7 +138,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -111,7 +154,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -123,7 +171,12 @@ steps:
     label: 'Jest Tests'
     parallelism: 8
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 90
     key: jest
 
@@ -131,35 +184,58 @@ steps:
     label: 'Jest Integration Tests'
     parallelism: 3
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     timeout_in_minutes: 120
     key: jest-integration
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 120
     key: api-integration
 
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
-      queue: n2-2
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     key: linting
     timeout_in_minutes: 90
 
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: c2-16
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: c2-standard-16
     key: linting_with_types
     timeout_in_minutes: 90
 
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-2
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -169,19 +245,34 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: c2-8
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-8
     timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh
     label: 'Build Storybooks'
     agents:
-      queue: c2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: c2-standard-4
     key: storybooks
     timeout_in_minutes: 60
 
   - command: .buildkite/scripts/steps/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      queue: n2-4
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-standard-4
     key: build_api_docs
     timeout_in_minutes: 60

.buildkite/pipelines/pull_request/post_build.yml

@@ -5,4 +5,7 @@ steps:
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2

.buildkite/pipelines/pull_request/response_ops.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/response_ops.sh
     label: 'Rules, Alerts and Exceptions ResponseOps Cypress Tests on Security Solution'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 120
     parallelism: 4

.buildkite/pipelines/pull_request/security_solution.yml

@@ -2,7 +2,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 16
@@ -16,7 +20,11 @@ steps:
   - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
     label: 'Security Solution Cypress tests, burning changed specs'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on: build
     timeout_in_minutes: 60
     parallelism: 1

.buildkite/pipelines/pull_request/uptime.yml

@@ -2,7 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/functional/uptime.sh
     label: 'Uptime @elastic/synthetics Tests'
     agents:
-      queue: ci-group-6
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      localSsds: 1
+      localSsdInterface: nvme
+      machineType: n2-custom-6-16384
     depends_on: build
     timeout_in_minutes: 120
     retry:

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -24,16 +24,6 @@ const getPipeline = (filename, removeSteps = true) => {
   return removeSteps ? str.replace(/^steps:/, '') : str;
 };
 
-const uploadPipeline = (pipelineContent) => {
-  const str =
-    typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
-
-  execSync('buildkite-agent pipeline upload', {
-    input: str,
-    stdio: ['pipe', 'inherit', 'inherit'],
-  });
-};
-
 (async () => {
   try {
     const skippable = await areChangesSkippable(SKIPPABLE_PR_MATCHERS, REQUIRED_PATHS);
@@ -93,9 +83,9 @@ const uploadPipeline = (pipelineContent) => {
 
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
-    uploadPipeline(pipeline.join('\n'));
+    console.log([...new Set(pipeline)].join('\n'));
   } catch (ex) {
-    console.error('PR pipeline generation error', ex.message);
+    console.error('Error while generating the pipeline steps: ' + ex.message, ex);
     process.exit(1);
   }
 })();

.buildkite/scripts/steps/storybooks/build_and_upload.js

@@ -67,6 +67,8 @@ const build = () => {
   }
 };
 
+const kibanaRoot = execSync('git rev-parse --show-toplevel').toString().trim();
+
 const upload = () => {
   const originalDirectory = process.cwd();
   try {
@@ -97,8 +99,12 @@ const upload = () => {
 
     fs.writeFileSync('index.html', html);
 
-    console.log('--- Uploading Storybooks');
+    const activateScript = path.relative(
+      process.cwd(),
+      path.join(kibanaRoot, '.buildkite', 'scripts', 'common', 'activate_service_account.sh')
+    );
     exec(`
+      ${activateScript} gs://ci-artifacts.kibana.dev
       gsutil -q -m cp -r -z js,css,html,json,map,txt,svg '*' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/${process.env.BUILDKITE_COMMIT}/'
       gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" cp -z html 'index.html' 'gs://${STORYBOOK_BUCKET}/${STORYBOOK_DIRECTORY}/latest/'
     `);

packages/kbn-cli-dev-mode/src/dev_server.test.ts

@@ -135,7 +135,6 @@ describe('#run$', () => {
             "env": Object {
               "<inheritted process.env>": true,
               "ELASTIC_APM_SERVICE_NAME": "kibana",
-              "FORCE_COLOR": "true",
               "isDevCliChild": "true",
             },
             "nodeOptions": Array [

src/cli_plugin/install/zip.test.js

@@ -15,7 +15,8 @@ import glob from 'glob';
 
 import { analyzeArchive, extractArchive } from './zip';
 
-const getMode = (path) => (fs.statSync(path).mode & parseInt('777', 8)).toString(8);
+const getExecFlags = (path) =>
+  (fs.statSync(path).mode & parseInt('111', 8)).toString(8).padStart(3, '0');
 
 describe('kibana cli', function () {
   describe('zip', function () {
@@ -83,8 +84,8 @@ describe('kibana cli', function () {
           ]
         `);
 
-        expect(getMode(path.resolve(tempPath, 'executable'))).toEqual('755');
-        expect(getMode(path.resolve(tempPath, 'not-executable'))).toEqual('644');
+        expect(getExecFlags(path.resolve(tempPath, 'executable'))).toEqual('111');
+        expect(getExecFlags(path.resolve(tempPath, 'not-executable'))).toEqual('000');
       });
     });