[BK] Buildkite migration: recreate pipelines with no history retention (#179822)

## Summary
This PR migrates some pipelines that can be migrated the cheap way: by
deleting them, and re-creating them the backstage-way.

Recreates pipelines removed in:
https://github.com/elastic/kibana-buildkite/pull/168

Plus:
- adds the coverage job as a RRE, as it was not previously managed by
terraform, but we got a green light to remove the manually created
pipeline, and recreate it
- adds a script to update locations.yml after any updates (useful to
settle conflicts, or manual additions)
- updates the slack channel for the grammar update script (as requested
by @stratoula)
 
 Todos:
- [x] Fix grammar sync script, to work on new infra (ssh/https
switchover + access rights)
(https://github.com/elastic/kibana/pull/179921)
- [x] Fix missing `antlr` issue:
https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/16#018e9fab-1609-4ae2-b771-3b346cc616ac

.buildkite/pipeline-resource-definitions/fix-location-collection.ts

@@ -0,0 +1,68 @@
+#!/usr/bin/env ts-node-script
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import fs from 'fs';
+import jsYaml from 'js-yaml';
+import path from 'path';
+import { execSync } from 'child_process';
+
+const EXCLUDE_LIST = ['locations.yml'];
+const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main';
+
+type BackstageLocationResource = object & {
+  spec: { targets: string[] };
+};
+
+async function main() {
+  const repoRoot = execSync('git rev-parse --show-toplevel').toString().trim();
+  const resourceDefinitionsFolder = path.resolve(
+    repoRoot,
+    '.buildkite',
+    'pipeline-resource-definitions'
+  );
+  const resourceDefinitionsBaseUrl = `${REPO_FILES_BASE}/.buildkite/pipeline-resource-definitions`;
+  const locationFile = path.resolve(resourceDefinitionsFolder, 'locations.yml');
+  const locationFileLines = fs.readFileSync(locationFile, 'utf8').split('\n');
+
+  const pipelines = readDirRecursively(resourceDefinitionsFolder)
+    .filter((file) => file.endsWith('.yml'))
+    .map((file) => file.replace(`${resourceDefinitionsFolder}/`, ''))
+    .filter((f) => !EXCLUDE_LIST.includes(f));
+
+  const preamble = locationFileLines.slice(0, 1);
+
+  const locationObj = jsYaml.load(
+    locationFileLines.slice(1).join('\n')
+  ) as BackstageLocationResource;
+  locationObj.spec.targets = pipelines.map(
+    (fileName) => `${resourceDefinitionsBaseUrl}/${fileName}`
+  );
+
+  const locationYaml = jsYaml.dump(locationObj, { lineWidth: 400 });
+
+  fs.writeFileSync(locationFile, `${preamble.join('\n')}\n${locationYaml}`);
+
+  console.log('Updated locations.yml');
+}
+
+function readDirRecursively(dir: string): string[] {
+  const files = fs.readdirSync(dir);
+  return files.reduce((acc, file) => {
+    const filePath = path.join(dir, file);
+    if (fs.statSync(filePath).isDirectory()) {
+      return [...acc, ...readDirRecursively(filePath)];
+    } else {
+      return [...acc, filePath];
+    }
+  }, [] as string[]);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml

@@ -0,0 +1,44 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-code-coverage-main
+  description: 'Collects code coverage for unit and e2e tests, publishes results on Kibana stats cluster'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-code-coverage-main'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:appex-qa'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / code-coverage / main
+      description: 'Collects code coverage for unit and e2e tests, publishes results on Kibana stats cluster'
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#appex-qa-bots'
+        GITHUB_COMMIT_STATUS_CONTEXT: kibana-code-coverage-main
+        CODE_COVERAGE: '1'
+        FTR_CONFIGS_RETRY_COUNT: '0'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/code_coverage/daily.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml

@@ -0,0 +1,48 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-esql-grammar-sync
+  description: Opens a PR if anything changes in the ES|QL grammar in Elasticsearch
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-esql-grammar-sync'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-esql'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / ES|QL grammar sync
+      description: Opens a PR if anything changes in the ES|QL grammar in Elasticsearch
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-esql-project-team'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/esql_grammar_sync.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        kibana-esql:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml

@@ -0,0 +1,45 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-purge-cloud-deployments
+  description: Purge stale cloud deployments
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-purge-cloud-deployments'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / purge-cloud-deployments
+      description: Purge stale cloud deployments
+    spec:
+      env:
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/purge_cloud_deployments.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -6,13 +6,16 @@ metadata:
 spec:
   type: url
   targets:
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-explore.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-detection-engine.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-entity-analytics.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-defend-workflows.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml
-    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml

.buildkite/pipelines/code_coverage/daily.yml

@@ -3,14 +3,20 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     env:
       FTR_CONFIGS_DEPS: ''
@@ -21,7 +27,11 @@ steps:
   - command: .buildkite/scripts/steps/code_coverage/ingest.sh
     label: 'Merge and Ingest'
     agents:
-      queue: n2-4-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-4
+      preemptible: true
     depends_on:
       - jest
       - jest-integration

.buildkite/pipelines/esql_grammar_sync.yml

@@ -3,4 +3,8 @@ steps:
     label: ES|QL Grammar Sync
     timeout_in_minutes: 10
     agents:
-      queue: n2-2-spot
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
+      preemptible: true

.buildkite/pipelines/purge_cloud_deployment.yml

@@ -1,13 +0,0 @@
-steps:
-  - block: "Purge deployment"
-    prompt: "Fill out the details to shutdown a PR deployment"
-    fields:
-      - text: "Pull Request Number"
-        key: "kibana-pull-request"
-    if: "build.env('KIBANA_PULL_REQUEST') == null || build.env('KIBANA_PULL_REQUEST') == ''"
-
-  - command: .buildkite/scripts/steps/cloud/purge_deployment.sh
-    label: Purge a cloud deployment
-    timeout_in_minutes: 10
-    agents:
-      queue: kibana-default

.buildkite/pipelines/purge_cloud_deployments.yml

@@ -3,4 +3,7 @@ steps:
     label: Purge old cloud deployments
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2

.buildkite/scripts/steps/cloud/purge_deployment.sh

@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-ts-node .buildkite/scripts/steps/cloud/purge_deployment.ts

.buildkite/scripts/steps/cloud/purge_deployment.ts

@@ -1,38 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-import { execSync } from 'child_process';
-
-const deploymentsListJson = execSync('ecctl deployment list --output json').toString();
-const { deployments } = JSON.parse(deploymentsListJson);
-const secretBasePath = process.env.VAULT_ADDR?.match(/secrets\.elastic\.co/g)
-  ? 'secret/kibana-issues/dev'
-  : 'secret/ci/elastic-kibana';
-
-const prNumber = parseInt(
-  process.env.KIBANA_PULL_REQUEST ||
-    execSync('buildkite-agent meta-data get kibana-pull-request').toString(),
-  10
-);
-const deploymentName = `kibana-pr-${prNumber}`;
-const deployment = deployments.find((d: any) => d.name === deploymentName);
-
-if (!prNumber || !deployment) {
-  console.error(`${deploymentName} not found`);
-  process.exit(1);
-}
-
-console.log(`Scheduling deployment for deletion: ${deployment.name} / ${deployment.id}`);
-try {
-  execSync(`ecctl deployment shutdown --force '${deployment.id}'`, { stdio: 'inherit' });
-  execSync(`vault delete ${secretBasePath}/cloud-deploy/${deployment.name}`, {
-    stdio: 'inherit',
-  });
-} catch (ex) {
-  console.error(ex.toString());
-}

.buildkite/scripts/steps/esql_grammar_sync.sh

@@ -7,7 +7,7 @@ synchronize_lexer_grammar () {
   destination_file="./packages/kbn-esql-ast/src/antlr/esql_lexer.g4"
 
   # Copy the file
-  cp "$source_file" "$destination_file" 
+  cp "$source_file" "$destination_file"
 
   # Insert the license header
   temp_file=$(mktemp)
@@ -15,11 +15,11 @@ synchronize_lexer_grammar () {
   mv "$temp_file" "$destination_file"
 
   # Replace the line containing "lexer grammar" with "lexer grammar esql_lexer;"
-  sed -i -e 's/lexer grammar.*$/lexer grammar esql_lexer;/' "$destination_file" 
+  sed -i -e 's/lexer grammar.*$/lexer grammar esql_lexer;/' "$destination_file"
 
   # Insert "options { caseInsensitive = true; }" one line below
   sed -i -e '/lexer grammar esql_lexer;/a\
-  options { caseInsensitive = true; }' "$destination_file" 
+  options { caseInsensitive = true; }' "$destination_file"
 
   echo "File copied and modified successfully."
 }
@@ -30,7 +30,7 @@ synchronize_parser_grammar () {
   destination_file="./packages/kbn-esql-ast/src/antlr/esql_parser.g4"
 
   # Copy the file
-  cp "$source_file" "$destination_file" 
+  cp "$source_file" "$destination_file"
 
   # Insert the license header
   temp_file=$(mktemp)
@@ -38,10 +38,10 @@ synchronize_parser_grammar () {
   mv "$temp_file" "$destination_file"
 
   # Replace the line containing "parser grammar" with "parser grammar esql_parser;"
-  sed -i -e 's/parser grammar.*$/parser grammar esql_parser;/' "$destination_file" 
+  sed -i -e 's/parser grammar.*$/parser grammar esql_parser;/' "$destination_file"
 
   # Replace options {tokenVocab=EsqlBaseLexer;} with options {tokenVocab=esql_lexer;}
-  sed -i -e 's/options {tokenVocab=EsqlBaseLexer;}/options {tokenVocab=esql_lexer;}/' "$destination_file" 
+  sed -i -e 's/options {tokenVocab=EsqlBaseLexer;}/options {tokenVocab=esql_lexer;}/' "$destination_file"
 
   echo "File copied and modified successfully."
 }
@@ -51,17 +51,17 @@ report_main_step () {
 }
 
 main () {
-  cd "$PARENT_DIR" 
+  cd "$PARENT_DIR"
 
   report_main_step "Cloning repositories"
 
   rm -rf elasticsearch
-  git clone https://github.com/elastic/elasticsearch --depth 1 
+  git clone https://github.com/elastic/elasticsearch --depth 1
 
   rm -rf open-source
-  git clone https://github.com/elastic/open-source --depth 1 
+  git clone https://github.com/elastic/open-source --depth 1
 
-  cd "$KIBANA_DIR" 
+  cd "$KIBANA_DIR"
 
   license_header=$(cat "$PARENT_DIR/open-source/legal/elastic-license-2.0-header.txt")
 
@@ -105,8 +105,8 @@ main () {
   .buildkite/scripts/bootstrap.sh
 
   # Build ANTLR stuff
-  cd ./packages/kbn-esql-ast/src  
-  yarn build:antlr4:esql 
+  cd ./packages/kbn-esql-ast/src
+  yarn build:antlr4:esql
 
   # Make a commit
   BRANCH_NAME="esql_grammar_sync_$(date +%s)"
@@ -118,11 +118,10 @@ main () {
 
   report_main_step "Changes committed. Creating pull request."
 
-  git remote add kibanamachine https://github.com/kibanamachine/kibana.git
-  git push kibanamachine "$BRANCH_NAME"
+  git push origin "$BRANCH_NAME"
 
   # Create a PR
-  gh pr create --draft --title "$PR_TITLE" --body "$PR_BODY" --base main --head "kibanamachine:${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' 
+  gh pr create --draft --title "$PR_TITLE" --body "$PR_BODY" --base main --head "${BRANCH_NAME}" --label 'release_note:skip' --label 'Team:ESQL' 
 }
 
 main