[CI] Remove kme leftovers (take 2) (#187947)

## Summary
Retries #187762 again. There was a partially removed step in the
previous attempt, that's now fully removed.

.buildkite/pipeline-utils/agent_images.ts

@@ -52,4 +52,19 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | AgentImageCo
   return config;
 }
 
-export { getAgentImageConfig };
+const expandAgentQueue = (queueName: string = 'n2-4-spot') => {
+  const [kind, cores, addition] = queueName.split('-');
+  const additionalProps =
+    {
+      spot: { preemptible: true },
+      virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+    }[addition] || {};
+
+  return {
+    ...getAgentImageConfig(),
+    machineType: `${kind}-standard-${cores}`,
+    ...additionalProps,
+  };
+};
+
+export { getAgentImageConfig, expandAgentQueue };

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -16,27 +16,10 @@ import { BuildkiteClient, BuildkiteStep } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
 import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
-import { getAgentImageConfig } from '#pipeline-utils';
+import { expandAgentQueue } from '#pipeline-utils';
 
 type RunGroup = TestGroupRunOrderResponse['types'][0];
 
-// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
-/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
-const getAgentRule = (queueName: string = 'n2-4-spot') => {
-  if (process.env?.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld') {
-    const [kind, cores, spot] = queueName.split('-');
-    return {
-      ...getAgentImageConfig(),
-      machineType: `${kind}-standard-${cores}`,
-      preemptible: spot === 'spot',
-    };
-  } else {
-    return {
-      queue: queueName,
-    };
-  }
-};
-
 const getRequiredEnv = (name: string) => {
   const value = process.env[name];
   if (typeof value !== 'string' || !value) {
@@ -436,7 +419,7 @@ export async function pickTestGroupRunOrder() {
             parallelism: unit.count,
             timeout_in_minutes: 120,
             key: 'jest',
-            agents: getAgentRule('n2-4-spot'),
+            agents: expandAgentQueue('n2-4-spot'),
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },
@@ -454,7 +437,7 @@ export async function pickTestGroupRunOrder() {
             parallelism: integration.count,
             timeout_in_minutes: 120,
             key: 'jest-integration',
-            agents: getAgentRule('n2-4-spot'),
+            agents: expandAgentQueue('n2-4-spot'),
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },
@@ -488,7 +471,7 @@ export async function pickTestGroupRunOrder() {
                   label: title,
                   command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
                   timeout_in_minutes: 90,
-                  agents: getAgentRule(queue),
+                  agents: expandAgentQueue(queue),
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,
                     ...FTR_EXTRA_ARGS,

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -7,7 +7,7 @@
  */
 
 import { groups } from './groups.json';
-import { BuildkiteStep } from '#pipeline-utils';
+import { BuildkiteStep, expandAgentQueue } from '#pipeline-utils';
 
 const configJson = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG;
 if (!configJson) {
@@ -32,34 +32,6 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
-// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
-/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
-const getAgentRule = (queueName: string = 'n2-4-spot') => {
-  if (
-    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
-    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
-  ) {
-    const [kind, cores, addition] = queueName.split('-');
-    const additionalProps =
-      {
-        spot: { preemptible: true },
-        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
-      }[addition] || {};
-
-    return {
-      provider: 'gcp',
-      image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-prod',
-      machineType: `${kind}-standard-${cores}`,
-      ...additionalProps,
-    };
-  } else {
-    return {
-      queue: queueName,
-    };
-  }
-};
-
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -150,7 +122,7 @@ const pipeline = {
 steps.push({
   command: '.buildkite/scripts/steps/build_kibana.sh',
   label: 'Build Kibana Distribution and Plugins',
-  agents: getAgentRule('c2-8'),
+  agents: expandAgentQueue('c2-8'),
   key: 'build',
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@@ -173,7 +145,7 @@ for (const testSuite of testSuites) {
       concurrency,
       concurrency_group: process.env.UUID,
       concurrency_method: 'eager',
-      agents: getAgentRule('n2-4-spot'),
+      agents: expandAgentQueue('n2-4-spot'),
       depends_on: 'build',
       timeout_in_minutes: 150,
       cancel_on_build_failing: true,
@@ -197,7 +169,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: getAgentRule(agentQueue),
+        agents: expandAgentQueue(agentQueue),
         key: `cypress-suite-${suiteIndex++}`,
         depends_on: 'build',
         timeout_in_minutes: 150,
@@ -233,7 +205,7 @@ pipeline.steps.push({
 pipeline.steps.push({
   command: 'ts-node .buildkite/pipelines/flaky_tests/post_stats_on_pr.ts',
   label: 'Post results on Github pull request',
-  agents: getAgentRule('n2-4-spot'),
+  agents: expandAgentQueue('n2-4-spot'),
   timeout_in_minutes: 15,
   retry: {
     automatic: [{ exit_status: '-1', limit: 3 }],

.buildkite/pipelines/on_merge.yml

@@ -16,28 +16,6 @@ steps:
           limit: 1
   - wait
 
-  - label: 'Triggering changes-based pipelines'
-    branches: main
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-2
-    # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15
-    plugins:
-      - chronotc/monorepo-diff#v2.0.4:
-          watch:
-            - path:
-                - 'versions.json'
-              config:
-                command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main'
-                label: 'Trigger pipeline deploy'
-                agents:
-                  image: family/kibana-ubuntu-2004
-                  imageProject: elastic-images-prod
-                  provider: gcp
-                  machineType: n2-standard-2
-
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Build Kibana Distribution and Plugins
     agents:

.buildkite/pull_requests.json

@@ -45,52 +45,6 @@
         "/__snapshots__/",
         "\\.test\\.(ts|tsx|js|jsx)"
       ]
-    },
-    {
-      "repoOwner": "elastic",
-      "repoName": "kibana",
-      "pipelineSlug": "kibana-kme-test",
-
-      "enabled": true,
-      "allow_org_users": true,
-      "allowed_repo_permissions": ["admin", "write"],
-      "set_commit_status": true,
-      "commit_status_context": "kibana-ci-test",
-      "build_on_commit": true,
-      "build_on_comment": false,
-      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
-      "skip_ci_labels": [],
-      "labels": ["kme-test"],
-      "skip_target_branches": ["6.8", "7.11", "7.12"],
-      "enable_skippable_commits": true,
-      "skip_ci_on_only_changed": [
-        "^dev_docs/",
-        "^docs/",
-        "^rfcs/",
-        "^\\.github/",
-        "\\.md$",
-        "\\.mdx$",
-        "^api_docs/.+\\.devdocs\\.json$",
-        "^\\.backportrc\\.json$",
-        "^nav-kibana-dev\\.docnav\\.json$",
-        "^src/dev/prs/kibana_qa_pr_list\\.json$",
-        "^\\.buildkite/pull_requests\\.json$",
-        "^\\.catalog-info\\.yaml$"
-      ],
-      "always_require_ci_on_changed": [
-        "^docs/developer/plugin-list.asciidoc$",
-        "^\\.github/CODEOWNERS$",
-        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
-      ],
-      "kibana_versions_check": true,
-      "kibana_build_reuse": true,
-      "kibana_build_reuse_pipeline_slugs": ["kibana-kme-test", "kibana-on-merge"],
-      "kibana_build_reuse_regexes": [
-        "^test/",
-        "^x-pack/test/",
-        "/__snapshots__/",
-        "\\.test\\.(ts|tsx|js|jsx)"
-      ]
     }
   ]
 }

.buildkite/scripts/common/vault_fns.sh

@@ -1,17 +1,9 @@
 #!/bin/bash
 
-# TODO: rewrite after https://github.com/elastic/kibana-operations/issues/15 is done
 export LEGACY_VAULT_ADDR="https://secrets.elastic.co:8200"
-if [[ "${VAULT_ADDR:-}" == "$LEGACY_VAULT_ADDR" ]]; then
+export VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
-  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
+export VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
-  VAULT_KV_PREFIX="secret/kibana-issues/dev"
+export IS_LEGACY_VAULT_ADDR=false
-  IS_LEGACY_VAULT_ADDR=true
-else
-  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
-  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
-  IS_LEGACY_VAULT_ADDR=false
-fi
-export IS_LEGACY_VAULT_ADDR
 
 retry() {
   local retries=$1; shift

catalog-info.yaml

@@ -97,34 +97,6 @@ spec:
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
 apiVersion: backstage.io/v1alpha1
 kind: Resource
-metadata:
-  name: buildkite-pipeline-kibana-kme-test
-spec:
-  implementation:
-    apiVersion: buildkite.elastic.dev/v1
-    kind: Pipeline
-    metadata:
-      description: Temporary pipeline for testing Kibana KME work
-      name: kibana-kme-test
-    spec:
-      pipeline_file: .buildkite/scripts/pipelines/pull_request/pipeline.sh
-      provider_settings:
-        build_branches: false
-        build_pull_requests: true
-        publish_commit_status: false
-        trigger_mode: none
-      repository: elastic/kibana
-      teams:
-        kibana-operations:
-          access_level: MANAGE_BUILD_AND_READ
-        everyone:
-          access_level: READ_ONLY
-  owner: group:kibana-operations
-  type: buildkite-pipeline
----
-# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
-apiVersion: backstage.io/v1alpha1
-kind: Resource
 metadata:
   name: buildkite-pipeline-kibana-sonarqube
   description: Run a SonarQube scan