[CI] Remove kme leftovers (take 2) (#187947)

## Summary Retries #187762 again. There was a partially removed step in the previous attempt, that's now fully removed.
2025-04-23 17:28:26 -04:00 · 2024-07-10 16:01:06 +02:00 · 2024-07-10 16:01:06 +02:00 · ba3f83cd3d
commit ba3f83cd3d
parent 209b0c52cb
7 changed files with 28 additions and 162 deletions
--- a/.buildkite/pipeline-utils/agent_images.ts
+++ b/.buildkite/pipeline-utils/agent_images.ts
@ -52,4 +52,19 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | AgentImageCo
  return config;
 }

-export { getAgentImageConfig };
+const expandAgentQueue = (queueName: string = 'n2-4-spot') => {
+  const [kind, cores, addition] = queueName.split('-');
+  const additionalProps =
+    {
+      spot: { preemptible: true },
+      virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+    }[addition] || {};
+
+  return {
+    ...getAgentImageConfig(),
+    machineType: `${kind}-standard-${cores}`,
+    ...additionalProps,
+  };
+};
+
+export { getAgentImageConfig, expandAgentQueue };
--- a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
+++ b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
@ -16,27 +16,10 @@ import { BuildkiteClient, BuildkiteStep } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';

 import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
-import { getAgentImageConfig } from '#pipeline-utils';
+import { expandAgentQueue } from '#pipeline-utils';

 type RunGroup = TestGroupRunOrderResponse['types'][0];

-// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
-/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
-const getAgentRule = (queueName: string = 'n2-4-spot') => {
-  if (process.env?.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld') {
-    const [kind, cores, spot] = queueName.split('-');
-    return {
-      ...getAgentImageConfig(),
-      machineType: `${kind}-standard-${cores}`,
-      preemptible: spot === 'spot',
-    };
-  } else {
-    return {
-      queue: queueName,
-    };
-  }
-};
-
 const getRequiredEnv = (name: string) => {
  const value = process.env[name];
  if (typeof value !== 'string' || !value) {
@ -436,7 +419,7 @@ export async function pickTestGroupRunOrder() {
            parallelism: unit.count,
            timeout_in_minutes: 120,
            key: 'jest',
-            agents: getAgentRule('n2-4-spot'),
+            agents: expandAgentQueue('n2-4-spot'),
            retry: {
              automatic: [
                { exit_status: '-1', limit: 3 },
@ -454,7 +437,7 @@ export async function pickTestGroupRunOrder() {
            parallelism: integration.count,
            timeout_in_minutes: 120,
            key: 'jest-integration',
-            agents: getAgentRule('n2-4-spot'),
+            agents: expandAgentQueue('n2-4-spot'),
            retry: {
              automatic: [
                { exit_status: '-1', limit: 3 },
@ -488,7 +471,7 @@ export async function pickTestGroupRunOrder() {
                  label: title,
                  command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
                  timeout_in_minutes: 90,
-                  agents: getAgentRule(queue),
+                  agents: expandAgentQueue(queue),
                  env: {
                    FTR_CONFIG_GROUP_KEY: key,
                    ...FTR_EXTRA_ARGS,
--- a/.buildkite/pipelines/flaky_tests/pipeline.ts
+++ b/.buildkite/pipelines/flaky_tests/pipeline.ts
@ -7,7 +7,7 @@
 */

 import { groups } from './groups.json';
-import { BuildkiteStep } from '#pipeline-utils';
+import { BuildkiteStep, expandAgentQueue } from '#pipeline-utils';

 const configJson = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG;
 if (!configJson) {
@ -32,34 +32,6 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;

-// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
-/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
-const getAgentRule = (queueName: string = 'n2-4-spot') => {
-  if (
-    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
-    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
-  ) {
-    const [kind, cores, addition] = queueName.split('-');
-    const additionalProps =
-      {
-        spot: { preemptible: true },
-        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
-      }[addition] || {};
-
-    return {
-      provider: 'gcp',
-      image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-prod',
-      machineType: `${kind}-standard-${cores}`,
-      ...additionalProps,
-    };
-  } else {
-    return {
-      queue: queueName,
-    };
-  }
-};
-
 function getTestSuitesFromJson(json: string) {
  const fail = (errorMsg: string) => {
    console.error('+++ Invalid test config provided');
@ -150,7 +122,7 @@ const pipeline = {
 steps.push({
  command: '.buildkite/scripts/steps/build_kibana.sh',
  label: 'Build Kibana Distribution and Plugins',
-  agents: getAgentRule('c2-8'),
+  agents: expandAgentQueue('c2-8'),
  key: 'build',
  if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@ -173,7 +145,7 @@ for (const testSuite of testSuites) {
      concurrency,
      concurrency_group: process.env.UUID,
      concurrency_method: 'eager',
-      agents: getAgentRule('n2-4-spot'),
+      agents: expandAgentQueue('n2-4-spot'),
      depends_on: 'build',
      timeout_in_minutes: 150,
      cancel_on_build_failing: true,
@ -197,7 +169,7 @@ for (const testSuite of testSuites) {
      steps.push({
        command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
        label: group.name,
-        agents: getAgentRule(agentQueue),
+        agents: expandAgentQueue(agentQueue),
        key: `cypress-suite-${suiteIndex++}`,
        depends_on: 'build',
        timeout_in_minutes: 150,
@ -233,7 +205,7 @@ pipeline.steps.push({
 pipeline.steps.push({
  command: 'ts-node .buildkite/pipelines/flaky_tests/post_stats_on_pr.ts',
  label: 'Post results on Github pull request',
-  agents: getAgentRule('n2-4-spot'),
+  agents: expandAgentQueue('n2-4-spot'),
  timeout_in_minutes: 15,
  retry: {
    automatic: [{ exit_status: '-1', limit: 3 }],
--- a/.buildkite/pipelines/on_merge.yml
+++ b/.buildkite/pipelines/on_merge.yml
@ -16,28 +16,6 @@ steps:
          limit: 1
  - wait

-  - label: 'Triggering changes-based pipelines'
-    branches: main
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-2
-    # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15
-    plugins:
-      - chronotc/monorepo-diff#v2.0.4:
-          watch:
-            - path:
-                - 'versions.json'
-              config:
-                command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main'
-                label: 'Trigger pipeline deploy'
-                agents:
-                  image: family/kibana-ubuntu-2004
-                  imageProject: elastic-images-prod
-                  provider: gcp
-                  machineType: n2-standard-2
-
  - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
    label: Build Kibana Distribution and Plugins
    agents:
--- a/.buildkite/pull_requests.json
+++ b/.buildkite/pull_requests.json
@ -45,52 +45,6 @@
        "/__snapshots__/",
        "\\.test\\.(ts|tsx|js|jsx)"
      ]
-    },
-    {
-      "repoOwner": "elastic",
-      "repoName": "kibana",
-      "pipelineSlug": "kibana-kme-test",
-
-      "enabled": true,
-      "allow_org_users": true,
-      "allowed_repo_permissions": ["admin", "write"],
-      "set_commit_status": true,
-      "commit_status_context": "kibana-ci-test",
-      "build_on_commit": true,
-      "build_on_comment": false,
-      "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
-      "skip_ci_labels": [],
-      "labels": ["kme-test"],
-      "skip_target_branches": ["6.8", "7.11", "7.12"],
-      "enable_skippable_commits": true,
-      "skip_ci_on_only_changed": [
-        "^dev_docs/",
-        "^docs/",
-        "^rfcs/",
-        "^\\.github/",
-        "\\.md$",
-        "\\.mdx$",
-        "^api_docs/.+\\.devdocs\\.json$",
-        "^\\.backportrc\\.json$",
-        "^nav-kibana-dev\\.docnav\\.json$",
-        "^src/dev/prs/kibana_qa_pr_list\\.json$",
-        "^\\.buildkite/pull_requests\\.json$",
-        "^\\.catalog-info\\.yaml$"
-      ],
-      "always_require_ci_on_changed": [
-        "^docs/developer/plugin-list.asciidoc$",
-        "^\\.github/CODEOWNERS$",
-        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
-      ],
-      "kibana_versions_check": true,
-      "kibana_build_reuse": true,
-      "kibana_build_reuse_pipeline_slugs": ["kibana-kme-test", "kibana-on-merge"],
-      "kibana_build_reuse_regexes": [
-        "^test/",
-        "^x-pack/test/",
-        "/__snapshots__/",
-        "\\.test\\.(ts|tsx|js|jsx)"
-      ]
    }
  ]
 }
--- a/.buildkite/scripts/common/vault_fns.sh
+++ b/.buildkite/scripts/common/vault_fns.sh
@ -1,17 +1,9 @@
 #!/bin/bash

-# TODO: rewrite after https://github.com/elastic/kibana-operations/issues/15 is done
 export LEGACY_VAULT_ADDR="https://secrets.elastic.co:8200"
-if [[ "${VAULT_ADDR:-}" == "$LEGACY_VAULT_ADDR" ]]; then
-  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
-  VAULT_KV_PREFIX="secret/kibana-issues/dev"
-  IS_LEGACY_VAULT_ADDR=true
-else
-  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
-  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
-  IS_LEGACY_VAULT_ADDR=false
-fi
-export IS_LEGACY_VAULT_ADDR
+export VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
+export VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
+export IS_LEGACY_VAULT_ADDR=false

 retry() {
  local retries=$1; shift
--- a/catalog-info.yaml
+++ b/catalog-info.yaml
@ -97,34 +97,6 @@ spec:
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
 apiVersion: backstage.io/v1alpha1
 kind: Resource
-metadata:
-  name: buildkite-pipeline-kibana-kme-test
-spec:
-  implementation:
-    apiVersion: buildkite.elastic.dev/v1
-    kind: Pipeline
-    metadata:
-      description: Temporary pipeline for testing Kibana KME work
-      name: kibana-kme-test
-    spec:
-      pipeline_file: .buildkite/scripts/pipelines/pull_request/pipeline.sh
-      provider_settings:
-        build_branches: false
-        build_pull_requests: true
-        publish_commit_status: false
-        trigger_mode: none
-      repository: elastic/kibana
-      teams:
-        kibana-operations:
-          access_level: MANAGE_BUILD_AND_READ
-        everyone:
-          access_level: READ_ONLY
-  owner: group:kibana-operations
-  type: buildkite-pipeline
---
-# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
-apiVersion: backstage.io/v1alpha1
-kind: Resource
 metadata:
  name: buildkite-pipeline-kibana-sonarqube
  description: Run a SonarQube scan