Add pipeline for serverless emergency release quality gate (#187251)

## Summary This PR adds separately quality gate pipelines for the emergency release process. More details in the original PR #186833, which is split into the creation of the new pipeline (this PR) and moving existing pipelines from `catalog-info.yaml` to `.buildkite/pipeline-resource-definitions` (#187253).
2025-04-23 09:19:04 -04:00 · 2024-07-02 13:15:31 +02:00 · 2024-07-02 13:15:31 +02:00 · 013bda4e60
commit 013bda4e60
parent 8bf9aa56b4
7 changed files with 162 additions and 1 deletions
--- a/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
+++ b/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
@ -0,0 +1,33 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-tests-emergency-pipeline
+  description: Definition of the kibana pipeline
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-tests-emergency
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-tech-leads
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-tests-emergency
+      description: Pipeline that tests the service integration in various environments
+    spec:
+      repository: elastic/kibana
+      pipeline_file: ./.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        cloud-tooling:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
--- a/.buildkite/pipeline-resource-definitions/locations.yml
+++ b/.buildkite/pipeline-resource-definitions/locations.yml
@ -27,6 +27,7 @@ spec:
    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml
    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml
    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml
--- a/.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml
+++ b/.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml
@ -0,0 +1,33 @@
+# This pipeline serves as the entry point for your service's quality gates definitions. When
+# properly configured, it will be invoked automatically as part of the automated
+# promotion process once a new version was rolled out in one of the various cloud stages.
+#
+# The updated environment is provided via ENVIRONMENT variable. The seedling
+# step will branch and execute pipeline snippets at the following location:
+# pipeline.tests-qa.yaml
+# pipeline.tests-staging.yaml
+# pipeline.tests-production.yaml
+#
+# Docs: https://docs.elastic.dev/serverless/qualitygates
+
+agents:
+  cpu: 2
+  ephemeralStorage: "20G"
+  memory: "8G"
+
+env:
+  SKIP_NODE_SETUP: true
+  TEAM_CHANNEL: "#kibana-mission-control"
+  ENVIRONMENT: ${ENVIRONMENT?}
+
+steps:
+  - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"
+    env:
+      QG_PIPELINE_LOCATION: ".buildkite/pipelines/quality-gates/emergency"
+    command: "make -C /agent run-environment-tests" # will trigger https://buildkite.com/elastic/kibana-tests-emergency
+    agents:
+      image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.4"
+
+notify:
+  - slack: "${TEAM_CHANNEL?}"
+    if: build.branch == "main" && build.state == "failed"
--- a/.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml
+++ b/.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml
@ -0,0 +1,37 @@
+# These pipeline steps constitute the quality gate for your service within the production environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":kibana: SLO check"
+    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+      env:
+        TARGET_ENV: production
+        CHECK_SLO: true
+        CHECK_SLO_TAG: kibana
+        CHECK_SLO_WAITING_PERIOD: 15m
+        CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
+    soft_fail: true
+
+  - label: ":rocket: control-plane e2e tests"
+    if: build.env("ENVIRONMENT") == "production-canary"
+    trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+
+  - label: ":cookie: 24h bake time before continuing promotion"
+    if: build.env("ENVIRONMENT") == "production-canary"
+    command: "sleep 86400"
+    soft_fail:
+      # A manual cancel of that step produces return code 255.
+      # We're treating this case as a soft fail to allow manual bake time skipping.
+      # To stop the promotion entirely, instead click the "Cancel" button at the top of the page
+      - exit_status: 255
+    agents:
+      # How long can this agent live for in minutes - 25 hours
+      instanceMaxAge: 1500
--- a/.buildkite/pipelines/quality-gates/emergency/pipeline.tests-qa.yaml
+++ b/.buildkite/pipelines/quality-gates/emergency/pipeline.tests-qa.yaml
@ -0,0 +1,12 @@
+# These pipeline steps constitute the quality gate for your service within the QA environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":rocket: control-plane e2e tests"
+    trigger: "ess-k8s-qa-e2e-tests-daily" # https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily
+    build:
+      env:
+        REGION_ID: aws-eu-west-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
--- a/.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml
+++ b/.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml
@ -0,0 +1,45 @@
+# These pipeline steps constitute the quality gate for your service within the staging environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":rocket: control-plane e2e tests"
+    trigger: "ess-k8s-staging-e2e-tests" # https://buildkite.com/elastic/ess-k8s-staging-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
+  - label: ":kibana: Kibana Serverless Tests for ${ENVIRONMENT}"
+    trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
+    soft_fail: true # Remove when tests stabilize
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+        EC_ENV: staging
+        EC_REGION: aws-us-east-1
+        RETRY_TESTS_ON_FAIL: "true"
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
+  - label: ":rocket: Fleet synthetic monitor to check the long standing project"
+    trigger: "serverless-quality-gates"
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+      env:
+        TARGET_ENV: staging
+        CHECK_SYNTHETICS: true
+        CHECK_SYNTHETICS_TAG: "fleet"
+        CHECK_SYNTHETICS_MINIMUM_RUNS: 3
+        MAX_FAILURES: 2
+        CHECK_SYNTHETIC_MAX_POLL: 50
+    soft_fail: true
+
+  - wait: ~
+
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
--- a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
+++ b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
@ -21,7 +21,7 @@ steps:
        EC_REGION: aws-us-east-1
        RETRY_TESTS_ON_FAIL: "true"
      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
-  
+
  - label: ":rocket: Fleet synthetic monitor to check the long standing project"
    trigger: "serverless-quality-gates"
    build: