[8.x] [ci] Click to deploy cloud (#205623) (#208042)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[ci] Click to deploy cloud
(#205623)](https://github.com/elastic/kibana/pull/205623)

<!--- Backport version: 9.6.4 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Alex
Szabo","email":"alex.szabo@elastic.co"},"sourceCommit":{"committedDate":"2025-01-23T09:26:03Z","message":"[ci]
Click to deploy cloud (#205623)\n\n## Summary\nSimilar to
https://github.com/elastic/kibana/pull/195581\n\nAdds a pipeline that
builds Kibana and starts cloud deployment without\ngoing through the CI
test suites (as in normal pull-request pipeline\nruns). It can be useful
if a developer would like to save time/compute\non
re-building/re-testing the whole project before deploying to
the\ncloud.\n\nAdded labels (`ci:cloud-deploy / ci:cloud-redeploy`) are
required\nsimilarly to the usual CI flow.\n\nRelated to:
https://github.com/elastic/kibana-operations/issues/121","sha":"e36833b3a60b62f794f47951f5ceae842d6c44b3","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Operations","release_note:skip","v9.0.0","backport:all-open"],"title":"[ci]
Click to deploy
cloud","number":205623,"url":"https://github.com/elastic/kibana/pull/205623","mergeCommit":{"message":"[ci]
Click to deploy cloud (#205623)\n\n## Summary\nSimilar to
https://github.com/elastic/kibana/pull/195581\n\nAdds a pipeline that
builds Kibana and starts cloud deployment without\ngoing through the CI
test suites (as in normal pull-request pipeline\nruns). It can be useful
if a developer would like to save time/compute\non
re-building/re-testing the whole project before deploying to
the\ncloud.\n\nAdded labels (`ci:cloud-deploy / ci:cloud-redeploy`) are
required\nsimilarly to the usual CI flow.\n\nRelated to:
https://github.com/elastic/kibana-operations/issues/121","sha":"e36833b3a60b62f794f47951f5ceae842d6c44b3"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/205623","number":205623,"mergeCommit":{"message":"[ci]
Click to deploy cloud (#205623)\n\n## Summary\nSimilar to
https://github.com/elastic/kibana/pull/195581\n\nAdds a pipeline that
builds Kibana and starts cloud deployment without\ngoing through the CI
test suites (as in normal pull-request pipeline\nruns). It can be useful
if a developer would like to save time/compute\non
re-building/re-testing the whole project before deploying to
the\ncloud.\n\nAdded labels (`ci:cloud-deploy / ci:cloud-redeploy`) are
required\nsimilarly to the usual CI flow.\n\nRelated to:
https://github.com/elastic/kibana-operations/issues/121","sha":"e36833b3a60b62f794f47951f5ceae842d6c44b3"}}]}]
BACKPORT-->
This commit is contained in:
Alex Szabo 2025-01-24 17:13:20 +01:00 committed by GitHub
parent db7b5e27bd
commit 19d6e0c33c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 451 additions and 6 deletions

View file

@ -13,12 +13,14 @@
"globby": "^11.1.0",
"js-yaml": "^4.1.0",
"minimatch": "^5.0.1",
"minimist": "^1.2.8",
"tslib": "*"
},
"devDependencies": {
"@types/chai": "^4.3.3",
"@types/js-yaml": "^4.0.9",
"@types/minimatch": "^3.0.5",
"@types/minimist": "^1.2.5",
"@types/mocha": "^10.0.1",
"@types/node": "^15.12.2",
"chai": "^4.3.10",
@ -365,6 +367,12 @@
"integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==",
"dev": true
},
"node_modules/@types/minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==",
"dev": true
},
"node_modules/@types/mocha": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.1.tgz",
@ -1224,6 +1232,14 @@
"node": ">=10"
}
},
"node_modules/minimist": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/minipass": {
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
@ -2226,6 +2242,12 @@
"integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==",
"dev": true
},
"@types/minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==",
"dev": true
},
"@types/mocha": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.1.tgz",
@ -2841,6 +2863,11 @@
"brace-expansion": "^2.0.1"
}
},
"minimist": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="
},
"minipass": {
"version": "7.1.2",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",

View file

@ -15,12 +15,14 @@
"globby": "^11.1.0",
"js-yaml": "^4.1.0",
"minimatch": "^5.0.1",
"minimist": "^1.2.8",
"tslib": "*"
},
"devDependencies": {
"@types/chai": "^4.3.3",
"@types/js-yaml": "^4.0.9",
"@types/minimatch": "^3.0.5",
"@types/minimist": "^1.2.5",
"@types/mocha": "^10.0.1",
"@types/node": "^15.12.2",
"chai": "^4.3.10",

View file

@ -0,0 +1,45 @@
# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
apiVersion: backstage.io/v1alpha1
kind: Resource
metadata:
name: bk-kibana-deploy-cloud-from-pr
description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR'
links:
- url: 'https://buildkite.com/elastic/kibana-deploy-cloud-from-pr'
title: Pipeline link
spec:
type: buildkite-pipeline
system: buildkite
owner: 'group:kibana-operations'
implementation:
apiVersion: buildkite.elastic.dev/v1
kind: Pipeline
metadata:
name: kibana / deploy cloud from PR
description: 'Builds Kibana and initiates a Kibana cloud deployment from a PR'
spec:
env:
ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'
allow_rebuilds: false
branch_configuration: main
default_branch: main
repository: elastic/kibana
pipeline_file: .buildkite/pipelines/build_pr_and_deploy_cloud.yml
provider_settings:
build_pull_requests: true
prefix_pull_request_fork_branch_names: false
skip_pull_request_builds_for_existing_commits: true
trigger_mode: none
cancel_intermediate_builds: true
teams:
kibana-operations:
access_level: MANAGE_BUILD_AND_READ
appex-qa:
access_level: MANAGE_BUILD_AND_READ
kibana-tech-leads:
access_level: MANAGE_BUILD_AND_READ
everyone:
access_level: BUILD_AND_READ
tags:
- kibana

View file

@ -9,6 +9,8 @@
import { Octokit, RestEndpointMethodTypes } from '@octokit/rest';
export const KIBANA_COMMENT_SIGIL = 'kbn-message-context';
const github = new Octokit({
auth: process.env.GITHUB_TOKEN,
});
@ -93,6 +95,76 @@ export const doAnyChangesMatch = async (
return anyFilesMatchRequired;
};
export function addComment(
comment: string,
owner = process.env.GITHUB_PR_BASE_OWNER,
repo = process.env.GITHUB_PR_BASE_REPO,
prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
) {
if (!owner || !repo || !prNumber) {
throw Error(
"Couldn't retrieve Github PR info from environment variables in order to add a comment"
);
}
return github.issues.createComment({
owner,
repo,
issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
body: comment,
});
}
export async function upsertComment(
messageOpts: {
commentBody: string;
commentContext: string;
clearPrevious: boolean;
},
owner = process.env.GITHUB_PR_BASE_OWNER,
repo = process.env.GITHUB_PR_BASE_REPO,
prNumber: undefined | string | number = process.env.GITHUB_PR_NUMBER
) {
const { commentBody, commentContext, clearPrevious } = messageOpts;
if (!owner || !repo || !prNumber) {
throw Error(
"Couldn't retrieve Github PR info from environment variables in order to add a comment"
);
}
if (!commentContext) {
throw Error('Comment context is required when updating a comment');
}
const commentMarker = `<!-- ${KIBANA_COMMENT_SIGIL}:${commentContext} -->`;
const body = `${commentMarker}\n${commentBody}`;
const existingComment = (
await github.paginate(github.issues.listComments, {
owner,
repo,
issue_number: typeof prNumber === 'number' ? prNumber : parseInt(prNumber, 10),
})
).find((comment) => comment.body?.includes(commentMarker));
if (!existingComment) {
return addComment(body, owner, repo, prNumber);
} else if (clearPrevious) {
await github.issues.deleteComment({
owner,
repo,
comment_id: existingComment.id,
});
return addComment(body, owner, repo, prNumber);
} else {
return github.issues.updateComment({
owner,
repo,
comment_id: existingComment.id,
body,
});
}
}
export function getGithubClient() {
return github;
}

View file

@ -0,0 +1,82 @@
env:
ELASTIC_PR_COMMENTS_ENABLED: 'true'
ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-cloud-from-pr
steps:
- group: 'Cloud Deployment'
if: "build.env('GITHUB_PR_LABELS') =~ /(ci:cloud-deploy|ci:cloud-redeploy)/"
steps:
- command: .buildkite/scripts/lifecycle/pre_build.sh
label: Pre-Build
timeout_in_minutes: 10
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: |
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
--message "PR Cloud deployment started at: $BUILDKITE_BUILD_URL" \
--context "cloud-deploy-job" \
--clear-previous
label: Comment with job URL
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
timeout_in_minutes: 5
- command: .buildkite/scripts/steps/build_kibana.sh
label: Build Kibana Distribution
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-8
preemptible: true
diskSizeGb: 125
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
timeout_in_minutes: 90
retry:
automatic:
- exit_status: '-1'
limit: 3
- wait: ~
- command: .buildkite/scripts/steps/cloud/build_and_deploy.sh
label: 'Build and Deploy to Cloud'
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
preemptible: true
timeout_in_minutes: 30
retry:
automatic:
- exit_status: '-1'
limit: 3
- wait: ~
- command: |
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
--message "Cloud deployment initiated, see credentials at: $BUILDKITE_BUILD_URL" \
--context "cloud-deploy-job" \
--clear-previous
label: Comment with job URL
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
timeout_in_minutes: 5

View file

@ -0,0 +1,78 @@
env:
ELASTIC_PR_COMMENTS_ENABLED: 'true'
ELASTIC_GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true'
GITHUB_BUILD_COMMIT_STATUS_CONTEXT: kibana-deploy-project-from-pr
steps:
- group: 'Project Deployment'
if: "build.env('GITHUB_PR_LABELS') =~ /ci:project-deploy-(elasticsearch|observability|security)/"
steps:
- command: .buildkite/scripts/lifecycle/pre_build.sh
label: Pre-Build
timeout_in_minutes: 10
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
retry:
automatic:
- exit_status: '*'
limit: 1
- command: |
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
--message "PR Project deployment started at: $BUILDKITE_BUILD_URL" \
--context "project-deploy-job" \
--clear-previous
label: Comment with job URL
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
timeout_in_minutes: 5
- wait: ~
- command: .buildkite/scripts/steps/artifacts/docker_image.sh
label: 'Build Project Image'
key: build_project_image
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-16
preemptible: true
timeout_in_minutes: 60
retry:
automatic:
- exit_status: '-1'
limit: 3
- wait: ~
- command: .buildkite/scripts/steps/serverless/deploy.sh
label: 'Deploy Project'
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-4
preemptible: true
timeout_in_minutes: 10
- wait: ~
- command: |
ts-node .buildkite/scripts/lifecycle/comment_on_pr.ts \
--message "Project deployed, see credentials at: $BUILDKITE_BUILD_URL" \
--context "project-deploy-job" \
--clear-previous
label: Comment with job URL
agents:
provider: gcp
image: family/kibana-ubuntu-2004
imageProject: elastic-images-prod
machineType: n2-standard-2
timeout_in_minutes: 5

View file

@ -8,28 +8,28 @@
"enabled": true,
"allow_org_users": true,
"allowed_repo_permissions": ["admin", "write"],
"allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
"set_commit_status": true,
"commit_status_context": "kibana-ci",
"build_on_commit": true,
"build_on_comment": true,
"build_drafts": false,
"build_on_ready": true,
"trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^\\/ci$",
"always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^\\/ci$",
"skip_ci_labels": ["skip-ci"],
"skip_target_branches": ["6.8", "7.11", "7.12"],
"enable_skippable_commits": true,
"skip_ci_on_only_changed": [
"^dev_docs/",
"^docs/",
"^rfcs/",
"^\\.github/",
"\\.md$",
"\\.mdx$",
"^api_docs/.+\\.devdocs\\.json$",
"^\\.backportrc\\.json$",
"^nav-kibana-dev\\.docnav\\.json$",
"^src/dev/prs/kibana_qa_pr_list\\.json$",
"^\\.buildkite/pull_requests\\.json$"
"^\\.buildkite/pull_requests\\.json$",
"^\\.devcontainer/"
],
"always_require_ci_on_changed": [
"^docs/developer/plugin-list.asciidoc$",
@ -45,6 +45,56 @@
"/__snapshots__/",
"\\.test\\.(ts|tsx|js|jsx)"
]
},
{
"repoOwner": "elastic",
"repoName": "kibana",
"pipelineSlug": "kibana-deploy-project-from-pr",
"skip_ci_labels": [],
"enabled": true,
"allow_org_users": true,
"allowed_repo_permissions": ["admin", "write"],
"allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
"set_commit_status": true,
"commit_status_context": "kibana-deploy-project-from-pr",
"build_on_commit": false,
"build_on_comment": true,
"build_drafts": false,
"trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:project))$",
"kibana_versions_check": true,
"kibana_build_reuse": true,
"kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-project-from-pr"],
"kibana_build_reuse_regexes": [
"^test/",
"^x-pack/test/",
"/__snapshots__/",
"\\.test\\.(ts|tsx|js|jsx)"
]
},
{
"repoOwner": "elastic",
"repoName": "kibana",
"pipelineSlug": "kibana-deploy-cloud-from-pr",
"skip_ci_labels": [],
"enabled": true,
"allow_org_users": true,
"allowed_repo_permissions": ["admin", "write"],
"allowed_list": ["elastic-vault-github-plugin-prod[bot]"],
"set_commit_status": true,
"commit_status_context": "kibana-deploy-cloud-from-pr",
"build_on_commit": false,
"build_on_comment": true,
"build_drafts": false,
"trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:deploy)\\W+(?:cloud))$",
"kibana_versions_check": true,
"kibana_build_reuse": true,
"kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge", "kibana-deploy-cloud-from-pr"],
"kibana_build_reuse_regexes": [
"^test/",
"^x-pack/test/",
"/__snapshots__/",
"\\.test\\.(ts|tsx|js|jsx)"
]
}
]
}

View file

@ -0,0 +1,89 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import parseArgs from 'minimist';
import { upsertComment, addComment } from '#pipeline-utils';
const ALLOWED_ENV_VARS = [
'BUILDKITE_BRANCH',
'BUILDKITE_BUILD_ID',
'BUILDKITE_BUILD_NUMBER',
'BUILDKITE_BUILD_URL',
'BUILDKITE_COMMIT',
'BUILDKITE_PIPELINE_NAME',
'BUILDKITE_PIPELINE_SLUG',
'GITHUB_PR_BASE_OWNER',
'GITHUB_PR_BASE_REPO',
'GITHUB_PR_BRANCH',
'GITHUB_PR_HEAD_SHA',
'GITHUB_PR_HEAD_USER',
'GITHUB_PR_LABELS',
'GITHUB_PR_NUMBER',
'GITHUB_PR_OWNER',
'GITHUB_PR_REPO',
'GITHUB_PR_TARGET_BRANCH',
'GITHUB_PR_TRIGGERED_SHA',
'GITHUB_PR_TRIGGER_USER',
'GITHUB_PR_USER',
];
export function commentOnPR({
messageTemplate,
context,
clearPrevious,
}: {
messageTemplate: string;
context?: string;
clearPrevious: boolean;
}) {
const message = messageTemplate.replace(/\${([^}]+)}/g, (_, envVar) => {
if (ALLOWED_ENV_VARS.includes(envVar)) {
return process.env[envVar] || '';
} else {
return '${' + envVar + '}';
}
});
if (context) {
return upsertComment({ commentBody: message, commentContext: context, clearPrevious });
} else {
return addComment(message);
}
}
if (require.main === module) {
const args = parseArgs<{
context?: string;
message: string;
'clear-previous'?: boolean | string;
}>(process.argv.slice(2), {
string: ['message', 'context'],
boolean: ['clear-previous'],
});
if (!args.message) {
throw new Error(
`No message template provided for ${process.argv[1]}, use --message to provide one.`
);
} else {
console.log(`Using message template: ${args.message}`);
}
commentOnPR({
messageTemplate: args.message,
context: args.context,
clearPrevious:
typeof args['clear-previous'] === 'string'
? !!args['clear-previous'].match(/(1|true)/i)
: !!args['clear-previous'],
}).catch((error) => {
console.error(error);
process.exit(1);
});
}

View file

@ -33,8 +33,8 @@ if (!prConfig) {
}
const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';
const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
const REQUIRED_PATHS = prConfig.always_require_ci_on_changed!.map((r) => new RegExp(r, 'i'));
const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed!.map((r) => new RegExp(r, 'i'));
const getPipeline = (filename: string, removeSteps = true) => {
const str = fs.readFileSync(filename).toString();