mirror of
https://github.com/elastic/kibana.git
synced 2025-04-24 01:38:56 -04:00
[ci] extract apm traces after in single user benchmarking pipeline (#130777)
* [ci] extract apm traces after in single user benchmarking pipeline * add performance-testing-dataset-extractor dependency * use BUILD_ID for queries * bump version for extractor, update script * bump extractor version * fix path * upload artifacts to the public bucket
This commit is contained in:
Dzmitry Lemechko
GitHub
parent
880c3218b4
commit
1d1e1f6430
5 changed files with 102 additions and 2 deletions
|
|
@ -17,6 +17,13 @@ steps:
|
|||
agents:
|
||||
queue: kb-static-ubuntu
|
||||
depends_on: build
|
||||
key: tests
|
||||
|
||||
- label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
|
||||
command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
|
||||
agents:
|
||||
queue: n2-2
|
||||
depends_on: tests
|
||||
|
||||
- wait: ~
|
||||
continue_on_failure: true
|
||||
|
|
|
|||
36
.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
Executable file
36
.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source .buildkite/scripts/common/util.sh
|
||||
|
||||
USER_FROM_VAULT="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/apm_parser_performance)"
|
||||
PASS_FROM_VAULT="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/apm_parser_performance)"
|
||||
ES_SERVER_URL="https://kibana-ops-e2e-perf.es.us-central1.gcp.cloud.es.io:9243"
|
||||
BUILD_ID=${BUILDKITE_BUILD_ID}
|
||||
GCS_BUCKET="gs://kibana-performance/scalability-tests"
|
||||
|
||||
.buildkite/scripts/bootstrap.sh
|
||||
|
||||
echo "--- Extract APM metrics"
|
||||
journeys=("login" "ecommerce_dashboard" "flight_dashboard" "web_logs_dashboard" "promotion_tracking_dashboard" "many_fields_discover")
|
||||
|
||||
for i in "${journeys[@]}"; do
|
||||
JOURNEY_NAME="${i}"
|
||||
echo "Looking for JOURNEY=${JOURNEY_NAME} and BUILD_ID=${BUILD_ID} in APM traces"
|
||||
|
||||
./node_modules/.bin/performance-testing-dataset-extractor -u "${USER_FROM_VAULT}" -p "${PASS_FROM_VAULT}" -c "${ES_SERVER_URL}" -b "${BUILD_ID}" -n "${JOURNEY_NAME}"
|
||||
done
|
||||
|
||||
# archive json files with traces and upload as build artifacts
|
||||
echo "--- Upload Kibana build, plugins and scalability traces to the public bucket"
|
||||
mkdir "${BUILD_ID}"
|
||||
tar -czf "${BUILD_ID}/scalability_traces.tar.gz" output
|
||||
buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
|
||||
buildkite-agent artifact download kibana-default.tar.gz ./"${BUILD_ID}"
|
||||
buildkite-agent artifact download kibana-default-plugins.tar.gz ./"${BUILD_ID}"
|
||||
echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
|
||||
gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
|
||||
echo "--- Update reference to the latest CI build"
|
||||
echo "${BUILD_ID}" > LATEST
|
||||
gsutil cp LATEST "${GCS_BUCKET}"
|
||||
|
|
@ -474,6 +474,7 @@
|
|||
"@elastic/eslint-plugin-eui": "0.0.2",
|
||||
"@elastic/github-checks-reporter": "0.0.20b3",
|
||||
"@elastic/makelogs": "^6.0.0",
|
||||
"@elastic/performance-testing-dataset-extractor": "^0.0.3",
|
||||
"@elastic/synthetics": "^1.0.0-beta.22",
|
||||
"@emotion/babel-preset-css-prop": "^11.2.0",
|
||||
"@emotion/jest": "^11.9.0",
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ export default async function ({ readConfigFile, log }: FtrConfigProviderContext
|
|||
performancePhase: process.env.TEST_PERFORMANCE_PHASE,
|
||||
journeyName: process.env.JOURNEY_NAME,
|
||||
testJobId,
|
||||
testBuildId,
|
||||
})
|
||||
.filter(([, v]) => !!v)
|
||||
.reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), ''),
|
||||
|
|
|
|||
59
yarn.lock
59
yarn.lock
|
|
@ -1496,6 +1496,16 @@
|
|||
dependencies:
|
||||
"@elastic/ecs-helpers" "^1.1.0"
|
||||
|
||||
"@elastic/elasticsearch@7.17.0":
|
||||
version "7.17.0"
|
||||
resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.17.0.tgz#589fb219234cf1b0da23744e82b1d25e2fe9a797"
|
||||
integrity sha512-5QLPCjd0uLmLj1lSuKSThjNpq39f6NmlTy9ROLFwG5gjyTgpwSqufDeYG/Fm43Xs05uF7WcscoO7eguI3HuuYA==
|
||||
dependencies:
|
||||
debug "^4.3.1"
|
||||
hpagent "^0.1.1"
|
||||
ms "^2.1.3"
|
||||
secure-json-parse "^2.4.0"
|
||||
|
||||
"@elastic/elasticsearch@npm:@elastic/elasticsearch-canary@8.2.0-canary.2":
|
||||
version "8.2.0-canary.2"
|
||||
resolved "https://registry.yarnpkg.com/@elastic/elasticsearch-canary/-/elasticsearch-canary-8.2.0-canary.2.tgz#2513926cdbfe7c070e1fa6926f7829171b27cdba"
|
||||
|
|
@ -1618,6 +1628,19 @@
|
|||
resolved "https://registry.yarnpkg.com/@elastic/numeral/-/numeral-2.5.1.tgz#96acf39c3d599950646ef8ccfd24a3f057cf4932"
|
||||
integrity sha512-Tby6TKjixRFY+atVNeYUdGr9m0iaOq8230KTwn8BbUhkh7LwozfgKq0U98HRX7n63ZL62szl+cDKTYzh5WPCFQ==
|
||||
|
||||
"@elastic/performance-testing-dataset-extractor@^0.0.3":
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/@elastic/performance-testing-dataset-extractor/-/performance-testing-dataset-extractor-0.0.3.tgz#c9823154c1d23c0dfec86f7183a5e2327999d0ca"
|
||||
integrity sha512-ND33m4P1yOLPqnKnwWTcwDNB5dCw5NK9503e2WaZzljoy75RN9Lg5+YsQM7RFZKDs/+yNp7XRCJszeiUOcMFvg==
|
||||
dependencies:
|
||||
"@elastic/elasticsearch" "7.17.0"
|
||||
axios "^0.26.1"
|
||||
axios-curlirize "1.3.7"
|
||||
lodash "^4.17.21"
|
||||
qs "^6.10.3"
|
||||
tslib "^2.3.1"
|
||||
yargs "^17.4.0"
|
||||
|
||||
"@elastic/react-search-ui-views@1.6.0":
|
||||
version "1.6.0"
|
||||
resolved "https://registry.yarnpkg.com/@elastic/react-search-ui-views/-/react-search-ui-views-1.6.0.tgz#7211d47c29ef0636c853721491b9905ac7ae58da"
|
||||
|
|
@ -8691,6 +8714,11 @@ axe-core@^4.2.0:
|
|||
resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.3.5.tgz#78d6911ba317a8262bfee292aeafcc1e04b49cc5"
|
||||
integrity sha512-WKTW1+xAzhMS5dJsxWkliixlO/PqC4VhmO9T4juNYcaTg9jzWiJsou6m5pxWYGfigWbwzJWeFY6z47a+4neRXA==
|
||||
|
||||
axios-curlirize@1.3.7:
|
||||
version "1.3.7"
|
||||
resolved "https://registry.yarnpkg.com/axios-curlirize/-/axios-curlirize-1.3.7.tgz#0153c51a5af0e92370169daea33f234d588baad1"
|
||||
integrity sha512-csSsuMyZj1dv1fL0zRPnDAHWrmlISMvK+wx9WJI/igRVDT4VMgbf2AVenaHghFLfI1nQijXUevYEguYV6u5hjA==
|
||||
|
||||
axios@^0.21.1:
|
||||
version "0.21.1"
|
||||
resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.1.tgz#22563481962f4d6bde9a76d516ef0e5d3c09b2b8"
|
||||
|
|
@ -8712,6 +8740,13 @@ axios@^0.25.0:
|
|||
dependencies:
|
||||
follow-redirects "^1.14.7"
|
||||
|
||||
axios@^0.26.1:
|
||||
version "0.26.1"
|
||||
resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9"
|
||||
integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==
|
||||
dependencies:
|
||||
follow-redirects "^1.14.8"
|
||||
|
||||
axobject-query@^2.2.0:
|
||||
version "2.2.0"
|
||||
resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-2.2.0.tgz#943d47e10c0b704aa42275e20edf3722648989be"
|
||||
|
|
@ -14758,7 +14793,7 @@ follow-redirects@1.12.1:
|
|||
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.12.1.tgz#de54a6205311b93d60398ebc01cf7015682312b6"
|
||||
integrity sha512-tmRv0AVuR7ZyouUHLeNSiO6pqulF7dYa3s19c6t+wz9LD69/uSzdMxJ2S91nTI9U3rt/IldxpzMOFejp6f0hjg==
|
||||
|
||||
follow-redirects@^1.0.0, follow-redirects@^1.10.0, follow-redirects@^1.14.4, follow-redirects@^1.14.7:
|
||||
follow-redirects@^1.0.0, follow-redirects@^1.10.0, follow-redirects@^1.14.4, follow-redirects@^1.14.7, follow-redirects@^1.14.8:
|
||||
version "1.14.9"
|
||||
resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7"
|
||||
integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==
|
||||
|
|
@ -16219,7 +16254,7 @@ hpack.js@^2.1.6:
|
|||
readable-stream "^2.0.1"
|
||||
wbuf "^1.1.0"
|
||||
|
||||
hpagent@^0.1.2:
|
||||
hpagent@^0.1.1, hpagent@^0.1.2:
|
||||
version "0.1.2"
|
||||
resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9"
|
||||
integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==
|
||||
|
|
@ -23481,6 +23516,13 @@ qs@^6.10.0:
|
|||
dependencies:
|
||||
side-channel "^1.0.4"
|
||||
|
||||
qs@^6.10.3:
|
||||
version "6.10.3"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.3.tgz#d6cde1b2ffca87b5aa57889816c5f81535e22e8e"
|
||||
integrity sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==
|
||||
dependencies:
|
||||
side-channel "^1.0.4"
|
||||
|
||||
qs@^6.7.0:
|
||||
version "6.9.4"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687"
|
||||
|
|
@ -30656,6 +30698,19 @@ yargs@^17.0.1, yargs@^17.2.1, yargs@^17.3.1:
|
|||
y18n "^5.0.5"
|
||||
yargs-parser "^21.0.0"
|
||||
|
||||
yargs@^17.4.0:
|
||||
version "17.4.1"
|
||||
resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.4.1.tgz#ebe23284207bb75cee7c408c33e722bfb27b5284"
|
||||
integrity sha512-WSZD9jgobAg3ZKuCQZSa3g9QOJeCCqLoLAykiWgmXnDo9EPnn4RPf5qVTtzgOx66o6/oqhcA5tHtJXpG8pMt3g==
|
||||
dependencies:
|
||||
cliui "^7.0.2"
|
||||
escalade "^3.1.1"
|
||||
get-caller-file "^2.0.5"
|
||||
require-directory "^2.1.1"
|
||||
string-width "^4.2.3"
|
||||
y18n "^5.0.5"
|
||||
yargs-parser "^21.0.0"
|
||||
|
||||
yargs@^3.15.0:
|
||||
version "3.32.0"
|
||||
resolved "https://registry.yarnpkg.com/yargs/-/yargs-3.32.0.tgz#03088e9ebf9e756b69751611d2a5ef591482c995"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue