mirror of
https://github.com/elastic/kibana.git
synced 2025-04-23 17:28:26 -04:00
Benchmark single apis (#146297)
## Summary This PR adds capability to run capacity testing for single apis #143066 Currently in main we have to 2 types of performance tests: - single user performance journey that simulates single end-user experience in browser - scalability journey that uses APM traces from single user performance journey to simulate multiple end-users experience This new type of performance tests allow to better understand how each single server api scale under the similar load. How to run locally: make sure to clone the latest main branch of [elastic/kibana-load-testing](https://github.com/elastic/kibana-load-testing) in Kibana repo run: `node scripts/run_scalability.js --journey-path x-pack/test/scalability/apis/api.core.capabilities.json` How it works: FTR is used to start Kibana/ES and run Gatling simulation with json file as input. After run the latest report matching journey name is parsed to get perf metrics and report using EBT to the Telemetry cluster. How will it run after merge: I plan to run pipeline every 3 hours on bare metal machine and report metrics to Telemetry staging cluster. <img width="2023" alt="image" src="https://user-images.githubusercontent.com/10977896/208771628-f4f5dbcb-cb73-40c6-9aa1-4ec3fbf5285b.png"> APM traces are collected and reported to Kibana stats cluster: <img width="1520" alt="image" src="https://user-images.githubusercontent.com/10977896/208771323-4cca531a-eeea-4941-8b01-50b890f932b1.png"> What metrics are collected: 1. warmupAvgResponseTime - average response time during warmup phase 2. rpsAtWarmup - average requests per second during warmup phase 3. warmupDuration 4. responseTimeMetric (default: 85%) Gatling has response time 25/50/75/80/85/90/95/99 percentiles, as well as min/max values 5. threshold1ResponseTime (default 3000 ms) 6. rpsAtThreshold1 requests per second when `responseTimeMetric` first reach threshold1ResponseTime 7. threshold2ResponseTime 8. rpsAtThreshold2 (default 9000 ms) 9. threshold3ResponseTime 10. rpsAtThreshold3 (default 15000 ms) As long as we agree on metrics I will update indexer for telemetry. Co-authored-by: Alejandro Fernández Haro <alejandro.haro@elastic.co> Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Dzmitry Lemechko
GitHub
parent
bc4e425f2c
commit
5f31ebf1ce
18 changed files with 3903 additions and 6 deletions
|
|
@ -68,15 +68,20 @@ upload_test_results() {
|
|||
buildkite-agent artifact upload "scalability_traces.tar.gz"
|
||||
}
|
||||
|
||||
echo "--- Download the latest artifacts from single user performance pipeline"
|
||||
download_artifacts
|
||||
|
||||
echo "--- Clone kibana-load-testing repo and compile project"
|
||||
checkout_and_compile_load_runner
|
||||
|
||||
echo "--- Run Scalability Tests"
|
||||
cd "$KIBANA_DIR"
|
||||
node scripts/run_scalability --kibana-install-dir "$KIBANA_BUILD_LOCATION" --journey-path "scalability_traces/server"
|
||||
echo "--- Download the latest artifacts from single user performance pipeline"
|
||||
download_artifacts
|
||||
|
||||
if [ "$BUILDKITE_PIPELINE_SLUG" == "kibana-scalability-benchmarking-1" ]; then
|
||||
echo "--- Run journey scalability tests"
|
||||
node scripts/run_scalability --kibana-install-dir "$KIBANA_BUILD_LOCATION" --journey-path "scalability_traces/server"
|
||||
else
|
||||
echo "--- Run single apis capacity tests"
|
||||
node scripts/run_scalability --kibana-install-dir "$KIBANA_BUILD_LOCATION" --journey-path "x-pack/test/scalability/apis"
|
||||
fi
|
||||
|
||||
echo "--- Upload test results"
|
||||
upload_test_results
|
||||
|
|
|
|||
|
|
@ -7,7 +7,11 @@
|
|||
*/
|
||||
|
||||
export { JourneyConfig } from './journey/journey_config';
|
||||
export type { ScalabilityAction, ScalabilitySetup } from './journey/journey_config';
|
||||
export type {
|
||||
ScalabilityAction,
|
||||
ScalabilitySetup,
|
||||
ResponseTimeMetric,
|
||||
} from './journey/journey_config';
|
||||
|
||||
export { Journey } from './journey/journey';
|
||||
export type { Step } from './journey/journey';
|
||||
|
|
|
|||
|
|
@ -39,6 +39,18 @@ export interface ConstantConcurrentUsersAction {
|
|||
|
||||
export type ScalabilityAction = RampConcurrentUsersAction | ConstantConcurrentUsersAction;
|
||||
|
||||
export type ResponseTimeMetric =
|
||||
| 'min'
|
||||
| '25%'
|
||||
| '50%'
|
||||
| '75%'
|
||||
| '80%'
|
||||
| '85%'
|
||||
| '90%'
|
||||
| '95%'
|
||||
| '99%'
|
||||
| 'max';
|
||||
|
||||
export interface ScalabilitySetup {
|
||||
/**
|
||||
* Duration strings must be formatted as string that starts with an integer and
|
||||
|
|
@ -47,6 +59,12 @@ export interface ScalabilitySetup {
|
|||
* eg: "1m" or "30s"
|
||||
*/
|
||||
maxDuration: string;
|
||||
responseTimeMetric?: ResponseTimeMetric;
|
||||
responseTimeThreshold?: {
|
||||
threshold1: number;
|
||||
threshold2: number;
|
||||
threshold3: number;
|
||||
};
|
||||
warmup: ScalabilityAction[];
|
||||
test: ScalabilityAction[];
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
46
x-pack/test/scalability/apis/api.core.capabilities.json
Normal file
46
x-pack/test/scalability/apis/api.core.capabilities.json
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"journeyName": "POST /api/core/capabilities",
|
||||
"scalabilitySetup": {
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 250,
|
||||
"duration": "4m"
|
||||
}
|
||||
],
|
||||
"maxDuration": "6m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": []
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "POST",
|
||||
"path": "/api/core/capabilities",
|
||||
"query": "?useDefaultCapabilities=true",
|
||||
"body": "{\"applications\":[\"error\",\"status\",\"kibana\",\"dev_tools\",\"r\",\"short_url_redirect\",\"home\",\"management\",\"space_selector\",\"security_access_agreement\",\"security_capture_url\",\"security_login\",\"security_logout\",\"security_logged_out\",\"security_overwritten_session\",\"security_account\",\"reportingRedirect\",\"graph\",\"discover\",\"integrations\",\"fleet\",\"ingestManager\",\"visualize\",\"canvas\",\"dashboards\",\"lens\",\"maps\",\"osquery\",\"observability-overview\",\"ml\",\"uptime\",\"synthetics\",\"securitySolutionUI\",\"siem\",\"logs\",\"metrics\",\"infra\",\"monitoring\",\"enterpriseSearch\",\"enterpriseSearchContent\",\"enterpriseSearchAnalytics\",\"elasticsearch\",\"appSearch\",\"workplaceSearch\",\"searchExperiences\",\"apm\",\"ux\",\"kibanaOverview\"]}",
|
||||
"headers": {
|
||||
"Cookie": "",
|
||||
"Kbn-Version": "",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
50
x-pack/test/scalability/apis/api.metrics.vis.data.json
Normal file
50
x-pack/test/scalability/apis/api.metrics.vis.data.json
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
{
|
||||
"journeyName": "POST /api/metrics/vis/data",
|
||||
"scalabilitySetup": {
|
||||
"responseTimeThreshold": {
|
||||
"threshold1": 5000,
|
||||
"threshold2": 10000,
|
||||
"threshold3": 30000
|
||||
},
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 250,
|
||||
"duration": "4m"
|
||||
}
|
||||
],
|
||||
"maxDuration": "6m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": ["x-pack/performance/es_archives/sample_data_ecommerce"],
|
||||
"kbnArchives": ["x-pack/performance/kbn_archives/promotion_tracking_dashboard"]
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "POST",
|
||||
"path": "/api/metrics/vis/data",
|
||||
"headers": {
|
||||
"Cookie": "",
|
||||
"Kbn-Version": "",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"body": "{\"timerange\":{\"timezone\":\"UTC\",\"min\":\"2022-11-14T00:00:00.000Z\",\"max\":\"2022-12-14T17:17:09.931Z\"},\"query\":[{\"query\":\"\",\"language\":\"kuery\"}],\"filters\":[],\"panels\":[{\"time_range_mode\":\"entire_time_range\",\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"ea20ae70-b88d-11e8-a451-f37365e9f268\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"ea20ae71-b88d-11e8-a451-f37365e9f268\",\"type\":\"sum\",\"field\":\"taxful_total_price\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"2\",\"point_size\":\"5\",\"fill\":\"0\",\"stacked\":\"none\",\"filter\":{\"query\":\"products.product_name:*trouser*\",\"language\":\"lucene\"},\"label\":\"Revenue Trousers\",\"value_template\":\"${{value}}\",\"split_color_mode\":\"gradient\"},{\"id\":\"062d77b0-b88e-11e8-a451-f37365e9f268\",\"color\":\"rgba(84,179,153,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"062d77b1-b88e-11e8-a451-f37365e9f268\",\"type\":\"sum\",\"field\":\"taxful_total_price\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"2\",\"point_size\":\"05\",\"fill\":\"0\",\"stacked\":\"none\",\"filter\":{\"query\":\"products.product_name:*watch*\",\"language\":\"lucene\"},\"label\":\"Revenue Watches\",\"value_template\":\"${{value}}\",\"split_color_mode\":\"gradient\"},{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"rgba(96,146,192,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"taxful_total_price\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"2\",\"point_size\":\"5\",\"fill\":\"0\",\"stacked\":\"none\",\"filter\":{\"query\":\"products.product_name:*bag*\",\"language\":\"lucene\"},\"label\":\"Revenue Bags\",\"value_template\":\"${{value}}\",\"split_color_mode\":\"gradient\"},{\"id\":\"faa2c170-b88d-11e8-a451-f37365e9f268\",\"color\":\"rgba(202,142,174,1)\",\"split_mode\":\"everything\",\"metrics\":[{\"id\":\"faa2c171-b88d-11e8-a451-f37365e9f268\",\"type\":\"sum\",\"field\":\"taxful_total_price\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":\"2\",\"point_size\":\"5\",\"fill\":\"0\",\"stacked\":\"none\",\"filter\":{\"query\":\"products.product_name:*cocktail dress*\",\"language\":\"lucene\"},\"label\":\"Revenue Cocktail Dresses\",\"value_template\":\"${{value}}\",\"split_color_mode\":\"gradient\"}],\"time_field\":\"order_date\",\"interval\":\"12h\",\"use_kibana_indexes\":true,\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"legend_position\":\"bottom\",\"annotations\":[{\"fields\":\"taxful_total_price\",\"template\":\"Ring the bell! ${{taxful_total_price}}\",\"query_string\":{\"query\":\"taxful_total_price:>250\",\"language\":\"lucene\"},\"id\":\"c8c30be0-b88f-11e8-a451-f37365e9f268\",\"color\":\"rgba(25,77,51,1)\",\"time_field\":\"order_date\",\"icon\":\"fa-bell\",\"ignore_global_filters\":1,\"ignore_panel_filters\":1,\"index_pattern\":{\"id\":\"ff959d40-b880-11e8-a6d9-e546fe2bba5f\"}}],\"tooltip_mode\":\"show_all\",\"drop_last_bucket\":0,\"isModelInvalid\":false,\"index_pattern\":{\"id\":\"ff959d40-b880-11e8-a6d9-e546fe2bba5f\"},\"truncate_legend\":1,\"max_lines_legend\":1}],\"state\":{},\"searchSession\":{\"sessionId\":\"8c3dfba6-dcf9-4402-aa7c-3b36fd9bda0c\",\"isRestore\":false,\"isStored\":false}}",
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
{
|
||||
"journeyName": "GET /api/saved_objects_tagging/tags",
|
||||
"scalabilitySetup": {
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 600,
|
||||
"duration": "275s"
|
||||
}
|
||||
],
|
||||
"maxDuration": "7m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": [
|
||||
"x-pack/test/functional/fixtures/kbn_archiver/saved_objects_management/saved_objects_mix.json"
|
||||
]
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"date": "2022-11-14T09:31:49.963Z",
|
||||
"http": {
|
||||
"method": "GET",
|
||||
"path": "/api/saved_objects_tagging/tags",
|
||||
"headers": {
|
||||
"Cookie": "sid=Fe26.2**b4f51707bfe081641d5680f3564a6294e67",
|
||||
"Kbn-Version": "8.7.0-SNAPSHOT",
|
||||
"Kbn-System-Request": "true",
|
||||
"Referer": "http://localhost:5620/app/home",
|
||||
"X-Kbn-Context": "%7B%22name%22%3A%22home%22%2C%22url%22%3A%22%2Fapp%2Fhome%22%7D",
|
||||
"Host": "localhost:5620",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"journeyName": "POST /api/telemetry/v2/clusters/_stats",
|
||||
"scalabilitySetup": {
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 490,
|
||||
"duration": "4m"
|
||||
}
|
||||
],
|
||||
"maxDuration": "6m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": []
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "POST",
|
||||
"path": "/api/telemetry/v2/clusters/_stats",
|
||||
"body": "{}",
|
||||
"headers": {
|
||||
"Cookie": "",
|
||||
"Kbn-Version": "",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
46
x-pack/test/scalability/apis/bundles.core.entry.json
Normal file
46
x-pack/test/scalability/apis/bundles.core.entry.json
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
{
|
||||
"journeyName": "GET /{buildNumber}/bundles/core/core.entry.js",
|
||||
"scalabilitySetup": {
|
||||
"responseTimeThreshold": {
|
||||
"threshold1": 1000,
|
||||
"threshold2": 3000,
|
||||
"threshold3": 5000
|
||||
},
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 1400,
|
||||
"duration": "278s"
|
||||
}
|
||||
],
|
||||
"maxDuration": "6m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": []
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "GET",
|
||||
"path": "/{buildNumber}/bundles/core/core.entry.js",
|
||||
"headers": {
|
||||
"Accept-Encoding": "gzip, deflate, br"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
44
x-pack/test/scalability/apis/internal.security.session.json
Normal file
44
x-pack/test/scalability/apis/internal.security.session.json
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
{
|
||||
"journeyName": "GET /internal/security/session",
|
||||
"scalabilitySetup": {
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 700,
|
||||
"duration": "345s"
|
||||
}
|
||||
],
|
||||
"maxDuration": "8m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": []
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "GET",
|
||||
"path": "/internal/security/session",
|
||||
"headers": {
|
||||
"Cookie": "",
|
||||
"Kbn-Version": "",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
{
|
||||
"journeyName": "GET /internal/security/user_profile",
|
||||
"scalabilitySetup": {
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 700,
|
||||
"duration": "345s"
|
||||
}
|
||||
],
|
||||
"maxDuration": "8m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": []
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "GET",
|
||||
"path": "/internal/security/user_profile",
|
||||
"query": "?dataPath=avatar",
|
||||
"headers": {
|
||||
"Cookie": "",
|
||||
"Kbn-Version": "",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
44
x-pack/test/scalability/apis/security.me.json
Normal file
44
x-pack/test/scalability/apis/security.me.json
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
{
|
||||
"journeyName": "GET /internal/security/me",
|
||||
"scalabilitySetup": {
|
||||
"warmup": [
|
||||
{
|
||||
"action": "constantUsersPerSec",
|
||||
"userCount": 10,
|
||||
"duration": "30s"
|
||||
}
|
||||
],
|
||||
"test": [
|
||||
{
|
||||
"action": "rampUsersPerSec",
|
||||
"minUsersCount": 10,
|
||||
"maxUsersCount": 1000,
|
||||
"duration": "198s"
|
||||
}
|
||||
],
|
||||
"maxDuration": "5m"
|
||||
},
|
||||
"testData": {
|
||||
"esArchives": [],
|
||||
"kbnArchives": []
|
||||
},
|
||||
"streams": [
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"http": {
|
||||
"method": "GET",
|
||||
"path": "/internal/security/me",
|
||||
"headers": {
|
||||
"Cookie": "",
|
||||
"Kbn-Version": "",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Content-Type": "application/json"
|
||||
},
|
||||
"statusCode": 200
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -12,10 +12,15 @@ import path from 'path';
|
|||
import { REPO_ROOT } from '@kbn/repo-info';
|
||||
import { createFlagError } from '@kbn/dev-cli-errors';
|
||||
import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
|
||||
import { v4 as uuidV4 } from 'uuid';
|
||||
import { ScalabilityTestRunner } from './runner';
|
||||
import { FtrProviderContext } from './ftr_provider_context';
|
||||
import { ScalabilityJourney } from './types';
|
||||
|
||||
// These "secret" values are intentionally written in the source.
|
||||
const APM_SERVER_URL = 'https://142fea2d3047486e925eb8b223559cae.apm.europe-west1.gcp.cloud.es.io';
|
||||
const APM_PUBLIC_TOKEN = 'pWFFEym07AKBBhUE2i';
|
||||
|
||||
const AGGS_SHARD_DELAY = process.env.LOAD_TESTING_SHARD_DELAY;
|
||||
const DISABLE_PLUGINS = process.env.LOAD_TESTING_DISABLE_PLUGINS;
|
||||
const scalabilityJsonPath = process.env.SCALABILITY_JOURNEY_PATH;
|
||||
|
|
@ -36,6 +41,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
|||
);
|
||||
}
|
||||
|
||||
const journey: ScalabilityJourney = JSON.parse(fs.readFileSync(scalabilityJsonPath, 'utf8'));
|
||||
|
||||
const baseConfig = (
|
||||
await readConfigFile(require.resolve('../../performance/journeys/login.ts'))
|
||||
).getAll();
|
||||
|
|
@ -63,6 +70,29 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
|||
...(!!AGGS_SHARD_DELAY ? ['--data.search.aggs.shardDelay.enabled=true'] : []),
|
||||
...(!!DISABLE_PLUGINS ? ['--plugins.initialize=false'] : []),
|
||||
],
|
||||
env: {
|
||||
ELASTIC_APM_ACTIVE: true,
|
||||
ELASTIC_APM_CENTRAL_CONFIG: false,
|
||||
ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '0.1',
|
||||
ELASTIC_APM_BREAKDOWN_METRICS: false,
|
||||
ELASTIC_APM_CAPTURE_SPAN_STACK_TRACES: false,
|
||||
ELASTIC_APM_METRICS_INTERVAL: '120s',
|
||||
ELASTIC_APM_MAX_QUEUE_SIZE: 20480,
|
||||
ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
|
||||
ELASTIC_APM_SERVER_URL: APM_SERVER_URL,
|
||||
ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN,
|
||||
ELASTIC_APM_GLOBAL_LABELS: Object.entries({
|
||||
testBuildId: process.env.BUILDKITE_BUILD_ID ?? `local-${uuidV4()}`,
|
||||
testJobId: process.env.BUILDKITE_JOB_ID ?? `local-${uuidV4()}`,
|
||||
journeyName: journey.journeyName,
|
||||
ftrConfig: path.basename(scalabilityJsonPath),
|
||||
branch: process.env.BUILDKITE_BRANCH,
|
||||
gitRev: process.env.BUILDKITE_COMMIT,
|
||||
ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG,
|
||||
})
|
||||
.flatMap(([key, value]) => (value == null ? [] : `${key}=${value}`))
|
||||
.join(','),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
54
x-pack/test/scalability/events_shipper.ts
Normal file
54
x-pack/test/scalability/events_shipper.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import fetch from 'node-fetch';
|
||||
import { MetricEvent } from './types';
|
||||
|
||||
const eventsToNDJSON = (events: MetricEvent[]) => {
|
||||
return `${events.map((event) => JSON.stringify(event)).join('\n')}\n`;
|
||||
};
|
||||
|
||||
const buildHeaders = (clusterUuid: string, version: string) => {
|
||||
return {
|
||||
'content-type': 'application/x-ndjson',
|
||||
'x-elastic-cluster-id': clusterUuid,
|
||||
'x-elastic-stack-version': version,
|
||||
};
|
||||
};
|
||||
|
||||
export class EventsShipper {
|
||||
url: string;
|
||||
clusterUuid: string;
|
||||
version: string;
|
||||
log: ToolingLog;
|
||||
|
||||
constructor(url: string, clusterUuid: string, version: string, log: ToolingLog) {
|
||||
this.url = url;
|
||||
this.clusterUuid = clusterUuid;
|
||||
this.version = version;
|
||||
this.log = log;
|
||||
}
|
||||
|
||||
async send(events: MetricEvent[]) {
|
||||
const body = eventsToNDJSON(events);
|
||||
|
||||
if (process.env.BUILDKITE_BUILD_ID) {
|
||||
const response = await fetch(this.url, {
|
||||
method: 'POST',
|
||||
body,
|
||||
headers: buildHeaders(this.clusterUuid, this.version),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Telemetry sending error: ${response.status} - ${await response.text()}`);
|
||||
} else {
|
||||
this.log.debug(`Telemetry cluster response status: ${response.status}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
210
x-pack/test/scalability/report_parser.ts
Normal file
210
x-pack/test/scalability/report_parser.ts
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import fs from 'fs';
|
||||
import { ScalabilitySetup, ResponseTimeMetric } from '@kbn/journeys';
|
||||
import { CapacityMetrics, DataPoint, ResponseMetric, RpsMetric } from './types';
|
||||
|
||||
const RESPONSE_METRICS_NAMES = [
|
||||
'min',
|
||||
'25%',
|
||||
'50%',
|
||||
'75%',
|
||||
'80%',
|
||||
'85%',
|
||||
'90%',
|
||||
'95%',
|
||||
'99%',
|
||||
'max',
|
||||
];
|
||||
const DEFAULT_THRESHOLD = {
|
||||
threshold1: 3000,
|
||||
threshold2: 6000,
|
||||
threshold3: 12000,
|
||||
};
|
||||
const DEFAULT_METRIC = '85%';
|
||||
const REQUESTS_REGEXP = /(?<=var requests = unpack\(\[)(.*)(?=\]\);)/g;
|
||||
const RESPONSES_PERCENTILES_REGEXP =
|
||||
/(?<=var responsetimepercentilesovertimeokPercentiles = unpack\(\[)(.*)(?=\]\);)/g;
|
||||
|
||||
/**
|
||||
* Returns Rps value for time point when response time is over threshold first time for specific metric
|
||||
* @param rpsData Rps dataset
|
||||
* @param responseTimeData Response time dataset
|
||||
* @param responseTimeThreshold Response time threshold
|
||||
* @param metricName Gatling response metric to compare with threshold
|
||||
* @param log logger
|
||||
* @returns
|
||||
*/
|
||||
const getRPSByResponseTime = (
|
||||
rpsData: RpsMetric[],
|
||||
responseTimeData: ResponseMetric[],
|
||||
responseTimeThreshold: number,
|
||||
metricName: ResponseTimeMetric,
|
||||
log: ToolingLog
|
||||
) => {
|
||||
const timestamp = getTimePoint(responseTimeData, metricName, responseTimeThreshold, log);
|
||||
if (timestamp === -1) {
|
||||
// Data point was not found, most likely 'responseTimeThreshold' should be adjusted
|
||||
// Returning '0' as invalid result
|
||||
return 0;
|
||||
} else {
|
||||
const rps = rpsData.find((i) => i.timestamp === timestamp)?.value;
|
||||
// In edge case Gatling might fail to report requests for specific timestamp, returning '0' as invalid result
|
||||
return !rps ? 0 : rps;
|
||||
}
|
||||
};
|
||||
|
||||
const parseData = (str: string, regex: RegExp) => {
|
||||
const found = str.match(regex);
|
||||
if (found == null) {
|
||||
throw Error('Failed to parse Html string');
|
||||
}
|
||||
return found[0]
|
||||
.replaceAll('],[', '].[')
|
||||
.split('.')
|
||||
.map((i) => {
|
||||
const pair = i
|
||||
.replaceAll(',[', '.[')
|
||||
.replaceAll(/^\[/g, '')
|
||||
.replaceAll(/\]$/g, '')
|
||||
.split('.');
|
||||
const arr = pair[1]?.replaceAll(/^\[/g, '')?.replaceAll(/\]$/g, '');
|
||||
const values: number[] = !arr ? [] : arr.split(',').map(Number);
|
||||
return { timestamp: parseInt(pair[0], 10), values };
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns timestamp for the first response time entry above the threshold
|
||||
* @param data Response time dataset
|
||||
* @param metricName Gatling response metric to compare with threshold
|
||||
* @param responseTimeValue Response time threshold
|
||||
* @param log logger
|
||||
* @returns
|
||||
*/
|
||||
const getTimePoint = (
|
||||
data: ResponseMetric[],
|
||||
metricName: ResponseTimeMetric,
|
||||
responseTimeValue: number,
|
||||
log: ToolingLog
|
||||
) => {
|
||||
const resultsAboveThreshold = data.filter((i) => i.metrics[metricName] >= responseTimeValue);
|
||||
if (resultsAboveThreshold.length === data.length) {
|
||||
log.debug(`Threshold '${responseTimeValue} is too low for '${metricName}' metric'`);
|
||||
return -1;
|
||||
} else if (resultsAboveThreshold.length === 0) {
|
||||
log.debug(`Threshold '${responseTimeValue} is too high for '${metricName}' metric'`);
|
||||
return -1;
|
||||
} else {
|
||||
return resultsAboveThreshold[0].timestamp;
|
||||
}
|
||||
};
|
||||
|
||||
const mapValuesWithMetrics = (data: DataPoint[], metrics: string[]) => {
|
||||
return data
|
||||
.filter((i) => i.values.length === metrics.length)
|
||||
.map((i) => {
|
||||
return {
|
||||
timestamp: i.timestamp,
|
||||
metrics: Object.fromEntries(metrics.map((_, index) => [metrics[index], i.values[index]])),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
export function getCapacityMetrics(
|
||||
htmlReportPath: string,
|
||||
scalabilitySetup: ScalabilitySetup,
|
||||
log: ToolingLog
|
||||
): CapacityMetrics {
|
||||
const htmlContent = fs.readFileSync(htmlReportPath, 'utf-8');
|
||||
// [timestamp, [activeUsers,requests,0]], e.g. [1669026394,[6,6,0]]
|
||||
const requests = parseData(htmlContent, REQUESTS_REGEXP);
|
||||
// [timestamp, [min, 25%, 50%, 75%, 80%, 85%, 90%, 95%, 99%, max]], e.g. 1669026394,[9,11,11,12,13,13,14,15,15,16]
|
||||
const responsePercentiles = parseData(htmlContent, RESPONSES_PERCENTILES_REGEXP);
|
||||
|
||||
const metricName = scalabilitySetup.responseTimeMetric || DEFAULT_METRIC;
|
||||
// warmup phase duration in seconds
|
||||
const warmupDuration = scalabilitySetup.warmup
|
||||
.map((action) => {
|
||||
const parsedValue = parseInt(action.duration.replace(/s|m/, ''), 10);
|
||||
return action.duration.endsWith('m') ? parsedValue * 60 : parsedValue;
|
||||
})
|
||||
.reduce((a, b) => a + b, 0);
|
||||
|
||||
const warmupData = mapValuesWithMetrics(
|
||||
responsePercentiles.slice(0, warmupDuration),
|
||||
RESPONSE_METRICS_NAMES
|
||||
);
|
||||
const testData = mapValuesWithMetrics(
|
||||
responsePercentiles.slice(warmupDuration, responsePercentiles.length - 1),
|
||||
RESPONSE_METRICS_NAMES
|
||||
);
|
||||
const rpsData = requests.map((r) => {
|
||||
return { timestamp: r.timestamp, value: r.values.length > 0 ? r.values[0] : 0 };
|
||||
});
|
||||
|
||||
const rpsMax = Math.max(...rpsData.map((i) => i.value));
|
||||
|
||||
const warmupAvgResponseTime = Math.round(
|
||||
warmupData
|
||||
.map((i) => i.metrics[metricName])
|
||||
.reduce((avg, value, _, { length }) => {
|
||||
return avg + value / length;
|
||||
}, 0)
|
||||
);
|
||||
const rpsAtWarmup = Math.round(
|
||||
rpsData.slice(0, warmupDuration).reduce((avg, rps, _, { length }) => {
|
||||
return avg + rps.value / length;
|
||||
}, 0)
|
||||
);
|
||||
log.info(
|
||||
`Warmup: Avg ${metricName} pct response time - ${warmupAvgResponseTime} ms, avg rps=${rpsAtWarmup}`
|
||||
);
|
||||
|
||||
// Collected response time metrics: 3 pre-defined thresholds
|
||||
const thresholds = scalabilitySetup.responseTimeThreshold || DEFAULT_THRESHOLD;
|
||||
|
||||
const rpsAtThreshold1 = getRPSByResponseTime(
|
||||
rpsData,
|
||||
testData,
|
||||
thresholds.threshold1,
|
||||
metricName,
|
||||
log
|
||||
);
|
||||
|
||||
const rpsAtThreshold2 = getRPSByResponseTime(
|
||||
rpsData,
|
||||
testData,
|
||||
thresholds.threshold2,
|
||||
metricName,
|
||||
log
|
||||
);
|
||||
|
||||
const rpsAtThreshold3 = getRPSByResponseTime(
|
||||
rpsData,
|
||||
testData,
|
||||
thresholds.threshold3,
|
||||
metricName,
|
||||
log
|
||||
);
|
||||
|
||||
return {
|
||||
warmupAvgResponseTime,
|
||||
rpsAtWarmup,
|
||||
warmupDuration,
|
||||
rpsMax,
|
||||
responseTimeMetric: metricName,
|
||||
threshold1ResponseTime: thresholds.threshold1,
|
||||
rpsAtThreshold1,
|
||||
threshold2ResponseTime: thresholds.threshold2,
|
||||
rpsAtThreshold2,
|
||||
threshold3ResponseTime: thresholds.threshold3,
|
||||
rpsAtThreshold3,
|
||||
};
|
||||
}
|
||||
|
|
@ -6,8 +6,58 @@
|
|||
*/
|
||||
|
||||
import { withProcRunner } from '@kbn/dev-proc-runner';
|
||||
// @ts-expect-error we have to check types with "allowJs: false" for now, causing this import to fail
|
||||
import { kibanaPackageJson } from '@kbn/repo-info';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { ToolingLog } from '@kbn/tooling-log';
|
||||
import { FtrProviderContext } from './ftr_provider_context';
|
||||
import { EventsShipper } from './events_shipper';
|
||||
import { getCapacityMetrics } from './report_parser';
|
||||
import { ScalabilityJourney, MetricEvent } from './types';
|
||||
|
||||
const telemetryChannel = 'scalability-metrics';
|
||||
|
||||
async function sendReportMetricsToTelemetry(
|
||||
gatlingProjectRootPath: string,
|
||||
scalabilityJsonPath: string,
|
||||
log: ToolingLog
|
||||
) {
|
||||
const reportRootPath = path.resolve(gatlingProjectRootPath, 'target', 'gatling');
|
||||
const fileName = path.basename(scalabilityJsonPath, path.extname(scalabilityJsonPath));
|
||||
const journeyReportDir = fs.readdirSync(reportRootPath).filter((f) => f.startsWith(fileName));
|
||||
const lastReportPath = journeyReportDir.pop();
|
||||
if (lastReportPath) {
|
||||
const journeyHtmlReportPath = path.resolve(reportRootPath, lastReportPath, 'index.html');
|
||||
|
||||
const journey: ScalabilityJourney = JSON.parse(fs.readFileSync(scalabilityJsonPath, 'utf8'));
|
||||
const metrics = getCapacityMetrics(journeyHtmlReportPath, journey.scalabilitySetup, log);
|
||||
const events: MetricEvent[] = [
|
||||
{
|
||||
...metrics,
|
||||
eventType: 'scalability_metric',
|
||||
eventName: 'capacity_test_summary',
|
||||
journeyName: journey.journeyName,
|
||||
kibanaVersion: kibanaPackageJson.version,
|
||||
branch: process.env.BUILDKITE_BRANCH,
|
||||
ciBuildId: process.env.BUILDKITE_BUILD_ID,
|
||||
ciBuildJobId: process.env.BUILDKITE_JOB_ID,
|
||||
ciBuildNumber: Number(process.env.BUILDKITE_BUILD_NUMBER) || 0,
|
||||
ciBuildName: process.env.BUILDKITE_PIPELINE_SLUG,
|
||||
gitRev: process.env.BUILDKITE_COMMIT,
|
||||
},
|
||||
];
|
||||
log.info(`Sending event: ${JSON.stringify(events)}`);
|
||||
const shipper = new EventsShipper(
|
||||
`https://telemetry-staging.elastic.co/v3/send/${telemetryChannel}?debug=true`,
|
||||
'scalability-test',
|
||||
'1',
|
||||
log
|
||||
);
|
||||
|
||||
await shipper.send(events);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ScalabilityTestRunner is used to run load simulation against local Kibana instance
|
||||
|
|
@ -41,4 +91,6 @@ export async function ScalabilityTestRunner(
|
|||
wait: true,
|
||||
});
|
||||
});
|
||||
|
||||
await sendReportMetricsToTelemetry(gatlingProjectRootPath, scalabilityJsonPath, log);
|
||||
}
|
||||
|
|
|
|||
61
x-pack/test/scalability/types.ts
Normal file
61
x-pack/test/scalability/types.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License
|
||||
* 2.0; you may not use this file except in compliance with the Elastic License
|
||||
* 2.0.
|
||||
*/
|
||||
|
||||
import { ScalabilitySetup } from '@kbn/journeys';
|
||||
|
||||
export interface ScalabilityJourney {
|
||||
journeyName: string;
|
||||
scalabilitySetup: ScalabilitySetup;
|
||||
testData?: {
|
||||
esArchives: string[];
|
||||
kbnArchives: string[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface CapacityMetrics {
|
||||
warmupAvgResponseTime: number;
|
||||
rpsAtWarmup: number;
|
||||
warmupDuration: number;
|
||||
rpsMax: number;
|
||||
responseTimeMetric: string;
|
||||
threshold1ResponseTime: number;
|
||||
rpsAtThreshold1: number;
|
||||
threshold2ResponseTime: number;
|
||||
rpsAtThreshold2: number;
|
||||
threshold3ResponseTime: number;
|
||||
rpsAtThreshold3: number;
|
||||
}
|
||||
|
||||
export interface MetricEvent extends CapacityMetrics {
|
||||
eventName: string;
|
||||
eventType: string;
|
||||
journeyName: string;
|
||||
kibanaVersion: string;
|
||||
branch: string | undefined;
|
||||
ciBuildId: string | undefined;
|
||||
ciBuildJobId: string | undefined;
|
||||
ciBuildName: string | undefined;
|
||||
ciBuildNumber: number;
|
||||
gitRev: string | undefined;
|
||||
}
|
||||
|
||||
export interface RpsMetric {
|
||||
timestamp: number;
|
||||
value: number;
|
||||
}
|
||||
|
||||
export interface ResponseMetric {
|
||||
timestamp: number;
|
||||
metrics: {
|
||||
[k: string]: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface DataPoint {
|
||||
timestamp: number;
|
||||
values: number[];
|
||||
}
|
||||
|
|
@ -112,5 +112,6 @@
|
|||
"@kbn/user-profile-components",
|
||||
"@kbn/apm-synthtrace-client",
|
||||
"@kbn/utils",
|
||||
"@kbn/journeys",
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue