Performance Tests for Login and Home Pages by using Playwright (#119681)

add performance tests for Login and Home pages

- [x] Rename and reduce the number of environment variables for performance test CI job
  - [x] ITERATION_COUNT_ENV =>  PERF_TEST_COUNT
  - [x] DISABLE_CACHE, TEST_RUNNER, TEST_THROTTLE_NETWORK removed
- [x] Use c2-16 queue for performance tests
- [x] Run performance tests by running ES and Kibana separately
- [x] Add warmup phase to performance tests (Not reported to APM). APM Label: PERF_TEST_PHASE
- [x] Add playwright dependency
- [x] Throttle network speed ( downloadThroughput: 750.000 B/s, uploadThroughput: 750.000 B/s, latency: 100 ms )
- [x] Disable cache by using - page.route (‘**’, ….)
- [x] Login and Home pages performance tests.
This commit is contained in:
Baturalp Gurdin 2022-01-18 14:00:06 +03:00 committed by GitHub
parent d7aacd723c
commit eeb5efcc62
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 219 additions and 103 deletions

View file

@ -6,7 +6,7 @@ steps:
key: "performance-test-iteration-count"
hint: "How many times you want to run tests? "
required: true
if: build.env('ITERATION_COUNT_ENV') == null
if: build.env('PERF_TEST_COUNT') == null
- label: ":male-mechanic::skin-tone-2: Pre-Build"
command: .buildkite/scripts/lifecycle/pre_build.sh
@ -19,10 +19,10 @@ steps:
queue: c2-16
key: build
- label: ":muscle: Performance Tests"
command: .buildkite/scripts/steps/functional/performance.sh
- label: ":muscle: Performance Tests with Playwright config"
command: .buildkite/scripts/steps/functional/performance_playwright.sh
agents:
queue: ci-group-6
queue: c2-16
depends_on: build
- wait: ~

View file

@ -2,20 +2,22 @@
set -uo pipefail
if [ -z "${ITERATION_COUNT_ENV+x}" ]; then
ITERATION_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
if [ -z "${PERF_TEST_COUNT+x}" ]; then
TEST_COUNT="$(buildkite-agent meta-data get performance-test-iteration-count)"
else
ITERATION_COUNT=$ITERATION_COUNT_ENV
TEST_COUNT=$PERF_TEST_COUNT
fi
tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${ITERATION_COUNT} times"
tput setab 2; tput setaf 0; echo "Performance test will be run at ${BUILDKITE_BRANCH} ${TEST_COUNT} times"
cat << EOF | buildkite-agent pipeline upload
steps:
- command: .buildkite/scripts/steps/functional/performance_sub.sh
parallelism: "$ITERATION_COUNT"
- command: .buildkite/scripts/steps/functional/performance_sub_playwright.sh
parallelism: "$TEST_COUNT"
concurrency: 20
concurrency_group: 'performance-test-group'
agents:
queue: c2-16
EOF

View file

@ -1,17 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
.buildkite/scripts/bootstrap.sh
.buildkite/scripts/download_build_artifacts.sh
cd "$XPACK_DIR"
echo --- Run Performance Tests
checks-reporter-with-killswitch "Run Performance Tests" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config test/performance/config.ts;

View file

@ -0,0 +1,41 @@
#!/usr/bin/env bash
set -euo pipefail
source .buildkite/scripts/common/util.sh
.buildkite/scripts/bootstrap.sh
.buildkite/scripts/download_build_artifacts.sh
echo --- Run Performance Tests with Playwright config
node scripts/es snapshot&
esPid=$!
export TEST_PERFORMANCE_PHASE=WARMUP
export TEST_ES_URL=http://elastic:changeme@localhost:9200
export TEST_ES_DISABLE_STARTUP=true
export ELASTIC_APM_ACTIVE=false
sleep 120
cd "$XPACK_DIR"
# warmup round 1
checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: WARMUP)" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config "test/performance/config.playwright.ts";
export TEST_PERFORMANCE_PHASE=TEST
export ELASTIC_APM_ACTIVE=true
checks-reporter-with-killswitch "Run Performance Tests with Playwright Config (Phase: TEST)" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_BUILD_LOCATION" \
--config "test/performance/config.playwright.ts";
kill "$esPid"

View file

@ -831,6 +831,7 @@
"pbf": "3.2.1",
"pirates": "^4.0.1",
"pixelmatch": "^5.1.0",
"playwright": "^1.17.1",
"postcss": "^7.0.32",
"postcss-loader": "^3.0.0",
"postcss-prefix-selector": "^1.7.2",

View file

@ -109,7 +109,9 @@ export async function runTests(options: RunTestsParams) {
let es;
try {
es = await runElasticsearch({ config, options: { ...options, log } });
if (process.env.TEST_ES_DISABLE_STARTUP !== 'true') {
es = await runElasticsearch({ config, options: { ...options, log } });
}
await runKibanaServer({ procs, config, options });
await runFtr({ configPath, options: { ...options, log } });
} finally {

View file

@ -12,23 +12,11 @@ interface NetworkOptions {
LATENCY: number;
}
const sec = 1_000;
const kB = 1024;
const sec = 10 ** 3;
const MBps = 10 ** 6 / 8; // megabyte per second (MB/s) (can be abbreviated as MBps)
// Download (kb/s) Upload (kb/s) Latency (ms)
// https://gist.github.com/theodorosploumis/fd4086ee58369b68aea6b0782dc96a2e
// Selenium uses B/s (bytes) for network throttling
// Download (B/s) Upload (B/s) Latency (ms)
export const NETWORK_PROFILES: { [key: string]: NetworkOptions } = {
DEFAULT: { DOWNLOAD: 5 * kB * sec, UPLOAD: 1 * kB * sec, LATENCY: 0.1 * sec },
GPRS: { DOWNLOAD: 0.05 * kB * sec, UPLOAD: 0.02 * kB * sec, LATENCY: 0.5 * sec },
MOBILE_EDGE: { DOWNLOAD: 0.24 * kB * sec, UPLOAD: 0.2 * kB * sec, LATENCY: 0.84 * sec },
'2G_REGULAR': { DOWNLOAD: 0.25 * kB * sec, UPLOAD: 0.05 * kB * sec, LATENCY: 0.3 * sec },
'2G_GOOD': { DOWNLOAD: 0.45 * kB * sec, UPLOAD: 0.15 * kB * sec, LATENCY: 0.15 * sec },
'3G_SLOW': { DOWNLOAD: 0.78 * kB * sec, UPLOAD: 0.33 * kB * sec, LATENCY: 0.2 * sec },
'3G_REGULAR': { DOWNLOAD: 0.75 * kB * sec, UPLOAD: 0.25 * kB * sec, LATENCY: 0.1 * sec },
'3G_GOOD': { DOWNLOAD: 1.5 * kB * sec, UPLOAD: 0.75 * kB * sec, LATENCY: 0.04 * sec },
'4G_REGULAR': { DOWNLOAD: 4 * kB * sec, UPLOAD: 3 * kB * sec, LATENCY: 0.02 * sec },
DSL: { DOWNLOAD: 2 * kB * sec, UPLOAD: 1 * kB * sec, LATENCY: 0.005 * sec },
CABLE_5MBPS: { DOWNLOAD: 5 * kB * sec, UPLOAD: 1 * kB * sec, LATENCY: 0.28 * sec },
CABLE_8MBPS: { DOWNLOAD: 8 * kB * sec, UPLOAD: 2 * kB * sec, LATENCY: 0.1 * sec },
WIFI: { DOWNLOAD: 30 * kB * sec, UPLOAD: 15 * kB * sec, LATENCY: 0.002 * sec },
CLOUD_USER: { DOWNLOAD: 6 * MBps, UPLOAD: 6 * MBps, LATENCY: 0.1 * sec },
};

View file

@ -39,6 +39,7 @@ const headlessBrowser: string = process.env.TEST_BROWSER_HEADLESS as string;
const browserBinaryPath: string = process.env.TEST_BROWSER_BINARY_PATH as string;
const remoteDebug: string = process.env.TEST_REMOTE_DEBUG as string;
const certValidation: string = process.env.NODE_TLS_REJECT_UNAUTHORIZED as string;
const noCache: string = process.env.TEST_DISABLE_CACHE as string;
const SECOND = 1000;
const MINUTE = 60 * SECOND;
const NO_QUEUE_COMMANDS = ['getLog', 'getStatus', 'newSession', 'quit'];
@ -118,6 +119,11 @@ function initChromiumOptions(browserType: Browsers, acceptInsecureCerts: boolean
options.setChromeBinaryPath(browserBinaryPath);
}
if (noCache === '1') {
options.addArguments('disk-cache-size', '0');
options.addArguments('disk-cache-dir', '/dev/null');
}
const prefs = new logging.Preferences();
prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL);
options.setUserPreferences(chromiumUserPrefs);
@ -291,12 +297,12 @@ async function attemptToCreateCommand(
const { session, consoleLog$ } = await buildDriverInstance();
if (throttleOption === '1' && browserType === 'chrome') {
const { KBN_NETWORK_TEST_PROFILE = 'DEFAULT' } = process.env;
const { KBN_NETWORK_TEST_PROFILE = 'CLOUD_USER' } = process.env;
const profile =
KBN_NETWORK_TEST_PROFILE in Object.keys(NETWORK_PROFILES)
? KBN_NETWORK_TEST_PROFILE
: 'DEFAULT';
: 'CLOUD_USER';
const {
DOWNLOAD: downloadThroughput,
@ -306,9 +312,16 @@ async function attemptToCreateCommand(
// Only chrome supports this option.
log.debug(
`NETWORK THROTTLED with profile ${profile}: ${downloadThroughput}kbps down, ${uploadThroughput}kbps up, ${latency} ms latency.`
`NETWORK THROTTLED with profile ${profile}: ${downloadThroughput} B/s down, ${uploadThroughput} B/s up, ${latency} ms latency.`
);
if (noCache) {
// @ts-expect-error
await session.sendDevToolsCommand('Network.setCacheDisabled', {
cacheDisabled: true,
});
}
// @ts-expect-error
session.setNetworkConditions({
offline: false,

View file

@ -91,7 +91,6 @@ const onlyNotInCoverageTests = [
require.resolve('../test/saved_object_tagging/api_integration/security_and_spaces/config.ts'),
require.resolve('../test/saved_object_tagging/api_integration/tagging_api/config.ts'),
require.resolve('../test/examples/config.ts'),
require.resolve('../test/performance/config.ts'),
require.resolve('../test/functional_execution_context/config.ts'),
];

View file

@ -10,15 +10,17 @@ import { FtrConfigProviderContext } from '@kbn/test';
import { services } from './services';
import { pageObjects } from './page_objects';
// These "secret" values are intentionally written in the source. We would make the APM server accept annonymous traffic if we could
// These "secret" values are intentionally written in the source. We would make the APM server accept anonymous traffic if we could
const APM_SERVER_URL = 'https://2fad4006bf784bb8a54e52f4a5862609.apm.us-west1.gcp.cloud.es.io:443';
const APM_PUBLIC_TOKEN = 'Q5q5rWQEw6tKeirBpw';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const functionalConfig = await readConfigFile(require.resolve('../functional/config'));
const testFiles = [require.resolve('./tests/playwright/home.ts')];
return {
testFiles: [require.resolve('./tests/index.ts')],
testFiles,
services,
pageObjects,
servers: functionalConfig.get('servers'),
@ -31,21 +33,15 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
kbnTestServer: {
...functionalConfig.get('kbnTestServer'),
env: {
ELASTIC_APM_ACTIVE: 'true',
ELASTIC_APM_ACTIVE: process.env.ELASTIC_APM_ACTIVE,
ELASTIC_APM_CONTEXT_PROPAGATION_ONLY: 'false',
ELASTIC_APM_ENVIRONMENT: process.env.CI ? 'ci' : 'development',
ELASTIC_APM_TRANSACTION_SAMPLE_RATE: '1.0',
ELASTIC_APM_SERVER_URL: APM_SERVER_URL,
ELASTIC_APM_SECRET_TOKEN: APM_PUBLIC_TOKEN,
ELASTIC_APM_GLOBAL_LABELS: Object.entries({
ftrConfig: `x-pack/test/performance`,
jenkinsJobName: process.env.JOB_NAME,
jenkinsBuildNumber: process.env.BUILD_NUMBER,
prId: process.env.PR_NUMBER,
branch: process.env.GIT_BRANCH,
commit: process.env.GIT_COMMIT,
mergeBase: process.env.PR_MERGE_BASE,
targetBranch: process.env.PR_TARGET_BRANCH,
ftrConfig: `x-pack/test/performance/tests/config.playwright`,
performancePhase: process.env.PERF_TEST_PHASE,
})
.filter(([, v]) => !!v)
.reduce((acc, [k, v]) => (acc ? `${acc},${k}=${v}` : `${k}=${v}`), ''),

View file

@ -5,7 +5,7 @@
* 2.0.
*/
import { FtrProviderContext } from '../ftr_provider_context';
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ getService, getPageObject }: FtrProviderContext) {
const retry = getService('retry');

View file

@ -1,25 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../ftr_provider_context';
export default function ({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'security']);
const testSubjects = getService('testSubjects');
describe('Login', () => {
it('login and navigate to homepage', async () => {
await PageObjects.common.navigateToApp('login');
await testSubjects.existOrFail('loginSubmit', { timeout: 2000 });
await PageObjects.security.login();
await testSubjects.existOrFail('homeApp', { timeout: 2000 });
});
});
}

View file

@ -1,16 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('performance', function () {
this.tags('ciGroup8');
loadTestFile(require.resolve('./home'));
});
}

View file

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import Url from 'url';
import { ChromiumBrowser, Page } from 'playwright';
import testSetup from './setup';
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ getService }: FtrProviderContext) {
describe('perf_login_and_home', () => {
const config = getService('config');
const kibanaUrl = Url.format({
protocol: config.get('servers.kibana.protocol'),
hostname: config.get('servers.kibana.hostname'),
port: config.get('servers.kibana.port'),
});
let page: Page | null = null;
let browser: ChromiumBrowser | null = null;
before(async () => {
const context = await testSetup();
page = context.page;
browser = context.browser;
});
after(async () => {
await browser?.close();
});
it('Go to Kibana login page', async () => {
await page?.goto(`${kibanaUrl}`);
});
it('Login to Kibana', async () => {
const usernameLocator = page?.locator('[data-test-subj=loginUsername]');
const passwordLocator = page?.locator('[data-test-subj=loginPassword]');
const submitButtonLocator = page?.locator('[data-test-subj=loginSubmit]');
await usernameLocator?.type('elastic', { delay: 500 });
await passwordLocator?.type('changeme', { delay: 500 });
await submitButtonLocator?.click({ delay: 1000 });
});
it('Dismiss Welcome Screen', async () => {
await page?.waitForLoadState();
const skipButtonLocator = page?.locator('[data-test-subj=skipWelcomeScreen]');
await skipButtonLocator?.click({ delay: 1000 });
await page?.waitForLoadState('networkidle');
});
});
}

View file

@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import playwright, { ChromiumBrowser, Page } from 'playwright';
interface ITestSetup {
browser: ChromiumBrowser;
page: Page;
}
const headless = process.env.TEST_BROWSER_HEADLESS === '1';
export default async (): Promise<ITestSetup> => {
const browser = await playwright.chromium.launch({ headless });
const page = await browser.newPage();
const client = await page.context().newCDPSession(page);
await client.send('Network.clearBrowserCache');
await client.send('Network.setCacheDisabled', { cacheDisabled: true });
await client.send('Network.emulateNetworkConditions', {
latency: 100,
downloadThroughput: 750_000,
uploadThroughput: 750_000,
offline: false,
});
await page.route('**', (route) => route.continue());
return { browser, page };
};

View file

@ -10225,6 +10225,11 @@ commander@^7.0.0:
resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
commander@^8.2.0:
version "8.3.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66"
integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
common-tags@^1.8.0:
version "1.8.0"
resolved "https://registry.yarnpkg.com/common-tags/-/common-tags-1.8.0.tgz#8e3153e542d4a39e9b10554434afaaf98956a937"
@ -21942,6 +21947,35 @@ playwright-chromium@=1.14.0:
ws "^7.4.6"
yazl "^2.5.1"
playwright-core@=1.17.1:
version "1.17.1"
resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.17.1.tgz#a16e0f89284a0ed8ae6d77e1c905c84b8a2ba022"
integrity sha512-C3c8RpPiC3qr15fRDN6dx6WnUkPLFmST37gms2aoHPDRvp7EaGDPMMZPpqIm/QWB5J40xDrQCD4YYHz2nBTojQ==
dependencies:
commander "^8.2.0"
debug "^4.1.1"
extract-zip "^2.0.1"
https-proxy-agent "^5.0.0"
jpeg-js "^0.4.2"
mime "^2.4.6"
pngjs "^5.0.0"
progress "^2.0.3"
proper-lockfile "^4.1.1"
proxy-from-env "^1.1.0"
rimraf "^3.0.2"
socks-proxy-agent "^6.1.0"
stack-utils "^2.0.3"
ws "^7.4.6"
yauzl "^2.10.0"
yazl "^2.5.1"
playwright@^1.17.1:
version "1.17.1"
resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.17.1.tgz#a6d63302ee40f41283c4bf869de261c4743a787c"
integrity sha512-DisCkW9MblDJNS3rG61p8LiLA2WA7IY/4A4W7DX4BphWe/HuWjKmGQptuk4NVIh5UuSwXpW/jaH2+ZgjHs3GMA==
dependencies:
playwright-core "=1.17.1"
plugin-error@^1.0.0, plugin-error@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/plugin-error/-/plugin-error-1.0.1.tgz#77016bd8919d0ac377fdcdd0322328953ca5781c"
@ -25712,7 +25746,16 @@ socks-proxy-agent@^5.0.0:
debug "4"
socks "^2.3.3"
socks@^2.3.3:
socks-proxy-agent@^6.1.0:
version "6.1.1"
resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-6.1.1.tgz#e664e8f1aaf4e1fb3df945f09e3d94f911137f87"
integrity sha512-t8J0kG3csjA4g6FTbsMOWws+7R7vuRC8aQ/wy3/1OWmsgwA68zs/+cExQ0koSitUDXqhufF/YJr9wtNMZHw5Ew==
dependencies:
agent-base "^6.0.2"
debug "^4.3.1"
socks "^2.6.1"
socks@^2.3.3, socks@^2.6.1:
version "2.6.1"
resolved "https://registry.yarnpkg.com/socks/-/socks-2.6.1.tgz#989e6534a07cf337deb1b1c94aaa44296520d30e"
integrity sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==