[kbn/journeys] fixes to run journeys against ESS cluster (#166923)

## Summary

I had to change `waitForRender` since `page.waitForFunction` tries to
run a script on page and it is not working due to CSP settings on Cloud.
Instead of injecting a script, we use a classical API to find
elements/attributes in the DOM.

Since `PUT /internal/core/_settings` is merged in 8.11.0, journeys run
on Cloud with on-fly labels update is supported starting deployments
8.11.0+. I added error message for 404 code just in case someone runs it
on earlier version.

`many_fields_discover` journey was update since on Cloud the data view
used by scenario is not selected by default.

How it works:

Create a deployment with QAF and re-configure it for journey run:
```
export EC_DEPLOYMENT_NAME=my-run-8.11
qaf elastic-cloud deployments create --stack-version 8.11.0-SNAPSHOT --environment staging --region gcp-us-central1
qaf elastic-cloud deployments configure-for-performance-journeys
```

Run any journey, e.g. many_fields_discover
```
TEST_CLOUD=1 TEST_ES_URL=https://username:pswd@es_url:443 TEST_KIBANA_URL=https://username:pswd@kibana-ur_url node scripts/functional_test_runner --config x-pack/performance/journeys/many_fields_discover.ts
```

You should see a log about labels being updated:

```
Updating telemetry & APM labels: {"testJobId":"local-a3272047-6724-44d1-9a61-5c79781b06a1","testBuildId":"local-d8edbace-f441-4ba9-ac83-5909be3acf2a","journeyName":"many_fields_discover","ftrConfig":"x-pack/performance/journeys/many_fields_discover.ts"}
```

And then able to find APM logs for the journey in
[Ops](https://kibana-ops-e2e-perf.kb.us-central1.gcp.cloud.es.io:9243/app/apm/services?comparisonEnabled=true&environment=ENVIRONMENT_ALL&kuery=labels.testJobId%20%3A%20%22local-d79a878c-cc7a-423b-b884-c9b6b1a8d781%22&latencyAggregationType=avg&offset=1d&rangeFrom=now-24h%2Fh&rangeTo=now&serviceGroup=&transactionType=request)
cluster

dev_docs/tutorials/performance/running_performance_journey_in_cloud.mdx

@@ -11,7 +11,7 @@ tags: ['kibana', 'onboarding', 'setup', 'performance', 'development', 'telemetry
 As a way to better understand user experience with Kibana in cloud, we support running performance journeys against
 Cloud deployments.
 The process takes a few steps:
-- Create a cloud deployment
+- Create a cloud deployment (8.11.0+ is supported)
 - Re-configure deployment with APM enabled and reporting metrics to the monitoring cluster
 - Create a user with `superuser` role to run tests with
 - Checkout the branch that matches your cloud deployment version
@@ -35,7 +35,7 @@ Navigate to `Advanced Edit` page and change `Deployment Configuration` by adding
 ```
 "user_settings_override_json": {
   "tracing.apm.enabled": "true",
-  "tracing.apm.environment": "development",
+  "tracing.apm.agent.environment": "development",
   "tracing.apm.agent.service_name": "elasticsearch",
   "tracing.apm.agent.server_url": "<SERVER_URL>",
   "tracing.apm.agent.metrics_interval": "120s",
@@ -50,6 +50,7 @@ Navigate to `Advanced Edit` page and change `Deployment Configuration` by adding
 
 ```
 "user_settings_override_json": {
+  "coreApp.allowDynamicConfigOverrides": true,
   "elastic.apm.active": true,
   "elastic.apm.breakdownMetrics": false,
   "elastic.apm.captureBody": "all",
@@ -74,8 +75,28 @@ Note: DEPLOYMENT_ID and YOUR_JOURNEY_NAME values are optional labels to find the
 
 Save changes and make sure cluster is restarted successfully.
 
+### Use QAF to prepare the deployment
+The quickest way to prepare ESS deployment is to use [QAF](https://github.com/elastic/qaf):
+
+- Make sure to add `~/.elastic/cloud.json` and ~/.elastic/cloud-admin.json with Cloud API (to create deployment) & Cloud Admin API (to modify it) keys
+```
+{
+  "api_key": {
+    "production": "<PROD KEY>",
+    "staging": "<STAGING KEY>",
+    "qa": "<QA KEY>"
+  }
+}
+```
+- Create deployment and modify it
+```
+export EC_DEPLOYMENT_NAME=kibana-perf-8.11
+qaf elastic-cloud deployments create --stack-version 8.11.0-SNAPSHOT --environment staging --region gcp-us-central1
+qaf elastic-cloud deployments configure-for-performance-journeys
+```
+
 ### Run the journey
-Make sure you have created user with `superuser` role and the Kibana repo branch is matching your deployment version.
+Make sure the Kibana repo branch is matching your deployment version.
 Set env variables to run FTR against your cloud deployment:
 
 ```
@@ -90,4 +111,6 @@ Run your journey with the command:
 node scripts/functional_test_runner.js --config x-pack/performance/journeys/$YOUR_JOURNEY_NAME.ts`
 ```
 
+APM & Telemetry labels will be updated on the fly and metrics/traces should be available in Telemetry Staging and kibana-ops-e2e-perf cluster.
+
 

packages/kbn-journeys/journey/journey.ts

@@ -12,14 +12,9 @@ import { Page } from 'playwright';
 import callsites from 'callsites';
 import { ToolingLog } from '@kbn/tooling-log';
 import { FtrConfigProvider } from '@kbn/test';
-import {
-  FtrProviderContext,
-  KibanaServer,
-  Es,
-  RetryService,
-} from '@kbn/ftr-common-functional-services';
+import { FtrProviderContext } from '../services/ftr_context_provider';
+import { Es, KibanaServer, Retry, Auth } from '../services';
 
-import { Auth } from '../services/auth';
 import { InputDelays } from '../services/input_delays';
 import { KibanaUrl } from '../services/kibana_url';
 
@@ -37,7 +32,7 @@ export interface BaseStepCtx {
   kbnUrl: KibanaUrl;
   kibanaServer: KibanaServer;
   es: Es;
-  retry: RetryService;
+  retry: Retry;
   auth: Auth;
 }
 
@@ -141,7 +136,7 @@ export class Journey<CtxExt extends object> {
       getService('kibanaServer'),
       getService('es'),
       getService('retry'),
-      new Auth(getService('config'), getService('log'), getService('kibanaServer')),
+      getService('auth'),
       this.config
     ).initMochaSuite(this.#steps);
   }

packages/kbn-journeys/journey/journey_ftr_config.ts

@@ -11,7 +11,7 @@ import Path from 'path';
 import { v4 as uuidV4 } from 'uuid';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { FtrConfigProviderContext, FtrConfigProvider } from '@kbn/test';
-import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
+import { services } from '../services';
 
 import { AnyStep } from './journey';
 import { JourneyConfig } from './journey_config';
@@ -66,7 +66,7 @@ export function makeFtrConfigProvider(
         bail: true,
       },
 
-      services: commonFunctionalServices,
+      services,
       pageObjects: {},
 
       servicesRequiredForTestAnalysis: ['performance', 'journeyConfig'],

packages/kbn-journeys/journey/journey_ftr_harness.ts

@@ -9,20 +9,19 @@
 import Url from 'url';
 import { inspect, format } from 'util';
 import { setTimeout } from 'timers/promises';
-
 import * as Rx from 'rxjs';
 import apmNode from 'elastic-apm-node';
 import playwright, { ChromiumBrowser, Page, BrowserContext, CDPSession, Request } from 'playwright';
 import { asyncMap, asyncForEach } from '@kbn/std';
 import { ToolingLog } from '@kbn/tooling-log';
 import { Config } from '@kbn/test';
-import { EsArchiver, KibanaServer, Es, RetryService } from '@kbn/ftr-common-functional-services';
 import {
   ELASTIC_HTTP_VERSION_HEADER,
   X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
 } from '@kbn/core-http-common';
 
-import { Auth } from '../services/auth';
+import { AxiosError } from 'axios';
+import { Auth, Es, EsArchiver, KibanaServer, Retry } from '../services';
 import { getInputDelays } from '../services/input_delays';
 import { KibanaUrl } from '../services/kibana_url';
 
@@ -40,7 +39,7 @@ export class JourneyFtrHarness {
     private readonly esArchiver: EsArchiver,
     private readonly kibanaServer: KibanaServer,
     private readonly es: Es,
-    private readonly retry: RetryService,
+    private readonly retry: Retry,
     private readonly auth: Auth,
     private readonly journeyConfig: JourneyConfig<any>
   ) {
@@ -63,6 +62,7 @@ export class JourneyFtrHarness {
   private async updateTelemetryAndAPMLabels(labels: { [k: string]: string }) {
     this.log.info(`Updating telemetry & APM labels: ${JSON.stringify(labels)}`);
 
+    try {
       await this.kibanaServer.request({
         path: '/internal/core/_settings',
         method: 'PUT',
@@ -72,6 +72,14 @@ export class JourneyFtrHarness {
         },
         body: { telemetry: { labels } },
       });
+    } catch (error) {
+      const statusCode = (error as AxiosError).response?.status;
+      if (statusCode === 404) {
+        throw new Error(
+          `Failed to update labels, supported Kibana version is 8.11.0+ and must be started with "coreApp.allowDynamicConfigOverrides:true"`
+        );
+      } else throw error;
+    }
   }
 
   private async setupApm() {
@@ -385,7 +393,7 @@ export class JourneyFtrHarness {
     }
 
     const isServerlessProject = !!this.config.get('serverless');
-    const kibanaPage = getNewPageObject(isServerlessProject, page, this.log);
+    const kibanaPage = getNewPageObject(isServerlessProject, page, this.log, this.retry);
 
     this.#_ctx = this.journeyConfig.getExtendedStepCtx({
       kibanaPage,

packages/kbn-journeys/services/auth.ts

@@ -10,9 +10,7 @@ import Url from 'url';
 import { format } from 'util';
 
 import axios, { AxiosResponse } from 'axios';
-import { ToolingLog } from '@kbn/tooling-log';
-import { Config } from '@kbn/test';
-import { KibanaServer } from '@kbn/ftr-common-functional-services';
+import { FtrService } from './ftr_context_provider';
 
 export interface Credentials {
   username: string;
@@ -22,12 +20,10 @@ export interface Credentials {
 function extractCookieValue(authResponse: AxiosResponse) {
   return authResponse.headers['set-cookie']?.[0].toString().split(';')[0].split('sid=')[1] ?? '';
 }
-export class Auth {
-  constructor(
-    private readonly config: Config,
-    private readonly log: ToolingLog,
-    private readonly kibanaServer: KibanaServer
-  ) {}
+export class AuthService extends FtrService {
+  private readonly config = this.ctx.getService('config');
+  private readonly log = this.ctx.getService('log');
+  private readonly kibanaServer = this.ctx.getService('kibanaServer');
 
   public async login(credentials?: Credentials) {
     const baseUrl = new URL(

packages/kbn-journeys/services/es.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+import { createEsClientForFtrConfig, ProvidedType } from '@kbn/test';
+import { FtrProviderContext } from './ftr_context_provider';
+
+export function EsProvider({ getService }: FtrProviderContext): Client {
+  const config = getService('config');
+
+  return createEsClientForFtrConfig(config);
+}
+
+export type Es = ProvidedType<typeof EsProvider>;

packages/kbn-journeys/services/ftr_context_provider.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test';
+import { services } from '.';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;
+export class FtrService extends GenericFtrService<FtrProviderContext> {}

packages/kbn-journeys/services/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { commonFunctionalServices, RetryService } from '@kbn/ftr-common-functional-services';
+import { EsArchiverProvider } from '@kbn/ftr-common-functional-services/services/es_archiver';
+import { KibanaServerProvider } from '@kbn/ftr-common-functional-services/services/kibana_server';
+import { ProvidedType } from '@kbn/test';
+import { EsProvider } from './es';
+import { AuthService } from './auth';
+
+export const services = {
+  es: EsProvider,
+  kibanaServer: commonFunctionalServices.kibanaServer,
+  esArchiver: commonFunctionalServices.esArchiver,
+  retry: commonFunctionalServices.retry,
+  auth: AuthService,
+};
+
+export type EsArchiver = ProvidedType<typeof EsArchiverProvider>;
+export type KibanaServer = ProvidedType<typeof KibanaServerProvider>;
+export type Es = ProvidedType<typeof EsProvider>;
+export type Auth = AuthService;
+export type Retry = RetryService;

packages/kbn-journeys/services/page/index.ts

@@ -8,9 +8,10 @@
 
 import { ToolingLog } from '@kbn/tooling-log';
 import { Page } from 'playwright';
+import { Retry } from '..';
 import { KibanaPage } from './kibana_page';
 import { ProjectPage } from './project_page';
 
-export function getNewPageObject(isServerless: boolean, page: Page, log: ToolingLog) {
-  return isServerless ? new ProjectPage(page, log) : new KibanaPage(page, log);
+export function getNewPageObject(isServerless: boolean, page: Page, log: ToolingLog, retry: Retry) {
+  return isServerless ? new ProjectPage(page, log, retry) : new KibanaPage(page, log, retry);
 }

packages/kbn-journeys/services/page/kibana_page.ts

@@ -9,6 +9,7 @@
 import { subj } from '@kbn/test-subj-selector';
 import { ToolingLog } from '@kbn/tooling-log';
 import { Page } from 'playwright';
+import { Retry } from '..';
 
 interface WaitForRenderArgs {
   expectedItemsCount: number;
@@ -19,10 +20,12 @@ interface WaitForRenderArgs {
 export class KibanaPage {
   readonly page: Page;
   readonly log: ToolingLog;
+  readonly retry: Retry;
 
-  constructor(page: Page, log: ToolingLog) {
+  constructor(page: Page, log: ToolingLog, retry: Retry) {
     this.page = page;
     this.log = log;
+    this.retry = retry;
   }
 
   async waitForHeader() {
@@ -36,26 +39,33 @@ export class KibanaPage {
   }
 
   async waitForRender({ expectedItemsCount, itemLocator, checkAttribute }: WaitForRenderArgs) {
-    try {
-      await this.page.waitForFunction(
-        function renderCompleted(args: WaitForRenderArgs) {
-          const renderingItems = Array.from(document.querySelectorAll(args.itemLocator));
-          const allItemsLoaded = renderingItems.length === args.expectedItemsCount;
-          return allItemsLoaded
-            ? renderingItems.every((e) => e.getAttribute(args.checkAttribute) === 'true')
-            : false;
-        },
-        { expectedItemsCount, itemLocator, checkAttribute }
+    // we can't use `page.waitForFunction` because of CSP while testing on Cloud
+    await this.retry.waitFor(
+      `rendering of ${expectedItemsCount} elements with selector ${itemLocator} is completed`,
+      async () => {
+        const renderingItems = await this.page.$$(itemLocator);
+        if (renderingItems.length === expectedItemsCount) {
+          // all components are loaded, checking if all are rendered
+          const renderStatuses = await Promise.all(
+            renderingItems.map(async (item) => {
+              return (await item.getAttribute(checkAttribute)) === 'true';
+            })
           );
-    } catch (err) {
-      const loaded = await this.page.$$(itemLocator);
-      const rendered = await this.page.$$(`${itemLocator}[${checkAttribute}="true"]`);
-      this.log.error(
-        `'waitForRendering' failed: loaded - ${loaded.length}, rendered - ${rendered.length}, expected count - ${expectedItemsCount}`
+          const rendered = renderStatuses.filter((isRendered) => isRendered === true);
+          this.log.debug(
+            `waitForRender: ${rendered.length} out of ${expectedItemsCount} are rendered...`
           );
-      throw err;
+          return rendered.length === expectedItemsCount;
+        } else {
+          // not all components are loaded yet
+          this.log.debug(
+            `waitForRender: ${renderingItems.length} out of ${expectedItemsCount} are loaded...`
+          );
+          return false;
         }
       }
+    );
+  }
 
   async waitForVisualizations(count: number) {
     await this.waitForRender({

x-pack/performance/journeys/many_fields_discover.ts

@@ -13,7 +13,11 @@ export const journey = new Journey({
   esArchives: ['test/functional/fixtures/es_archiver/many_fields'],
 })
   .step('Go to Discover Page', async ({ page, kbnUrl, kibanaPage }) => {
-    await page.goto(kbnUrl.get(`/app/discover`));
+    await page.goto(
+      kbnUrl.get(
+        `/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:60000),time:(from:now-15m,to:now))&_a=(columns:!(),filters:!(),index:'35796250-bb09-11ec-a8e4-a9868e049a39',interval:auto,query:(language:kuery,query:''),sort:!())`
+      )
+    );
     await kibanaPage.waitForHeader();
     await page.waitForSelector('[data-test-subj="discoverDocTable"][data-render-complete="true"]');
     await page.waitForSelector(subj('globalLoadingIndicator-hidden'));