[kbn/journeys] fixes to run journeys against ESS cluster (#166923)

## Summary

I had to change `waitForRender` since `page.waitForFunction` tries to
run a script on page and it is not working due to CSP settings on Cloud.
Instead of injecting a script, we use a classical API to find
elements/attributes in the DOM.

Since `PUT /internal/core/_settings` is merged in 8.11.0, journeys run
on Cloud with on-fly labels update is supported starting deployments
8.11.0+. I added error message for 404 code just in case someone runs it
on earlier version.

`many_fields_discover` journey was update since on Cloud the data view
used by scenario is not selected by default.

How it works:

Create a deployment with QAF and re-configure it for journey run:
```
export EC_DEPLOYMENT_NAME=my-run-8.11
qaf elastic-cloud deployments create --stack-version 8.11.0-SNAPSHOT --environment staging --region gcp-us-central1
qaf elastic-cloud deployments configure-for-performance-journeys
```

Run any journey, e.g. many_fields_discover
```
TEST_CLOUD=1 TEST_ES_URL=https://username:pswd@es_url:443 TEST_KIBANA_URL=https://username:pswd@kibana-ur_url node scripts/functional_test_runner --config x-pack/performance/journeys/many_fields_discover.ts
```

You should see a log about labels being updated:

```
Updating telemetry & APM labels: {"testJobId":"local-a3272047-6724-44d1-9a61-5c79781b06a1","testBuildId":"local-d8edbace-f441-4ba9-ac83-5909be3acf2a","journeyName":"many_fields_discover","ftrConfig":"x-pack/performance/journeys/many_fields_discover.ts"}
```

And then able to find APM logs for the journey in
[Ops](https://kibana-ops-e2e-perf.kb.us-central1.gcp.cloud.es.io:9243/app/apm/services?comparisonEnabled=true&environment=ENVIRONMENT_ALL&kuery=labels.testJobId%20%3A%20%22local-d79a878c-cc7a-423b-b884-c9b6b1a8d781%22&latencyAggregationType=avg&offset=1d&rangeFrom=now-24h%2Fh&rangeTo=now&serviceGroup=&transactionType=request)
cluster

packages/kbn-journeys/journey/journey.ts

@@ -12,14 +12,9 @@ import { Page } from 'playwright';
 import callsites from 'callsites';
 import { ToolingLog } from '@kbn/tooling-log';
 import { FtrConfigProvider } from '@kbn/test';
-import {
-  FtrProviderContext,
-  KibanaServer,
-  Es,
-  RetryService,
-} from '@kbn/ftr-common-functional-services';
+import { FtrProviderContext } from '../services/ftr_context_provider';
+import { Es, KibanaServer, Retry, Auth } from '../services';
 
-import { Auth } from '../services/auth';
 import { InputDelays } from '../services/input_delays';
 import { KibanaUrl } from '../services/kibana_url';
 
@@ -37,7 +32,7 @@ export interface BaseStepCtx {
   kbnUrl: KibanaUrl;
   kibanaServer: KibanaServer;
   es: Es;
-  retry: RetryService;
+  retry: Retry;
   auth: Auth;
 }
 
@@ -141,7 +136,7 @@ export class Journey<CtxExt extends object> {
       getService('kibanaServer'),
       getService('es'),
       getService('retry'),
-      new Auth(getService('config'), getService('log'), getService('kibanaServer')),
+      getService('auth'),
       this.config
     ).initMochaSuite(this.#steps);
   }

packages/kbn-journeys/journey/journey_ftr_config.ts

@@ -11,7 +11,7 @@ import Path from 'path';
 import { v4 as uuidV4 } from 'uuid';
 import { REPO_ROOT } from '@kbn/repo-info';
 import type { FtrConfigProviderContext, FtrConfigProvider } from '@kbn/test';
-import { commonFunctionalServices } from '@kbn/ftr-common-functional-services';
+import { services } from '../services';
 
 import { AnyStep } from './journey';
 import { JourneyConfig } from './journey_config';
@@ -66,7 +66,7 @@ export function makeFtrConfigProvider(
         bail: true,
       },
 
-      services: commonFunctionalServices,
+      services,
       pageObjects: {},
 
       servicesRequiredForTestAnalysis: ['performance', 'journeyConfig'],

packages/kbn-journeys/journey/journey_ftr_harness.ts

@@ -9,20 +9,19 @@
 import Url from 'url';
 import { inspect, format } from 'util';
 import { setTimeout } from 'timers/promises';
-
 import * as Rx from 'rxjs';
 import apmNode from 'elastic-apm-node';
 import playwright, { ChromiumBrowser, Page, BrowserContext, CDPSession, Request } from 'playwright';
 import { asyncMap, asyncForEach } from '@kbn/std';
 import { ToolingLog } from '@kbn/tooling-log';
 import { Config } from '@kbn/test';
-import { EsArchiver, KibanaServer, Es, RetryService } from '@kbn/ftr-common-functional-services';
 import {
   ELASTIC_HTTP_VERSION_HEADER,
   X_ELASTIC_INTERNAL_ORIGIN_REQUEST,
 } from '@kbn/core-http-common';
 
-import { Auth } from '../services/auth';
+import { AxiosError } from 'axios';
+import { Auth, Es, EsArchiver, KibanaServer, Retry } from '../services';
 import { getInputDelays } from '../services/input_delays';
 import { KibanaUrl } from '../services/kibana_url';
 
@@ -40,7 +39,7 @@ export class JourneyFtrHarness {
     private readonly esArchiver: EsArchiver,
     private readonly kibanaServer: KibanaServer,
     private readonly es: Es,
-    private readonly retry: RetryService,
+    private readonly retry: Retry,
     private readonly auth: Auth,
     private readonly journeyConfig: JourneyConfig<any>
   ) {
@@ -63,15 +62,24 @@ export class JourneyFtrHarness {
   private async updateTelemetryAndAPMLabels(labels: { [k: string]: string }) {
     this.log.info(`Updating telemetry & APM labels: ${JSON.stringify(labels)}`);
 
-    await this.kibanaServer.request({
-      path: '/internal/core/_settings',
-      method: 'PUT',
-      headers: {
-        [ELASTIC_HTTP_VERSION_HEADER]: '1',
-        [X_ELASTIC_INTERNAL_ORIGIN_REQUEST]: 'ftr',
-      },
-      body: { telemetry: { labels } },
-    });
+    try {
+      await this.kibanaServer.request({
+        path: '/internal/core/_settings',
+        method: 'PUT',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '1',
+          [X_ELASTIC_INTERNAL_ORIGIN_REQUEST]: 'ftr',
+        },
+        body: { telemetry: { labels } },
+      });
+    } catch (error) {
+      const statusCode = (error as AxiosError).response?.status;
+      if (statusCode === 404) {
+        throw new Error(
+          `Failed to update labels, supported Kibana version is 8.11.0+ and must be started with "coreApp.allowDynamicConfigOverrides:true"`
+        );
+      } else throw error;
+    }
   }
 
   private async setupApm() {
@@ -385,7 +393,7 @@ export class JourneyFtrHarness {
     }
 
     const isServerlessProject = !!this.config.get('serverless');
-    const kibanaPage = getNewPageObject(isServerlessProject, page, this.log);
+    const kibanaPage = getNewPageObject(isServerlessProject, page, this.log, this.retry);
 
     this.#_ctx = this.journeyConfig.getExtendedStepCtx({
       kibanaPage,

packages/kbn-journeys/services/auth.ts

@@ -10,9 +10,7 @@ import Url from 'url';
 import { format } from 'util';
 
 import axios, { AxiosResponse } from 'axios';
-import { ToolingLog } from '@kbn/tooling-log';
-import { Config } from '@kbn/test';
-import { KibanaServer } from '@kbn/ftr-common-functional-services';
+import { FtrService } from './ftr_context_provider';
 
 export interface Credentials {
   username: string;
@@ -22,12 +20,10 @@ export interface Credentials {
 function extractCookieValue(authResponse: AxiosResponse) {
   return authResponse.headers['set-cookie']?.[0].toString().split(';')[0].split('sid=')[1] ?? '';
 }
-export class Auth {
-  constructor(
-    private readonly config: Config,
-    private readonly log: ToolingLog,
-    private readonly kibanaServer: KibanaServer
-  ) {}
+export class AuthService extends FtrService {
+  private readonly config = this.ctx.getService('config');
+  private readonly log = this.ctx.getService('log');
+  private readonly kibanaServer = this.ctx.getService('kibanaServer');
 
   public async login(credentials?: Credentials) {
     const baseUrl = new URL(

packages/kbn-journeys/services/es.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+import { createEsClientForFtrConfig, ProvidedType } from '@kbn/test';
+import { FtrProviderContext } from './ftr_context_provider';
+
+export function EsProvider({ getService }: FtrProviderContext): Client {
+  const config = getService('config');
+
+  return createEsClientForFtrConfig(config);
+}
+
+export type Es = ProvidedType<typeof EsProvider>;

packages/kbn-journeys/services/ftr_context_provider.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { GenericFtrProviderContext, GenericFtrService } from '@kbn/test';
+import { services } from '.';
+
+export type FtrProviderContext = GenericFtrProviderContext<typeof services, {}>;
+export class FtrService extends GenericFtrService<FtrProviderContext> {}

packages/kbn-journeys/services/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { commonFunctionalServices, RetryService } from '@kbn/ftr-common-functional-services';
+import { EsArchiverProvider } from '@kbn/ftr-common-functional-services/services/es_archiver';
+import { KibanaServerProvider } from '@kbn/ftr-common-functional-services/services/kibana_server';
+import { ProvidedType } from '@kbn/test';
+import { EsProvider } from './es';
+import { AuthService } from './auth';
+
+export const services = {
+  es: EsProvider,
+  kibanaServer: commonFunctionalServices.kibanaServer,
+  esArchiver: commonFunctionalServices.esArchiver,
+  retry: commonFunctionalServices.retry,
+  auth: AuthService,
+};
+
+export type EsArchiver = ProvidedType<typeof EsArchiverProvider>;
+export type KibanaServer = ProvidedType<typeof KibanaServerProvider>;
+export type Es = ProvidedType<typeof EsProvider>;
+export type Auth = AuthService;
+export type Retry = RetryService;

packages/kbn-journeys/services/page/index.ts

@@ -8,9 +8,10 @@
 
 import { ToolingLog } from '@kbn/tooling-log';
 import { Page } from 'playwright';
+import { Retry } from '..';
 import { KibanaPage } from './kibana_page';
 import { ProjectPage } from './project_page';
 
-export function getNewPageObject(isServerless: boolean, page: Page, log: ToolingLog) {
-  return isServerless ? new ProjectPage(page, log) : new KibanaPage(page, log);
+export function getNewPageObject(isServerless: boolean, page: Page, log: ToolingLog, retry: Retry) {
+  return isServerless ? new ProjectPage(page, log, retry) : new KibanaPage(page, log, retry);
 }

packages/kbn-journeys/services/page/kibana_page.ts

@@ -9,6 +9,7 @@
 import { subj } from '@kbn/test-subj-selector';
 import { ToolingLog } from '@kbn/tooling-log';
 import { Page } from 'playwright';
+import { Retry } from '..';
 
 interface WaitForRenderArgs {
   expectedItemsCount: number;
@@ -19,10 +20,12 @@ interface WaitForRenderArgs {
 export class KibanaPage {
   readonly page: Page;
   readonly log: ToolingLog;
+  readonly retry: Retry;
 
-  constructor(page: Page, log: ToolingLog) {
+  constructor(page: Page, log: ToolingLog, retry: Retry) {
     this.page = page;
     this.log = log;
+    this.retry = retry;
   }
 
   async waitForHeader() {
@@ -36,25 +39,32 @@ export class KibanaPage {
   }
 
   async waitForRender({ expectedItemsCount, itemLocator, checkAttribute }: WaitForRenderArgs) {
-    try {
-      await this.page.waitForFunction(
-        function renderCompleted(args: WaitForRenderArgs) {
-          const renderingItems = Array.from(document.querySelectorAll(args.itemLocator));
-          const allItemsLoaded = renderingItems.length === args.expectedItemsCount;
-          return allItemsLoaded
-            ? renderingItems.every((e) => e.getAttribute(args.checkAttribute) === 'true')
-            : false;
-        },
-        { expectedItemsCount, itemLocator, checkAttribute }
-      );
-    } catch (err) {
-      const loaded = await this.page.$$(itemLocator);
-      const rendered = await this.page.$$(`${itemLocator}[${checkAttribute}="true"]`);
-      this.log.error(
-        `'waitForRendering' failed: loaded - ${loaded.length}, rendered - ${rendered.length}, expected count - ${expectedItemsCount}`
-      );
-      throw err;
-    }
+    // we can't use `page.waitForFunction` because of CSP while testing on Cloud
+    await this.retry.waitFor(
+      `rendering of ${expectedItemsCount} elements with selector ${itemLocator} is completed`,
+      async () => {
+        const renderingItems = await this.page.$$(itemLocator);
+        if (renderingItems.length === expectedItemsCount) {
+          // all components are loaded, checking if all are rendered
+          const renderStatuses = await Promise.all(
+            renderingItems.map(async (item) => {
+              return (await item.getAttribute(checkAttribute)) === 'true';
+            })
+          );
+          const rendered = renderStatuses.filter((isRendered) => isRendered === true);
+          this.log.debug(
+            `waitForRender: ${rendered.length} out of ${expectedItemsCount} are rendered...`
+          );
+          return rendered.length === expectedItemsCount;
+        } else {
+          // not all components are loaded yet
+          this.log.debug(
+            `waitForRender: ${renderingItems.length} out of ${expectedItemsCount} are loaded...`
+          );
+          return false;
+        }
+      }
+    );
   }
 
   async waitForVisualizations(count: number) {