[EDR Workflows] Reenable and improve endpoint FTR tests on serverless (#189248)

## Summary

This PR re-enables our `endpoint` FTRs, and tries to address the
serverless login/forceLogout flakiness both in `endpoint` and
`integration` FTRs, by using a new `svlCommonPage.loginWithRole()` login
functionality, suggested in this comment:
https://github.com/elastic/kibana/pull/184202#discussion_r1658424380

flaky runner:
https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6678

closes: #186089
closes: #186086

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/test/security_solution_api_integration/tsconfig.json

@@ -13,6 +13,7 @@
     "target/**/*"
   ],
   "kbn_references": [
+    { "path": "../security_solution_endpoint/tsconfig.json" },
     "@kbn/test-suites-serverless",
     { "path": "../../test_serverless/api_integration/**/*" },
     { "path": "../../test_serverless/shared/**/*" },

x-pack/test/security_solution_endpoint/apps/endpoint/index.ts

@@ -12,8 +12,7 @@ import { FtrProviderContext } from '../../configs/ftr_provider_context';
 export default function (providerContext: FtrProviderContext) {
   const { loadTestFile, getService, getPageObjects } = providerContext;
 
-  // Flaky: https://github.com/elastic/kibana/issues/186089
-  describe('@skipInServerless endpoint', function () {
+  describe('endpoint', function () {
     const ingestManager = getService('ingestManager');
     const log = getService('log');
     const endpointTestResources = getService('endpointTestResources');
@@ -38,7 +37,7 @@ export default function (providerContext: FtrProviderContext) {
       if (await isServerlessKibanaFlavor(kbnClient)) {
         log.info('login for serverless environment');
         const pageObjects = getPageObjects(['svlCommonPage']);
-        await pageObjects.svlCommonPage.login();
+        await pageObjects.svlCommonPage.loginWithRole('endpoint_operations_analyst');
       }
     });
     loadTestFile(require.resolve('./endpoint_list'));

x-pack/test/security_solution_endpoint/apps/integrations/index.ts

@@ -12,7 +12,6 @@ import { FtrProviderContext } from '../../configs/ftr_provider_context';
 export default function (providerContext: FtrProviderContext) {
   const { loadTestFile, getService, getPageObjects } = providerContext;
 
-  // Flaky: https://github.com/elastic/kibana/issues/186086
   describe('endpoint', function () {
     const ingestManager = getService('ingestManager');
     const log = getService('log');
@@ -37,7 +36,7 @@ export default function (providerContext: FtrProviderContext) {
       if (await isServerlessKibanaFlavor(kbnClient)) {
         log.info('login for serverless environment');
         const pageObjects = getPageObjects(['svlCommonPage']);
-        await pageObjects.svlCommonPage.login();
+        await pageObjects.svlCommonPage.loginWithRole('admin');
       }
     });
     loadTestFile(require.resolve('./policy_list'));

x-pack/test/security_solution_endpoint/configs/config.base.ts

@@ -8,8 +8,9 @@
 import { Config } from '@kbn/test';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { SecuritySolutionEndpointRegistryHelpers } from '../../common/services/security_solution';
-import { pageObjects } from '../page_objects';
 import type { TargetTags } from '../target_tags';
+import { PageObjects } from '../page_objects';
+import { Services } from '../services';
 
 export const SUITE_TAGS: Record<
   'ess' | 'serverless',
@@ -33,6 +34,7 @@ export const generateConfig = async ({
   kbnServerArgs = [],
   target,
   services,
+  pageObjects,
 }: {
   ftrConfigProviderContext: FtrConfigProviderContext;
   baseConfig: Config;
@@ -40,7 +42,8 @@ export const generateConfig = async ({
   junitReportName: string;
   kbnServerArgs?: string[];
   target: keyof typeof SUITE_TAGS;
-  services: any;
+  services: Services;
+  pageObjects: PageObjects;
 }): Promise<Config> => {
   const { readConfigFile } = ftrConfigProviderContext;
   // services are not ready yet, so we need to import them here

x-pack/test/security_solution_endpoint/configs/endpoint.config.ts

@@ -9,6 +9,7 @@ import { FtrConfigProviderContext } from '@kbn/test';
 import { resolve } from 'path';
 import { generateConfig } from './config.base';
 import { services } from '../services';
+import { pageObjects } from '../page_objects';
 
 // eslint-disable-next-line import/no-default-export
 export default async function (ftrConfigProviderContext: FtrConfigProviderContext) {
@@ -25,5 +26,6 @@ export default async function (ftrConfigProviderContext: FtrConfigProviderContex
     junitReportName: 'X-Pack Endpoint Functional Tests on ESS',
     target: 'ess',
     services,
+    pageObjects,
   });
 }

x-pack/test/security_solution_endpoint/configs/ftr_provider_context.d.ts

@@ -7,7 +7,10 @@
 
 import { GenericFtrProviderContext } from '@kbn/test';
 
-import { pageObjects } from '../page_objects';
-import { services } from '../services';
+import { pageObjects, svlPageObjects } from '../page_objects';
+import { services, svlServices } from '../services';
 
-export type FtrProviderContext = GenericFtrProviderContext<typeof services, typeof pageObjects>;
+export type FtrProviderContext = GenericFtrProviderContext<
+  typeof services & typeof svlServices,
+  typeof pageObjects & typeof svlPageObjects
+>;

x-pack/test/security_solution_endpoint/configs/integrations.config.ts

@@ -9,6 +9,7 @@ import { resolve } from 'path';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { generateConfig } from './config.base';
 import { services } from '../services';
+import { pageObjects } from '../page_objects';
 
 // eslint-disable-next-line import/no-default-export
 export default async function (ftrConfigProviderContext: FtrConfigProviderContext) {
@@ -29,5 +30,6 @@ export default async function (ftrConfigProviderContext: FtrConfigProviderContex
       '--xpack.securitySolution.packagerTaskInterval=5s',
     ],
     services,
+    pageObjects,
   });
 }

x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts

@@ -9,6 +9,7 @@ import { FtrConfigProviderContext } from '@kbn/test';
 import { resolve } from 'path';
 import { generateConfig } from './config.base';
 import { svlServices } from '../services';
+import { svlPageObjects } from '../page_objects';
 
 // eslint-disable-next-line import/no-default-export
 export default async function (ftrConfigProviderContext: FtrConfigProviderContext) {
@@ -26,5 +27,6 @@ export default async function (ftrConfigProviderContext: FtrConfigProviderContex
     kbnServerArgs: ['--serverless=security'],
     target: 'serverless',
     services: svlServices,
+    pageObjects: svlPageObjects,
   });
 }

x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts

@@ -9,6 +9,7 @@ import { resolve } from 'path';
 import { FtrConfigProviderContext } from '@kbn/test';
 import { generateConfig } from './config.base';
 import { svlServices } from '../services';
+import { svlPageObjects } from '../page_objects';
 
 // eslint-disable-next-line import/no-default-export
 export default async function (ftrConfigProviderContext: FtrConfigProviderContext) {
@@ -30,5 +31,6 @@ export default async function (ftrConfigProviderContext: FtrConfigProviderContex
       '--xpack.securitySolution.packagerTaskInterval=5s',
     ],
     services: svlServices,
+    pageObjects: svlPageObjects,
   });
 }

x-pack/test/security_solution_endpoint/page_objects/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { SvlCommonPageProvider } from '@kbn/test-suites-serverless/functional/page_objects/svl_common_page';
 import { pageObjects as xpackFunctionalPageObjects } from '../../functional/page_objects';
 import { EndpointPageProvider } from './endpoint_page';
 import { EndpointPageUtils } from './page_utils';
@@ -17,7 +18,6 @@ import { EndpointPolicyPageProvider } from './policy_page';
 import { TrustedAppsPageProvider } from './trusted_apps_page';
 import { FleetIntegrations } from './fleet_integrations_page';
 import { ArtifactEntriesListPageProvider } from './artifact_entries_list_page';
-import { SvlCommonPageProvider } from './svl_common_page';
 
 export const pageObjects = {
   ...xpackFunctionalPageObjects,
@@ -33,6 +33,12 @@ export const pageObjects = {
   trustedApps: TrustedAppsPageProvider,
   artifactEntriesList: ArtifactEntriesListPageProvider,
   fleetIntegrations: FleetIntegrations,
+};
+
+export const svlPageObjects = {
+  ...pageObjects,
 
   svlCommonPage: SvlCommonPageProvider,
 };
+
+export type PageObjects = typeof pageObjects | typeof svlPageObjects;

x-pack/test/security_solution_endpoint/page_objects/svl_common_page.ts

@@ -1,118 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { FtrProviderContext } from '../configs/ftr_provider_context';
-
-/** copied from `x-pack/test_serverless/functional/page_objects` in order to be able to login when testing against serverless,
- *  without importing from a different project
- */
-export function SvlCommonPageProvider({ getService, getPageObjects }: FtrProviderContext) {
-  const testSubjects = getService('testSubjects');
-  const config = getService('config');
-  const pageObjects = getPageObjects(['security', 'common']);
-  const retry = getService('retry');
-  const deployment = getService('deployment');
-  const log = getService('log');
-  const browser = getService('browser');
-
-  const delay = (ms: number) =>
-    new Promise((resolve) => {
-      setTimeout(resolve, ms);
-    });
-
-  return {
-    async navigateToLoginForm() {
-      const url = deployment.getHostPort() + '/login';
-      await browser.get(url);
-      // ensure welcome screen won't be shown. This is relevant for environments which don't allow
-      // to use the yml setting, e.g. cloud
-      await browser.setLocalStorageItem('home:welcome:show', 'false');
-
-      log.debug('Waiting for Login Form to appear.');
-      await retry.waitForWithTimeout('login form', 10_000, async () => {
-        return await pageObjects.security.isLoginFormVisible();
-      });
-    },
-
-    async login() {
-      await pageObjects.security.forceLogout({ waitForLoginPage: false });
-
-      // adding sleep to settle down logout
-      await pageObjects.common.sleep(2500);
-
-      await retry.waitForWithTimeout(
-        'Waiting for successful authentication',
-        90_000,
-        async () => {
-          if (!(await testSubjects.exists('loginUsername', { timeout: 1000 }))) {
-            await this.navigateToLoginForm();
-
-            await testSubjects.setValue('loginUsername', config.get('servers.kibana.username'));
-            await testSubjects.setValue('loginPassword', config.get('servers.kibana.password'));
-            await testSubjects.click('loginSubmit');
-          }
-
-          if (await testSubjects.exists('userMenuButton', { timeout: 10_000 })) {
-            log.debug('userMenuButton is found, logged in passed');
-            return true;
-          } else {
-            throw new Error(`Failed to login to Kibana via UI`);
-          }
-        },
-        async () => {
-          // Sometimes authentication fails and user is redirected to Cloud login page
-          // [plugins.security.authentication] Authentication attempt failed: UNEXPECTED_SESSION_ERROR
-          const currentUrl = await browser.getCurrentUrl();
-          if (currentUrl.startsWith('https://cloud.elastic.co')) {
-            log.debug(
-              'Probably authentication attempt failed, we are at Cloud login page. Retrying from scratch'
-            );
-          } else {
-            const authError = await testSubjects.exists('promptPage', { timeout: 2500 });
-            if (authError) {
-              log.debug('Probably SAML callback page, doing logout again');
-              await pageObjects.security.forceLogout({ waitForLoginPage: false });
-            } else {
-              const isOnLoginPage = await testSubjects.exists('loginUsername', { timeout: 1000 });
-              if (isOnLoginPage) {
-                log.debug(
-                  'Probably ES user profile activation failed, waiting 2 seconds and pressing Login button again'
-                );
-                await delay(2000);
-                await testSubjects.click('loginSubmit');
-              } else {
-                log.debug('New behaviour, trying to navigate and login again');
-              }
-            }
-          }
-        }
-      );
-      log.debug('Logged in successfully');
-    },
-
-    async forceLogout() {
-      await pageObjects.security.forceLogout({ waitForLoginPage: false });
-      log.debug('Logged out successfully');
-    },
-
-    async assertProjectHeaderExists() {
-      await testSubjects.existOrFail('kibanaProjectHeader');
-    },
-
-    async clickUserAvatar() {
-      testSubjects.click('userMenuAvatar');
-    },
-
-    async assertUserAvatarExists() {
-      await testSubjects.existOrFail('userMenuAvatar');
-    },
-
-    async assertUserMenuExists() {
-      await testSubjects.existOrFail('userMenu');
-    },
-  };
-}

x-pack/test/security_solution_endpoint/services/index.ts

@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+import { SvlUserManagerProvider } from '@kbn/test-suites-serverless/shared/services/svl_user_manager';
+import { SvlCommonApiServiceProvider } from '@kbn/test-suites-serverless/shared/services/svl_common_api';
 import { services as xPackFunctionalServices } from '../../functional/services';
 import { IngestManagerProvider } from '../../common/services/ingest_manager';
 import { EndpointTelemetryTestResourcesProvider } from './endpoint_telemetry';
@@ -39,4 +41,9 @@ export const svlServices = {
 
   supertest: KibanaSupertestWithCertProvider,
   supertestWithoutAuth: KibanaSupertestWithCertWithoutAuthProvider,
+
+  svlCommonApi: SvlCommonApiServiceProvider,
+  svlUserManager: SvlUserManagerProvider,
 };
+
+export type Services = typeof services | typeof svlServices;

x-pack/test/security_solution_endpoint/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "node",
+      "@kbn/ambient-ftr-types"
+    ],
+    "allowJs": false
+  },
+  "include": [
+    "**/*",
+    "../../../typings/**/*",
+    "../../../packages/kbn-test/types/ftr_globals/**/*"
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/test-suites-serverless",
+    "@kbn/expect",
+    "@kbn/security-solution-plugin",
+    "@kbn/repo-info",
+    "@kbn/securitysolution-list-constants",
+    "@kbn/fleet-plugin",
+    "@kbn/securitysolution-io-ts-list-types",
+    "@kbn/ftr-common-functional-ui-services",
+    "@kbn/test",
+    "@kbn/test-subj-selector",
+  ]
+}

x-pack/test/tsconfig.json

@@ -24,7 +24,8 @@
     "*/plugins/**/*",
     "*/packages/**/*",
     "*/*/packages/**/*",
-    "security_solution_api_integration/**/*"
+    "security_solution_api_integration/**/*",
+    "security_solution_endpoint/**/*"
   ],
   "kbn_references": [
     "@kbn/test-suites-src",
@@ -166,7 +167,6 @@
     "@kbn/alerting-state-types",
     "@kbn/reporting-server",
     "@kbn/data-quality-plugin",
-    "@kbn/securitysolution-io-ts-list-types",
     "@kbn/ml-trained-models-utils",
     "@kbn/openapi-common",
     "@kbn/securitysolution-lists-common",