[Cloud Security] Fix serverless quality gate tests and introduce a better solution tests that are only run in MKI Serverless Quality Gates (#202038)

.buildkite/ftr_security_serverless_configs.yml

@@ -19,6 +19,9 @@ disabled:
   - x-pack/test_serverless/functional/config.base.ts
   - x-pack/test_serverless/shared/config.base.ts
 
+  # MKI only configs files
+  - x-pack/test_serverless/functional/test_suites/security/config.mki_only.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts
@@ -32,7 +35,6 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless.ts
-  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless_api.ts
   - x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/security/config.context_awareness.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts

.github/CODEOWNERS

@@ -2102,7 +2102,9 @@ x-pack/test/api_integration/apis/management/index_management/inference_endpoints
 /x-pack/test/api_integration/services/security_solution_*.gen.ts @elastic/security-solution
 /x-pack/test/accessibility/apps/group3/security_solution.ts @elastic/security-solution
 /x-pack/test_serverless/functional/test_suites/security/config.ts @elastic/security-solution @elastic/appex-qa
-/x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts @elastic/security-solution
+x-pack/test_serverless/functional/test_suites/security/config.mki_only.ts @elastic/security-solution @elastic/appex-qa 
+x-pack/test_serverless/functional/test_suites/security/index.mki_only.ts @elastic/security-solution @elastic/appex-qa @elastic/kibana-cloud-security-posture
+/x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts @elastic/security-solution @elastic/kibana-cloud-security-posture
 /x-pack/test_serverless/api_integration/test_suites/observability/config.feature_flags.ts @elastic/security-solution
 /x-pack/test_serverless/functional/test_suites/common/spaces/multiple_spaces_enabled.ts @elastic/security-solution
 /x-pack/test/functional/es_archives/endpoint/ @elastic/security-solution

x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless.ts

@@ -16,6 +16,8 @@ export default createTestConfig({
     reportName: 'Serverless Security Cloud Security Agentless Onboarding Functional Tests',
   },
   kbnServerArgs: [
+    `--xpack.cloud.serverless.project_id=some_fake_project_id`,
+
     `--xpack.fleet.packages.0.name=cloud_security_posture`,
     `--xpack.fleet.packages.0.version=${CLOUD_CREDENTIALS_PACKAGE_VERSION}`,
     `--xpack.fleet.agentless.enabled=true`,
@@ -26,11 +28,10 @@ export default createTestConfig({
     `--xpack.fleet.agentPolicies.0.id=agentless`,
     `--xpack.fleet.agentPolicies.0.name=agentless`,
     `--xpack.fleet.agentPolicies.0.package_policies=[]`,
-    `--xpack.cloud.serverless.project_id=some_fake_project_id`,
     `--xpack.fleet.agentPolicies.0.is_default=true`,
     `--xpack.fleet.agentPolicies.0.is_default_fleet_server=true`,
 
-    // Serverless Agentless API
+    `--xpack.fleet.agentless.enabled=true`,
     `--xpack.fleet.agentless.api.url=http://localhost:8089`,
     `--xpack.fleet.agentless.api.tls.certificate=${KBN_CERT_PATH}`,
     `--xpack.fleet.agentless.api.tls.key=${KBN_KEY_PATH}`,

x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless_api.ts

@@ -1,34 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { CLOUD_CREDENTIALS_PACKAGE_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants';
-import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
-import { createTestConfig } from '../../config.base';
-
-export default createTestConfig({
-  serverlessProject: 'security',
-  junit: {
-    reportName: 'Serverless Security Cloud Security Agentless API Onboarding Functional Tests',
-  },
-  kbnServerArgs: [
-    `--xpack.fleet.packages.0.name=cloud_security_posture`,
-    `--xpack.fleet.packages.0.version=${CLOUD_CREDENTIALS_PACKAGE_VERSION}`,
-
-    `--xpack.fleet.agents.fleet_server.hosts=["https://ftr.kibana:8220"]`,
-    `--xpack.fleet.internal.fleetServerStandalone=true`,
-
-    // Agentless Configuration based on Serverless Security Dev Yaml - config/serverless.security.dev.yml
-    `--xpack.fleet.agentless.enabled=true`,
-    `--xpack.fleet.agentless.api.url=http://localhost:8089`,
-    `--xpack.fleet.agentless.api.tls.certificate=${KBN_CERT_PATH}`,
-    `--xpack.fleet.agentless.api.tls.key=${KBN_KEY_PATH}`,
-    `--xpack.fleet.agentless.api.tls.ca=${CA_CERT_PATH}`,
-    `--xpack.cloud.serverless.project_id=some_fake_project_id`,
-  ],
-  // load tests in the index file
-  testFiles: [require.resolve('./ftr/cloud_security_posture/agentless_api')],
-});

x-pack/test_serverless/functional/test_suites/security/config.mki_only.ts

@@ -0,0 +1,30 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils';
+import { createTestConfig } from '../../config.base';
+
+export default createTestConfig({
+  serverlessProject: 'security',
+  testFiles: [require.resolve('./index.mki_only.ts')],
+  junit: {
+    reportName: 'Serverless Security MKI Functional Tests',
+  },
+  suiteTags: { exclude: ['skipSvlSec'] },
+
+  // include settings from project controller
+  // https://github.com/elastic/project-controller/blob/main/internal/project/security/config/elasticsearch.yml
+  esServerArgs: ['xpack.ml.nlp.enabled=true'],
+  kbnServerArgs: [
+    '--xpack.dataUsage.enabled=true',
+    '--xpack.dataUsage.enableExperimental=[]',
+    // dataUsage.autoops* config is set in kibana controller
+    '--xpack.dataUsage.autoops.enabled=true',
+    '--xpack.dataUsage.autoops.api.url=http://localhost:9000',
+    `--xpack.dataUsage.autoops.api.tls.certificate=${KBN_CERT_PATH}`,
+    `--xpack.dataUsage.autoops.api.tls.key=${KBN_KEY_PATH}`,
+  ],
+});

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless/cis_integration_aws.ts

@@ -8,7 +8,7 @@ import { CLOUD_CREDENTIALS_PACKAGE_VERSION } from '@kbn/cloud-security-posture-p
 import expect from '@kbn/expect';
 import * as http from 'http';
 import type { FtrProviderContext } from '../../../../../ftr_provider_context';
-import { setupMockServer } from '../agentless_api/mock_agentless_api';
+import { setupMockServer } from './mock_agentless_api';
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const mockAgentlessApiService = setupMockServer();
   const pageObjects = getPageObjects([

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless/cis_integration_gcp.ts

@@ -9,7 +9,7 @@ import expect from '@kbn/expect';
 import { CLOUD_CREDENTIALS_PACKAGE_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants';
 import * as http from 'http';
 import type { FtrProviderContext } from '../../../../../ftr_provider_context';
-import { setupMockServer } from '../agentless_api/mock_agentless_api';
+import { setupMockServer } from './mock_agentless_api';
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const pageObjects = getPageObjects(['common', 'svlCommonPage', 'cisAddIntegration', 'header']);
 

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless/create_agent.ts

@@ -8,12 +8,9 @@
 import { CLOUD_CREDENTIALS_PACKAGE_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants';
 import * as http from 'http';
 import expect from '@kbn/expect';
-import equals from 'fast-deep-equal';
 import { setupMockServer } from './mock_agentless_api';
 import type { FtrProviderContext } from '../../../../../ftr_provider_context';
 export default function ({ getPageObjects, getService }: FtrProviderContext) {
-  const agentCreationTimeout = 1000 * 60 * 1; // 1 minute
-  const retry = getService('retry');
   const mockAgentlessApiService = setupMockServer();
   const pageObjects = getPageObjects([
     'svlCommonPage',
@@ -28,12 +25,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
   const AWS_SINGLE_ACCOUNT_TEST_ID = 'awsSingleTestId';
 
   describe('Agentless API Serverless', function () {
+    this.tags(['skipMKI', 'cloud_security_posture_agentless']);
     let mockApiServer: http.Server;
     let cisIntegration: typeof pageObjects.cisAddIntegration;
 
     before(async () => {
-      // If process.env.TEST_CLOUD is set, then the test is running in the Serverless Quality Gates
-      // and this MSW server will be listening for a request that will never come.
       mockApiServer = mockAgentlessApiService.listen(8089); // Start the usage api mock server on port 8089
       await pageObjects.svlCommonPage.loginAsAdmin();
       cisIntegration = pageObjects.cisAddIntegration;
@@ -57,22 +53,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
       await cisIntegration.selectSetupTechnology('agentless');
       await cisIntegration.selectAwsCredentials('direct');
 
-      if (
-        process.env.TEST_CLOUD &&
-        process.env.CSPM_AWS_ACCOUNT_ID &&
-        process.env.CSPM_AWS_SECRET_KEY
-      ) {
-        await cisIntegration.fillInTextField(
-          cisIntegration.testSubjectIds.DIRECT_ACCESS_KEY_ID_TEST_ID,
-          process.env.CSPM_AWS_ACCOUNT_ID
-        );
-
-        await cisIntegration.fillInTextField(
-          cisIntegration.testSubjectIds.DIRECT_ACCESS_SECRET_KEY_TEST_ID,
-          process.env.CSPM_AWS_SECRET_KEY
-        );
-      }
-
       await pageObjects.header.waitUntilLoadingHasFinished();
 
       await cisIntegration.clickSaveButton();
@@ -85,13 +65,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         integrationPolicyName
       );
 
-      // wait for eventually Pending or Healthy status
-      // purpose of this retry is to wait for the agent to be created and the status to be updated
-      // not to wait for the agent to be healthy
-      await retry.tryForTime(agentCreationTimeout, async () => {
-        const resStatus = await cisIntegration.getFirstCspmIntegrationPageAgentlessStatus();
-        expect(equals(resStatus, 'Healthy') || equals(resStatus, 'Pending')).to.be(true);
-      });
+      const resStatus = await cisIntegration.getFirstCspmIntegrationPageAgentlessStatus();
+      // The status can only be Pending because the agentless agent will never be created
+      expect(resStatus).to.be('Pending');
     });
 
     it(`should create default agent-based agent`, async () => {

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/agentless/index.ts

@@ -12,5 +12,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     this.tags(['cloud_security_posture_agentless']);
     loadTestFile(require.resolve('./cis_integration_aws'));
     loadTestFile(require.resolve('./cis_integration_gcp'));
+    loadTestFile(require.resolve('./create_agent'));
   });
 }

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/mki_only/README.md

@@ -0,0 +1,15 @@
+# MKI Serverless Quality Gates
+
+This folder contains tests that **ONLY** run in the MKI Serverless Quality Gates. These tests are designed to ensure the security and functionality of the system in a serverless environment.
+
+## Contributing
+
+Please prefix the tests in this folder with `mki_` so that is clear to the following developer that these tests run only in MKI and Serverless Quality Gates.
+
+New MKI only test files should be loaded from the root index.ts file of the mki_only directory
+
+```
+x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/mki_only/index.ts
+```
+
+If you would like to contribute to these tests, please follow the contribution guidelines outlined in the main project repository.

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/mki_only/agentless/mki_create_agent.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { CLOUD_CREDENTIALS_PACKAGE_VERSION } from '@kbn/cloud-security-posture-plugin/common/constants';
+import expect from '@kbn/expect';
+import type { FtrProviderContext } from '../../../../../../ftr_provider_context';
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const testSubjects = getService('testSubjects');
+  const pageObjects = getPageObjects([
+    'svlCommonPage',
+    'cspSecurity',
+    'security',
+    'header',
+    'cisAddIntegration',
+  ]);
+
+  const CIS_AWS_OPTION_TEST_ID = 'cisAwsTestId';
+
+  const AWS_SINGLE_ACCOUNT_TEST_ID = 'awsSingleTestId';
+
+  // This test suite is only running in the Serverless Quality Gates environment
+  describe('Agentless API Serverless MKI only', function () {
+    this.tags(['cloud_security_posture_agentless']);
+    let cisIntegration: typeof pageObjects.cisAddIntegration;
+
+    before(async () => {
+      await pageObjects.svlCommonPage.loginAsAdmin();
+      cisIntegration = pageObjects.cisAddIntegration;
+    });
+
+    it(`should create agentless-agent`, async () => {
+      const integrationPolicyName = `cloud_security_posture-${new Date().toISOString()}`;
+      await cisIntegration.navigateToAddIntegrationCspmWithVersionPage(
+        CLOUD_CREDENTIALS_PACKAGE_VERSION
+      );
+
+      await cisIntegration.clickOptionButton(CIS_AWS_OPTION_TEST_ID);
+      await cisIntegration.clickOptionButton(AWS_SINGLE_ACCOUNT_TEST_ID);
+
+      await cisIntegration.inputIntegrationName(integrationPolicyName);
+
+      await cisIntegration.selectSetupTechnology('agentless');
+      await cisIntegration.selectAwsCredentials('direct');
+
+      await pageObjects.header.waitUntilLoadingHasFinished();
+
+      if (process.env.CSPM_AWS_ACCOUNT_ID && process.env.CSPM_AWS_SECRET_KEY) {
+        await cisIntegration.fillInTextField(
+          cisIntegration.testSubjectIds.DIRECT_ACCESS_KEY_ID_TEST_ID,
+          process.env.CSPM_AWS_ACCOUNT_ID
+        );
+
+        await cisIntegration.fillInTextField(
+          cisIntegration.testSubjectIds.DIRECT_ACCESS_SECRET_KEY_TEST_ID,
+          process.env.CSPM_AWS_SECRET_KEY
+        );
+      }
+
+      await cisIntegration.clickSaveButton();
+      await pageObjects.header.waitUntilLoadingHasFinished();
+
+      await cisIntegration.navigateToIntegrationCspList();
+      await pageObjects.header.waitUntilLoadingHasFinished();
+
+      expect(await cisIntegration.getFirstCspmIntegrationPageAgentlessIntegration()).to.be(
+        integrationPolicyName
+      );
+
+      const agentStatusBadge = testSubjects.find('agentlessStatusBadge');
+      // The status badge could be either "Pending", "Healthy",  or "Unhealthy" so we are just checking that it exists
+      expect(agentStatusBadge).to.be.ok();
+    });
+
+    it(`should create default agent-based agent`, async () => {
+      const integrationPolicyName = `cloud_security_posture-${new Date().toISOString()}`;
+
+      await cisIntegration.navigateToAddIntegrationCspmWithVersionPage(
+        CLOUD_CREDENTIALS_PACKAGE_VERSION
+      );
+
+      await cisIntegration.clickOptionButton(CIS_AWS_OPTION_TEST_ID);
+      await cisIntegration.clickOptionButton(AWS_SINGLE_ACCOUNT_TEST_ID);
+
+      await cisIntegration.inputIntegrationName(integrationPolicyName);
+
+      await cisIntegration.clickSaveButton();
+      await pageObjects.header.waitUntilLoadingHasFinished();
+
+      const agentPolicyName = await cisIntegration.getAgentBasedPolicyValue();
+
+      await cisIntegration.navigateToIntegrationCspList();
+      await pageObjects.header.waitUntilLoadingHasFinished();
+
+      expect(await cisIntegration.getFirstCspmIntegrationPageIntegration()).to.be(
+        integrationPolicyName
+      );
+      expect(await cisIntegration.getFirstCspmIntegrationPageAgent()).to.be(agentPolicyName);
+    });
+  });
+}

x-pack/test_serverless/functional/test_suites/security/ftr/cloud_security_posture/mki_only/index.ts

@@ -9,7 +9,9 @@ import { FtrProviderContext } from '../../../../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('cloud_security_posture', function () {
-    this.tags(['failsOnMKI', 'cloud_security_posture']);
-    loadTestFile(require.resolve('./create_agent'));
+    this.tags(['cloud_security_posture']);
+
+    // do not resolve files which are ending with `.essentials.ts`
+    loadTestFile(require.resolve('./agentless/mki_create_agent'));
   });
 }

x-pack/test_serverless/functional/test_suites/security/index.mki_only.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('serverless security UI MKI only', function () {
+    this.tags(['security-mki-only ']);
+    loadTestFile(require.resolve('./ftr/cloud_security_posture/mki_only'));
+  });
+}

x-pack/test_serverless/functional/test_suites/security/index.ts

@@ -8,8 +8,6 @@
 import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function ({ loadTestFile }: FtrProviderContext) {
-  const isCloud = !!process.env.TEST_CLOUD;
-
   describe('serverless security UI', function () {
     this.tags(['esGate']);
 
@@ -17,9 +15,5 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./ftr/cases'));
     loadTestFile(require.resolve('./ftr/advanced_settings'));
     loadTestFile(require.resolve('./ml'));
-    if (isCloud) {
-      // only run the agentless API tests in the Serverless Quality Gates
-      loadTestFile(require.resolve('./ftr/cloud_security_posture/agentless_api'));
-    }
   });
 }