[Security Solution][Exceptions][API testing] Refactor commands in package.json (#170025)

## Summary

- Refactor package.json to remove duplicates 
- Move the new term rules

x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts

@@ -21,7 +21,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
     loadTestFile(require.resolve('./create_index'));
     loadTestFile(require.resolve('./preview_rules'));
     loadTestFile(require.resolve('./create_rules_bulk'));
-    loadTestFile(require.resolve('./create_new_terms'));
     loadTestFile(require.resolve('./delete_rules'));
     loadTestFile(require.resolve('./delete_rules_bulk'));
     loadTestFile(require.resolve('./export_rules'));

x-pack/test/security_solution_api_integration/README.md

@@ -40,7 +40,53 @@ ex:
 
 In the `package.json` file, you'll find commands to configure the server for each environment and to run tests against that specific environment. These commands adhere to the Mocha tagging system, allowing for the inclusion and exclusion of tags, mirroring the setup of the CI pipeline.
 
+## Running Commands with Different Parameters
+
+In this project, you can run various commands to execute tests and workflows, each of which can be customized by specifying different parameters. Below, how to define the commands based on the parameters and their order.
+
+### Command Structure
+
+The command structure follows this pattern:
+
+- `<command-name>`: The name of the specific command or test case.
+- `<folder>`: The test folder or workflow you want to run.
+- `<type>`: The type of operation, either "server" or "runner."
+- `<environment>`: The testing environment, such as "serverlessEnv," "essEnv," or "qaEnv."
+- `<licenseFolder>`: The license folder the test is defined under such as "default_license", by default the value is "default_license"
+- `<area>`: The area the test is defined under, such as "detection_engine", by default the value is "detection_engine"
+
+### Serverless and Ess Configuration
+
+- When using "serverless" or "ess" in the script, it specifies the correct configuration file for the tests.
+- "Serverless" and "ess" help determine the configuration specific to the chosen test.
+
+### serverlessEnv, essEnv, qaEnv Grep Command
+
+- When using "serverlessEnv,.." in the script, it appends the correct grep command for filtering tests in the serverless testing environment.
+- "serverlessEnv,..." is used to customize the test execution based on the serverless environment.
 
 
+### Command Examples
 
+Here are some command examples using the provided parameters:
 
+1. **Run the server for "exception_workflows" in the "serverlessEnv" environment:**
+   ```shell
+   npm run initialize-server exceptions/workflows serverless
+   ```
+2. **To run tests for the "exception_workflows" using the serverless runner in the "serverlessEnv" environment, you can use the following command:**
+    ```shell
+    npm run run-tests exceptions/workflows serverless serverlessEnv
+    ```
+3. **Run tests for "exception_workflows" using the serverless runner in the "qaEnv" environment:**
+    ```shell
+    npm run run-tests exceptions/workflows serverless qaEnv
+    ```
+4. **Run the server for "exception_workflows" in the "essEnv" environment:**
+   ```shell
+    npm run initialize-server exceptions/workflows ess
+   ```
+5. **Run tests for "exception_workflows" using the ess runner in the "essEnv" environment:**   
+   ```shell
+   npm run run-tests exceptions/workflows ess essEnv
+   ```

x-pack/test/security_solution_api_integration/package.json

@@ -5,30 +5,32 @@
   "private": true,
   "license": "Elastic License 2.0",
   "scripts": {
-    "exception_workflows:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts",
-    "exception_workflows:runner:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts --grep @serverless --grep  @brokenInServerless --invert",
-    "exception_workflows:qa:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert",
-    "exception_workflows:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts",
-    "exception_workflows:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts --grep @ess",
-    "exception_operators_date_numeric_types:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts",
-    "exception_operators_date_numeric_types:runner:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts --grep @serverless --grep  @brokenInServerless --invert",
-    "exception_operators_date_numeric_types:qa:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts --grep @serverless --grep  '@brokenInServerless|@skipInQA' --invert",
-    "exception_operators_date_numeric_types:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts",
-    "exception_operators_date_numeric_types:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts --grep @ess",
-    "exception_operators_keyword_text_long:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts",
-    "exception_operators_keyword_text_long:runner:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts --grep @serverless --grep  @brokenInServerless --invert",
-    "exception_operators_keyword_text_long:qa:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts --grep @serverless --grep  '@brokenInServerless|@skipInQA' --invert",
-    "exception_operators_keyword_text_long:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts",
-    "exception_operators_keyword_text_long:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts --grep @ess",
-    "exception_operators_ips_text_array:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts",
-    "exception_operators_ips_text_array:runner:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts --grep @serverless --grep  @brokenInServerless --invert",
-    "exception_operators_ips_text_array:qa:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts --grep @serverless --grep  '@brokenInServerless|@skipInQA' --invert",
-    "exception_operators_ips_text_array:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts",
-    "exception_operators_ips_text_array:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts --grep @ess",
-    "rule_creation:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts",
-    "rule_creation:runner:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts --grep @serverless --grep  @brokenInServerless --invert",
-    "rule_creation:qa:serverless": "node ../../../scripts/functional_test_runner  --config=test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts --grep @serverless --grep  '@brokenInServerless|@skipInQA' --invert",
-    "rule_creation:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts",
-    "rule_creation:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts --grep @ess"
+    "initialize-server": "node ./scripts/index.js server",
+    "run-tests": "node ./scripts/index.js runner",
+    "exception_workflows:server:serverless": "npm run initialize-server exceptions/workflows serverless",
+    "exception_workflows:runner:serverless": "npm run run-tests exceptions/workflows serverless serverlessEnv",
+    "exception_workflows:qa:serverless": "npm run run-tests exceptions/workflows serverless qaEnv",
+    "exception_workflows:server:ess": "npm run initialize-server exceptions/workflows ess",
+    "exception_workflows:runner:ess": "npm run run-tests exceptions/workflows ess essEnv",
+    "exception_operators_date_numeric_types:server:serverless": "npm run initialize-server exceptions/operators_data_types/date_numeric_types serverless",
+    "exception_operators_date_numeric_types:runner:serverless": "npm run run-tests exceptions/operators_data_types/date_numeric_types serverless serverlessEnv",
+    "exception_operators_date_numeric_types:qa:serverless": "npm run run-tests exceptions/operators_data_types/date_numeric_types serverless qaEnv",
+    "exception_operators_date_numeric_types:server:ess": "npm run initialize-server exceptions/operators_data_types/date_numeric_types ess",
+    "exception_operators_date_numeric_types:runner:ess": "npm run run-tests exceptions/operators_data_types/date_numeric_types ess essEnv",
+    "exception_operators_keyword_text_long:server:serverless": "npm run initialize-server exceptions/operators_data_types/keyword_text_long serverless",
+    "exception_operators_keyword_text_long:runner:serverless": "npm run run-tests exceptions/operators_data_types/keyword_text_long serverless serverlessEnv",
+    "exception_operators_keyword_text_long:qa:serverless": "npm run run-tests exceptions/operators_data_types/keyword_text_long serverless qaEnv",
+    "exception_operators_keyword_text_long:server:ess": "npm run initialize-server exceptions/operators_data_types/keyword_text_long ess",
+    "exception_operators_keyword_text_long:runner:ess": "npm run run-tests exceptions/operators_data_types/keyword_text_long ess essEnv",
+    "exception_operators_ips_text_array:server:serverless": "npm run initialize-server exceptions/operators_data_types/ips_text_array serverless",
+    "exception_operators_ips_text_array:runner:serverless": "npm run run-tests exceptions/operators_data_types/ips_text_array serverless serverlessEnv",
+    "exception_operators_ips_text_array:qa:serverless": "npm run run-tests exceptions/operators_data_types/ips_text_array serverless qaEnv",
+    "exception_operators_ips_text_array:server:ess": "npm run initialize-server exceptions/operators_data_types/ips_text_array ess",
+    "exception_operators_ips_text_array:runner:ess": "npm run run-tests exceptions/operators_data_types/ips_text_array ess essEnv",
+    "rule_creation:server:serverless": "npm run initialize-server rule_creation serverless",
+    "rule_creation:runner:serverless": "npm run run-tests rule_creation serverless serverlessEnv",
+    "rule_creation:qa:serverless": "npm run run-tests rule_creation serverless qaEnv",
+    "rule_creation:server:ess": "npm run initialize-server rule_creation ess",
+    "rule_creation:runner:ess": "npm run run-tests rule_creation ess essEnv"
   }
 }

x-pack/test/security_solution_api_integration/scripts/index.js

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+const { spawn } = require('child_process');
+
+const [
+  ,
+  ,
+  type,
+  folder,
+  projectType,
+  environment,
+  area = 'detections_response',
+  licenseFolder = 'default_license',
+  ...args
+] = process.argv;
+
+const configPath = `./test_suites/${area}/${licenseFolder}/${folder}/configs/${projectType}.config.ts`;
+
+const command =
+  type === 'server'
+    ? '../../scripts/functional_tests_server.js'
+    : '../../scripts/functional_test_runner';
+
+let grepArgs = [];
+
+if (type !== 'server') {
+  switch (environment) {
+    case 'serverlessEnv':
+      grepArgs = ['--grep', '@serverless', '--grep', '@brokenInServerless', '--invert'];
+      break;
+
+    case 'essEnv':
+      grepArgs = ['--grep', '@ess'];
+      break;
+
+    case 'qaEnv':
+      grepArgs = ['--grep', '@serverless', '--grep', '@brokenInServerless|@skipInQA', '--invert'];
+      break;
+
+    default:
+      console.error(`Unsupported environment: ${environment}`);
+      process.exit(1);
+  }
+}
+
+const child = spawn('node', [command, '--config', configPath, ...grepArgs, ...args], {
+  stdio: 'inherit',
+});
+
+child.on('close', (code) => {
+  console.log(`child process exited with code ${code}`);
+});

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_new_terms.ts

@@ -9,10 +9,10 @@ import expect from '@kbn/expect';
 
 import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants';
 import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks';
-import { FtrProviderContext } from '../../common/ftr_provider_context';
-import { deleteAllRules } from '../../utils';
 
-// eslint-disable-next-line import/no-default-export
+import { deleteAllRules } from '../../utils';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const log = getService('log');
@@ -20,11 +20,13 @@ export default ({ getService }: FtrProviderContext) => {
   /**
    * Specific api integration tests for new terms rule type
    */
-  describe('create_new_terms', () => {
-    afterEach(async () => {
+  describe('@serverless @ess create_new_terms', () => {
+    beforeEach(async () => {
+      await deleteAllRules(supertest, log);
+    });
+    after(async () => {
       await deleteAllRules(supertest, log);
     });
-
     it('should not be able to create a new terms rule with too small history window', async () => {
       const rule = {
         ...getCreateNewTermsRulesSchemaMock('rule-1'),

x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/index.ts

@@ -9,5 +9,6 @@ import { FtrProviderContext } from '../../../../ftr_provider_context';
 export default function ({ loadTestFile }: FtrProviderContext) {
   describe('Rule creation API', function () {
     loadTestFile(require.resolve('./create_rules'));
+    loadTestFile(require.resolve('./create_new_terms'));
   });
 }