[APM] Create dev script to be able to run Deployment agnostic tests (#201351)

## Summary

Closes #201346

This PR adds a `dat` dev script, which lets you easily run APM
deployment-agnostic tests, as we already do with `api` tests.

## Examples

Run only server with stateful config
```sh
node x-pack/plugins/observability_solution/apm/scripts/test/dat --stateful --server
```
Run only runner with stateful config
```sh
node x-pack/plugins/observability_solution/apm/scripts/test/dat --stateful --runner
```
Run everything with serverless config
```sh
node x-pack/plugins/observability_solution/apm/scripts/test/dat --serverless
```

More information
[here](https://github.com/elastic/kibana/pull/201351/files#diff-5ca943bbe3962fc31cce510407cc1563a959b0fdb9cb52619f57e1f182a9751a)

---------

Co-authored-by: Cauê Marcondes <55978943+cauemarcondes@users.noreply.github.com>

x-pack/plugins/observability_solution/apm/dev_docs/testing.md

@@ -3,7 +3,9 @@
 We've got three ways of testing our code:
 
 - Unit testing with Jest
-- API testing
+- Integration Tests
+  - Deployment specific (stateful or serverless) API testing
+  - Deployment-agnostic (both stateful and serverless) testing
 - End-to-end testing (with Cypress)
 
 API tests are usually preferred. They're stable and reasonably quick, and give a good approximation of real-world usage.
@@ -73,7 +75,59 @@ node x-pack/plugins/observability_solution/apm/scripts/test/api --runner --basic
 
 #### API Test tips
 
-- For data generation in API tests have a look at the [kbn-apm-synthtrace](../../../../packages/kbn-apm-synthtrace/README.md) package
+- For data generation in API tests have a look at the [kbn-apm-synthtrace](../../../../../packages/kbn-apm-synthtrace/README.md) package
+- For debugging access Elasticsearch on http://localhost:9220 and Kibana on http://localhost:5620 (`elastic` / `changeme`)
+
+---
+
+## Deployment-agnostic Tests (dat)
+
+| Option       | Description                                     |
+| ------------ | ----------------------------------------------- |
+| --serverless | Loads serverless configuration                  |
+| --stateful   | Loads stateful configuration                    |
+| --server     | Only start ES and Kibana                        |
+| --runner     | Only run tests                                  |
+| --grep       | Specify the specs to run                        |
+| --grep-files | Specify the files to run                        |
+| --inspect    | Add --inspect-brk flag to the ftr for debugging |
+| --times      | Repeat the test n number of times               |
+
+Deployment-agnostic tests are located in [`x-pack/test/deployment_agnostic/apis/observability/apm/index.ts`](../../../../test/api_integration/deployment_agnostic/apis/observability/apm/index.ts).
+
+#### Start server and run test (single process)
+
+```
+node x-pack/plugins/observability_solution/apm/scripts/test/dat [--serverless/--stateful] [--help]
+```
+
+The above command will start an ES instance on http://localhost:9220, a Kibana instance on http://localhost:5620 and run the api tests.
+Once the tests finish, the instances will be terminated.
+
+#### Start server and run test (separate processes)
+
+```sh
+
+# start server
+node x-pack/plugins/observability_solution/apm/scripts/test/dat --server --stateful
+
+# run tests
+node x-pack/plugins/observability_solution/apm/scripts/test/dat --runner --stateful --grep-files=error_group_list
+```
+
+### Update snapshots (from Kibana root)
+
+To update snapshots append `--updateSnapshots` to the `--runner` command:
+
+```
+node x-pack/plugins/observability_solution/apm/scripts/test/dat --runner --stateful --updateSnapshots
+```
+
+(The test server needs to be running)
+
+#### API Test tips
+
+- For data generation in Deployment-agnostic tests have a look at the [kbn-apm-synthtrace](../../../../../packages/kbn-apm-synthtrace/README.md) package
 - For debugging access Elasticsearch on http://localhost:9220 and Kibana on http://localhost:5620 (`elastic` / `changeme`)
 
 ---

x-pack/plugins/observability_solution/apm/scripts/test/dat.js

@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/* eslint-disable no-console */
+const { times } = require('lodash');
+const yargs = require('yargs');
+const path = require('path');
+const childProcess = require('child_process');
+const { REPO_ROOT } = require('@kbn/repo-info');
+
+const { argv } = yargs(process.argv.slice(2))
+  .option('serverless', {
+    default: false,
+    type: 'boolean',
+    description: 'Loads serverless configuration',
+  })
+  .option('stateful', {
+    default: false,
+    type: 'boolean',
+    description: 'Loads stateful configuration',
+  })
+  .option('server', {
+    default: false,
+    type: 'boolean',
+    description: 'Only start ES and Kibana',
+  })
+  .option('runner', {
+    default: false,
+    type: 'boolean',
+    description: 'Only run tests',
+  })
+  .option('grep', {
+    alias: 'spec',
+    type: 'string',
+    description: 'Specify the specs to run',
+  })
+  .option('grep-files', {
+    alias: 'files',
+    type: 'array',
+    string: true,
+    description: 'Specify the files to run',
+  })
+  .option('inspect', {
+    default: false,
+    type: 'boolean',
+    description: 'Add --inspect-brk flag to the ftr for debugging',
+  })
+  .option('times', {
+    type: 'number',
+    description: 'Repeat the test n number of times',
+  })
+  .option('updateSnapshots', {
+    default: false,
+    type: 'boolean',
+    description: 'Update snapshots',
+  })
+  .option('bail', {
+    default: false,
+    type: 'boolean',
+    description: 'Stop the test run at the first failure',
+  })
+  .check((argv) => {
+    const { inspect, runner } = argv;
+    if (inspect && !runner) {
+      throw new Error('--inspect can only be used with --runner');
+    } else {
+      return true;
+    }
+  })
+  .help();
+
+const { serverless, stateful, bail, server, runner, grep, grepFiles, inspect, updateSnapshots } =
+  argv;
+
+if ((serverless === false && stateful === false) || (serverless && stateful)) {
+  throw new Error('Please specify either --stateful or --serverless');
+}
+
+let ftrScript = 'functional_tests';
+if (server) {
+  ftrScript = 'functional_tests_server';
+} else if (runner) {
+  ftrScript = 'functional_test_runner';
+}
+
+const environment = serverless ? 'serverless' : 'stateful';
+
+const cmd = [
+  'node',
+  ...(inspect ? ['--inspect-brk'] : []),
+  `${REPO_ROOT}/scripts/${ftrScript}`,
+  ...(grep ? [`--grep "${grep}"`] : []),
+  ...(updateSnapshots ? [`--updateSnapshots`] : []),
+  ...(bail ? [`--bail`] : []),
+  `--config ${REPO_ROOT}/x-pack/test/api_integration/deployment_agnostic/configs/${environment}/oblt.apm.${environment}.config.ts`,
+].join(' ');
+
+console.log(`Running: "${cmd}"`);
+
+function runTests() {
+  childProcess.execSync(cmd, {
+    cwd: path.join(__dirname),
+    stdio: 'inherit',
+    env: { ...process.env, APM_TEST_GREP_FILES: JSON.stringify(grepFiles) },
+  });
+}
+
+if (argv.times) {
+  const runCounter = { succeeded: 0, failed: 0, remaining: argv.times };
+  let exitStatus = 0;
+  times(argv.times, () => {
+    try {
+      runTests();
+      runCounter.succeeded++;
+    } catch (e) {
+      if (bail) {
+        throw e;
+      }
+
+      exitStatus = 1;
+      runCounter.failed++;
+    }
+    runCounter.remaining--;
+    if (argv.times > 1) {
+      console.log(runCounter);
+    }
+  });
+  process.exit(exitStatus);
+} else {
+  runTests();
+}