Fixed alerting_api_integration/security_and_spaces tests failing if actions proxy set on for parallel process running using commands 'scripts/functional_tests_server' and 'scripts/functional_test_runner' (#75232)

* Fixed alerting_api_integration/security_and_spaces tests failing if actions proxy set on for parallel process running using commands 'scripts/functional_tests_server' and 'scripts/functional_test_runner'

* -

* Fixed get port from range for Slack and webhook simulators, removed some test warnings

* Added check for listening proxy server

* changed logger to debug removed not useful error

* -

* changed proxy to dynamic target in a single place

* test retry

* -

* -

* -

* -

* test with no cleanup

* -

* -

* -

* -

* Added environment variable ALERTING_PROXY_PORT

* fixed type checks

* fixed clean up proxy server port

vars/kibanaPipeline.groovy

@@ -86,6 +86,7 @@ def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) {
   def esPort = "61${parallelId}2"
   def esTransportPort = "61${parallelId}3"
   def ingestManagementPackageRegistryPort = "61${parallelId}4"
+  def alertingProxyPort = "61${parallelId}5"
 
   withEnv([
     "CI_GROUP=${parallelId}",
@@ -98,6 +99,7 @@ def withFunctionalTestEnv(List additionalEnvs = [], Closure closure) {
     "TEST_ES_TRANSPORT_PORT=${esTransportPort}",
     "KBN_NP_PLUGINS_BUILT=true",
     "INGEST_MANAGEMENT_PACKAGE_REGISTRY_PORT=${ingestManagementPackageRegistryPort}",
+    "ALERTING_PROXY_PORT=${alertingProxyPort}"
   ] + additionalEnvs) {
     closure()
   }

x-pack/package.json

@@ -269,7 +269,7 @@
     "font-awesome": "4.7.0",
     "formsy-react": "^1.1.5",
     "fp-ts": "^2.3.1",
-    "get-port": "^4.2.0",
+    "get-port": "^5.0.0",
     "getos": "^3.1.0",
     "git-url-parse": "11.1.2",
     "github-markdown-css": "^2.10.0",

x-pack/plugins/actions/server/builtin_action_types/servicenow/service.ts

@@ -78,8 +78,6 @@ export const createExternalService = (
 
   const createIncident = async ({ incident }: ExternalServiceParams) => {
     try {
-      logger.warn(`incident error : ${JSON.stringify(proxySettings)}`);
-      logger.warn(`incident error : ${url}`);
       const res = await request({
         axios: axiosInstance,
         url: `${incidentUrl}`,

x-pack/plugins/actions/server/builtin_action_types/slack.test.ts

@@ -209,7 +209,7 @@ describe('execute()', () => {
         rejectUnauthorizedCertificates: false,
       },
     });
-    expect(mockedLogger.info).toHaveBeenCalledWith(
+    expect(mockedLogger.debug).toHaveBeenCalledWith(
       'IncomingWebhook was called with proxyUrl https://someproxyhost'
     );
   });

x-pack/plugins/actions/server/builtin_action_types/slack.ts

@@ -119,7 +119,7 @@ async function slackExecutor(
   let proxyAgent: HttpsProxyAgent | HttpProxyAgent | undefined;
   if (execOptions.proxySettings) {
     proxyAgent = getProxyAgent(execOptions.proxySettings, logger);
-    logger.info(`IncomingWebhook was called with proxyUrl ${execOptions.proxySettings.proxyUrl}`);
+    logger.debug(`IncomingWebhook was called with proxyUrl ${execOptions.proxySettings.proxyUrl}`);
   }
 
   try {
@@ -130,8 +130,6 @@ async function slackExecutor(
     });
     result = await webhook.send(message);
   } catch (err) {
-    logger.error(`error on ${actionId} slack event: ${err.message}`);
-
     if (err.original == null || err.original.response == null) {
       return serviceErrorResult(actionId, err.message);
     }

x-pack/test/alerting_api_integration/common/config.ts

@@ -58,8 +58,13 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
       fs.statSync(path.resolve(__dirname, 'fixtures', 'plugins', file)).isDirectory()
     );
 
+    const proxyPort =
+      process.env.ALERTING_PROXY_PORT ?? (await getPort({ port: getPort.makeRange(6200, 6300) }));
     const actionsProxyUrl = options.enableActionsProxy
-      ? [`--xpack.actions.proxyUrl=http://localhost:${await getPort()}`]
+      ? [
+          `--xpack.actions.proxyUrl=http://localhost:${proxyPort}`,
+          '--xpack.actions.rejectUnauthorizedCertificates=false',
+        ]
       : [];
 
     return {
@@ -89,7 +94,6 @@ export function createTestConfig(name: string, options: CreateTestConfigOptions)
           '--xpack.encryptedSavedObjects.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"',
           `--xpack.actions.enabledActionTypes=${JSON.stringify(enabledActionTypes)}`,
           ...actionsProxyUrl,
-          '--xpack.actions.rejectUnauthorizedCertificates=false',
 
           '--xpack.eventLog.logEntries=true',
           `--xpack.actions.preconfigured=${JSON.stringify({

x-pack/test/alerting_api_integration/common/fixtures/plugins/actions_simulators/server/slack_simulation.ts

@@ -64,6 +64,9 @@ export async function initPlugin() {
         response.statusCode = 400;
         response.end('unknown request to slack simulator');
       });
+    } else {
+      response.writeHead(400, { 'Content-Type': 'text/plain' });
+      response.end('Not supported http method to request slack simulator');
     }
   });
 }

x-pack/test/alerting_api_integration/common/lib/get_proxy_server.ts

@@ -4,27 +4,42 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import http from 'http';
 import httpProxy from 'http-proxy';
+import { ToolingLog } from '@kbn/dev-utils';
 
-export const getHttpProxyServer = (
-  targetUrl: string,
-  onProxyResHandler: (proxyRes?: unknown, req?: unknown, res?: unknown) => void
-): httpProxy => {
-  const proxyServer = httpProxy.createProxyServer({
-    target: targetUrl,
-    secure: false,
-    selfHandleResponse: false,
-  });
-  proxyServer.on('proxyRes', (proxyRes: unknown, req: unknown, res: unknown) => {
-    onProxyResHandler(proxyRes, req, res);
+export const getHttpProxyServer = async (
+  defaultKibanaTargetUrl: string,
+  kbnTestServerConfig: any,
+  log: ToolingLog
+): Promise<http.Server> => {
+  const proxy = httpProxy.createProxyServer({ secure: false, selfHandleResponse: false });
+
+  const proxyPort = getProxyPort(kbnTestServerConfig);
+  const proxyServer = http.createServer((req: http.IncomingMessage, res: http.ServerResponse) => {
+    const targetUrl = new URL(req.url ?? defaultKibanaTargetUrl);
+
+    if (targetUrl.hostname !== 'some.non.existent.com') {
+      proxy.web(req, res, {
+        target: `${targetUrl.protocol}//${targetUrl.hostname}:${targetUrl.port}`,
+      });
+    } else {
+      res.writeHead(500, { 'Content-Type': 'text/plain' });
+      res.write('error on call some.non.existent.com');
+      res.end();
+    }
   });
+
+  proxyServer.listen(proxyPort);
+
   return proxyServer;
 };
 
-export const getProxyUrl = (kbnTestServerConfig: any) => {
+export const getProxyPort = (kbnTestServerConfig: any): number => {
   const proxyUrl = kbnTestServerConfig
     .find((val: string) => val.startsWith('--xpack.actions.proxyUrl='))
     .replace('--xpack.actions.proxyUrl=', '');
 
-  return new URL(proxyUrl);
+  const urlObject = new URL(proxyUrl);
+  return Number(urlObject.port);
 };

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/jira.ts

@@ -6,7 +6,6 @@
 
 import expect from '@kbn/expect';
 
-import { getHttpProxyServer, getProxyUrl } from '../../../../common/lib/get_proxy_server';
 import { FtrProviderContext } from '../../../../common/ftr_provider_context';
 
 import {
@@ -36,7 +35,6 @@ const mapping = [
 export default function jiraTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const kibanaServer = getService('kibanaServer');
-  const config = getService('config');
 
   const mockJira = {
     config: {
@@ -75,20 +73,12 @@ export default function jiraTest({ getService }: FtrProviderContext) {
   };
 
   let jiraSimulatorURL: string = '<could not determine kibana url>';
-  let proxyServer: any;
-  let proxyHaveBeenCalled = false;
 
-  // FLAKY: https://github.com/elastic/kibana/issues/75722
-  describe.skip('Jira', () => {
+  describe('Jira', () => {
     before(() => {
       jiraSimulatorURL = kibanaServer.resolveUrl(
         getExternalServiceSimulatorPath(ExternalServiceSimulator.JIRA)
       );
-      proxyServer = getHttpProxyServer(kibanaServer.resolveUrl('/'), () => {
-        proxyHaveBeenCalled = true;
-      });
-      const proxyUrl = getProxyUrl(config.get('kbnTestServer.serverArgs'));
-      proxyServer.listen(Number(proxyUrl.port));
     });
 
     describe('Jira - Action Creation', () => {
@@ -539,8 +529,6 @@ export default function jiraTest({ getService }: FtrProviderContext) {
             })
             .expect(200);
 
-          expect(proxyHaveBeenCalled).to.equal(true);
-
           expect(body).to.eql({
             status: 'ok',
             actionId: simulatedActionId,
@@ -554,9 +542,5 @@ export default function jiraTest({ getService }: FtrProviderContext) {
         });
       });
     });
-
-    after(() => {
-      proxyServer.close();
-    });
   });
 }

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/pagerduty.ts

@@ -6,7 +6,6 @@
 
 import expect from '@kbn/expect';
 
-import { getHttpProxyServer, getProxyUrl } from '../../../../common/lib/get_proxy_server';
 import { FtrProviderContext } from '../../../../common/ftr_provider_context';
 
 import {
@@ -18,26 +17,16 @@ import {
 export default function pagerdutyTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const kibanaServer = getService('kibanaServer');
-  const config = getService('config');
 
-  // FLAKY: https://github.com/elastic/kibana/issues/75386
-  describe.skip('pagerduty action', () => {
+  describe('pagerduty action', () => {
     let simulatedActionId = '';
     let pagerdutySimulatorURL: string = '<could not determine kibana url>';
-    let proxyServer: any;
-    let proxyHaveBeenCalled = false;
 
     // need to wait for kibanaServer to settle ...
     before(() => {
       pagerdutySimulatorURL = kibanaServer.resolveUrl(
         getExternalServiceSimulatorPath(ExternalServiceSimulator.PAGERDUTY)
       );
-
-      proxyServer = getHttpProxyServer(kibanaServer.resolveUrl('/'), () => {
-        proxyHaveBeenCalled = true;
-      });
-      const proxyUrl = getProxyUrl(config.get('kbnTestServer.serverArgs'));
-      proxyServer.listen(Number(proxyUrl.port));
     });
 
     it('should return successfully when passed valid create parameters', async () => {
@@ -155,7 +144,6 @@ export default function pagerdutyTest({ getService }: FtrProviderContext) {
           },
         })
         .expect(200);
-      expect(proxyHaveBeenCalled).to.equal(true);
 
       expect(result).to.eql({
         status: 'ok',
@@ -215,9 +203,5 @@ export default function pagerdutyTest({ getService }: FtrProviderContext) {
       expect(result.message).to.match(/error posting pagerduty event: http status 502/);
       expect(result.retry).to.equal(true);
     });
-
-    after(() => {
-      proxyServer.close();
-    });
   });
 }

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/resilient.ts

@@ -6,7 +6,6 @@
 
 import expect from '@kbn/expect';
 
-import { getHttpProxyServer, getProxyUrl } from '../../../../common/lib/get_proxy_server';
 import { FtrProviderContext } from '../../../../common/ftr_provider_context';
 
 import {
@@ -36,7 +35,6 @@ const mapping = [
 export default function resilientTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const kibanaServer = getService('kibanaServer');
-  const config = getService('config');
 
   const mockResilient = {
     config: {
@@ -75,19 +73,12 @@ export default function resilientTest({ getService }: FtrProviderContext) {
   };
 
   let resilientSimulatorURL: string = '<could not determine kibana url>';
-  let proxyServer: any;
-  let proxyHaveBeenCalled = false;
 
   describe('IBM Resilient', () => {
     before(() => {
       resilientSimulatorURL = kibanaServer.resolveUrl(
         getExternalServiceSimulatorPath(ExternalServiceSimulator.RESILIENT)
       );
-      proxyServer = getHttpProxyServer(kibanaServer.resolveUrl('/'), () => {
-        proxyHaveBeenCalled = true;
-      });
-      const proxyUrl = getProxyUrl(config.get('kbnTestServer.serverArgs'));
-      proxyServer.listen(Number(proxyUrl.port));
     });
 
     describe('IBM Resilient - Action Creation', () => {
@@ -538,8 +529,6 @@ export default function resilientTest({ getService }: FtrProviderContext) {
             })
             .expect(200);
 
-          expect(proxyHaveBeenCalled).to.equal(true);
-
           expect(body).to.eql({
             status: 'ok',
             actionId: simulatedActionId,
@@ -553,9 +542,5 @@ export default function resilientTest({ getService }: FtrProviderContext) {
         });
       });
     });
-
-    after(() => {
-      proxyServer.close();
-    });
   });
 }

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/servicenow.ts

@@ -6,7 +6,6 @@
 
 import expect from '@kbn/expect';
 
-import { getHttpProxyServer, getProxyUrl } from '../../../../common/lib/get_proxy_server';
 import { FtrProviderContext } from '../../../../common/ftr_provider_context';
 
 import {
@@ -36,7 +35,6 @@ const mapping = [
 export default function servicenowTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const kibanaServer = getService('kibanaServer');
-  const config = getService('config');
 
   const mockServiceNow = {
     config: {
@@ -74,21 +72,12 @@ export default function servicenowTest({ getService }: FtrProviderContext) {
   };
 
   let servicenowSimulatorURL: string = '<could not determine kibana url>';
-  let proxyServer: any;
-  let proxyHaveBeenCalled = false;
 
-  // FLAKY: https://github.com/elastic/kibana/issues/75522
-  describe.skip('ServiceNow', () => {
+  describe('ServiceNow', () => {
     before(() => {
       servicenowSimulatorURL = kibanaServer.resolveUrl(
         getExternalServiceSimulatorPath(ExternalServiceSimulator.SERVICENOW)
       );
-
-      proxyServer = getHttpProxyServer(kibanaServer.resolveUrl('/'), () => {
-        proxyHaveBeenCalled = true;
-      });
-      const proxyUrl = getProxyUrl(config.get('kbnTestServer.serverArgs'));
-      proxyServer.listen(Number(proxyUrl.port));
     });
 
     describe('ServiceNow - Action Creation', () => {
@@ -459,7 +448,6 @@ export default function servicenowTest({ getService }: FtrProviderContext) {
               },
             })
             .expect(200);
-          expect(proxyHaveBeenCalled).to.equal(true);
 
           expect(result).to.eql({
             status: 'ok',
@@ -474,9 +462,5 @@ export default function servicenowTest({ getService }: FtrProviderContext) {
         });
       });
     });
-
-    after(() => {
-      proxyServer.close();
-    });
   });
 }

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/slack.ts

@@ -7,7 +7,6 @@
 import expect from '@kbn/expect';
 import http from 'http';
 import getPort from 'get-port';
-import { getHttpProxyServer, getProxyUrl } from '../../../../common/lib/get_proxy_server';
 import { FtrProviderContext } from '../../../../common/ftr_provider_context';
 
 import { getSlackServer } from '../../../../common/fixtures/plugins/actions_simulators/server/plugin';
@@ -15,27 +14,20 @@ import { getSlackServer } from '../../../../common/fixtures/plugins/actions_simu
 // eslint-disable-next-line import/no-default-export
 export default function slackTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
-  const config = getService('config');
 
   describe('slack action', () => {
     let simulatedActionId = '';
-
     let slackSimulatorURL: string = '';
     let slackServer: http.Server;
-    let proxyServer: any;
-    let proxyHaveBeenCalled = false;
+
     // need to wait for kibanaServer to settle ...
     before(async () => {
       slackServer = await getSlackServer();
-      const availablePort = await getPort({ port: 9000 });
-      slackServer.listen(availablePort);
+      const availablePort = await getPort({ port: getPort.makeRange(9000, 9100) });
+      if (!slackServer.listening) {
+        slackServer.listen(availablePort);
+      }
       slackSimulatorURL = `http://localhost:${availablePort}`;
-
-      proxyServer = getHttpProxyServer(slackSimulatorURL, () => {
-        proxyHaveBeenCalled = true;
-      });
-      const proxyUrl = getProxyUrl(config.get('kbnTestServer.serverArgs'));
-      proxyServer.listen(Number(proxyUrl.port));
     });
 
     it('should return 200 when creating a slack action successfully', async () => {
@@ -165,7 +157,6 @@ export default function slackTest({ getService }: FtrProviderContext) {
         })
         .expect(200);
       expect(result.status).to.eql('ok');
-      expect(proxyHaveBeenCalled).to.equal(true);
     });
 
     it('should handle an empty message error', async () => {
@@ -233,7 +224,6 @@ export default function slackTest({ getService }: FtrProviderContext) {
 
     after(() => {
       slackServer.close();
-      proxyServer.close();
     });
   });
 }

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/builtin_action_types/webhook.ts

@@ -5,10 +5,9 @@
  */
 
 import http from 'http';
-import getPort from 'get-port';
 import expect from '@kbn/expect';
 import { URL, format as formatUrl } from 'url';
-import { getHttpProxyServer, getProxyUrl } from '../../../../common/lib/get_proxy_server';
+import getPort from 'get-port';
 import { FtrProviderContext } from '../../../../common/ftr_provider_context';
 import {
   getExternalServiceSimulatorPath,
@@ -32,7 +31,6 @@ function parsePort(url: Record<string, string>): Record<string, string | null |
 export default function webhookTest({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
   const kibanaServer = getService('kibanaServer');
-  const configService = getService('config');
 
   async function createWebhookAction(
     webhookSimulatorURL: string,
@@ -71,21 +69,14 @@ export default function webhookTest({ getService }: FtrProviderContext) {
     let webhookSimulatorURL: string = '';
     let webhookServer: http.Server;
     let kibanaURL: string = '<could not determine kibana url>';
-    let proxyServer: any;
-    let proxyHaveBeenCalled = false;
+
     // need to wait for kibanaServer to settle ...
     before(async () => {
       webhookServer = await getWebhookServer();
-      const availablePort = await getPort({ port: 9000 });
+      const availablePort = await getPort({ port: getPort.makeRange(9000, 9100) });
       webhookServer.listen(availablePort);
       webhookSimulatorURL = `http://localhost:${availablePort}`;
 
-      proxyServer = getHttpProxyServer(webhookSimulatorURL, () => {
-        proxyHaveBeenCalled = true;
-      });
-      const proxyUrl = getProxyUrl(configService.get('kbnTestServer.serverArgs'));
-      proxyServer.listen(Number(proxyUrl.port));
-
       kibanaURL = kibanaServer.resolveUrl(
         getExternalServiceSimulatorPath(ExternalServiceSimulator.WEBHOOK)
       );
@@ -150,7 +141,6 @@ export default function webhookTest({ getService }: FtrProviderContext) {
         .expect(200);
 
       expect(result.status).to.eql('ok');
-      expect(proxyHaveBeenCalled).to.equal(true);
     });
 
     it('should support the POST method against webhook target', async () => {
@@ -251,7 +241,6 @@ export default function webhookTest({ getService }: FtrProviderContext) {
 
     after(() => {
       webhookServer.close();
-      proxyServer.close();
     });
   });
 }

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/index.ts

@@ -4,11 +4,24 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import http from 'http';
+import { getHttpProxyServer } from '../../../common/lib/get_proxy_server';
 import { FtrProviderContext } from '../../../common/ftr_provider_context';
 
 // eslint-disable-next-line import/no-default-export
-export default function actionsTests({ loadTestFile }: FtrProviderContext) {
+export default function actionsTests({ loadTestFile, getService }: FtrProviderContext) {
+  const configService = getService('config');
+  const kibanaServer = getService('kibanaServer');
+  const log = getService('log');
   describe('Actions', () => {
+    let proxyServer: http.Server | undefined;
+    before(async () => {
+      proxyServer = await getHttpProxyServer(
+        kibanaServer.resolveUrl('/'),
+        configService.get('kbnTestServer.serverArgs'),
+        log
+      );
+    });
     loadTestFile(require.resolve('./builtin_action_types/email'));
     loadTestFile(require.resolve('./builtin_action_types/es_index'));
     loadTestFile(require.resolve('./builtin_action_types/es_index_preconfigured'));
@@ -26,5 +39,11 @@ export default function actionsTests({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./get'));
     loadTestFile(require.resolve('./list_action_types'));
     loadTestFile(require.resolve('./update'));
+
+    after(() => {
+      if (proxyServer) {
+        proxyServer.close();
+      }
+    });
   });
 }

yarn.lock

@@ -13694,10 +13694,10 @@ get-own-enumerable-property-symbols@^3.0.0:
   resolved "https://registry.yarnpkg.com/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz#b877b49a5c16aefac3655f2ed2ea5b684df8d203"
   integrity sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg==
 
-get-port@^4.2.0:
-  version "4.2.0"
-  resolved "https://registry.yarnpkg.com/get-port/-/get-port-4.2.0.tgz#e37368b1e863b7629c43c5a323625f95cf24b119"
-  integrity sha512-/b3jarXkH8KJoOMQc3uVGHASwGLPq3gSFJ7tgJm2diza+bydJPTGOibin2steecKeOylE8oY2JERlVWkAJO6yw==
+get-port@^5.0.0:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193"
+  integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==
 
 get-stdin@^4.0.1:
   version "4.0.1"