Mark slack rate-limiting errors as user errors (#192200)

Resolves: #191082

This PR marks rate-limiting responses of the slack webhook connector as
user error.

### To verify:

Put the below codes[ in the try-catch
block](dfd85051cf/x-pack/plugins/stack_connectors/server/connector_types/slack/index.ts (L168))
of the webhook connector to force it to fail.
```
const err = new Error('test');
err.original = { response: { status: 429, statusText: 'failure', headers: {} } };
throw err;
```
run Kibana and create a rule with slack webhook connector.

After the first run, there must be an error on your terminal and a user
error record for the .slack-webhook connector in the metrics JSON on
`/api/task_manager/metrics`

x-pack/plugins/stack_connectors/server/connector_types/slack/index.test.ts

@@ -4,7 +4,7 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+import { IncomingWebhook } from '@slack/webhook';
 import { Logger } from '@kbn/core/server';
 import {
   Services,
@@ -22,14 +22,9 @@ import { actionsConfigMock } from '@kbn/actions-plugin/server/actions_config.moc
 import { actionsMock } from '@kbn/actions-plugin/server/mocks';
 import { ActionsConfigurationUtilities } from '@kbn/actions-plugin/server/actions_config';
 import { loggerMock } from '@kbn/logging-mocks';
+import { TaskErrorSource } from '@kbn/task-manager-plugin/common';
 
-jest.mock('@slack/webhook', () => {
-  return {
-    IncomingWebhook: jest.fn().mockImplementation(() => {
-      return { send: (message: string) => {} };
-    }),
-  };
-});
+const sendSpy = jest.spyOn(IncomingWebhook.prototype, 'send');
 
 const services: Services = actionsMock.createServices();
 const mockedLogger: jest.Mocked<Logger> = loggerMock.create();
@@ -364,4 +359,42 @@ describe('execute()', () => {
     );
     expect(params.message).toBe('`*bold*`');
   });
+
+  test('returns a user error for rate-limiting responses', async () => {
+    const configUtils = actionsConfigMock.create();
+
+    configUtils.getProxySettings.mockReturnValue({
+      proxyUrl: 'https://someproxyhost',
+      proxySSLSettings: {
+        verificationMode: 'none',
+      },
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: undefined,
+    });
+
+    sendSpy.mockRejectedValueOnce({
+      original: { response: { status: 429, statusText: 'failure', headers: {} } },
+    });
+
+    connectorType = getConnectorType({});
+
+    expect(
+      await connectorType.executor({
+        actionId: 'some-id',
+        services,
+        config: {},
+        secrets: { webhookUrl: 'http://example.com' },
+        params: { message: '429' },
+        configurationUtilities: configUtils,
+        logger: mockedLogger,
+        connectorUsageCollector,
+      })
+    ).toEqual({
+      actionId: 'some-id',
+      errorSource: TaskErrorSource.USER,
+      message: 'error posting a slack message, retry later',
+      retry: true,
+      status: 'error',
+    });
+  });
 });

x-pack/plugins/stack_connectors/server/connector_types/slack/index.ts

@@ -28,6 +28,7 @@ import {
 } from '@kbn/actions-plugin/common/types';
 import { renderMustacheString } from '@kbn/actions-plugin/server/lib/mustache_renderer';
 import { getCustomAgents } from '@kbn/actions-plugin/server/lib/get_custom_agents';
+import { TaskErrorSource } from '@kbn/task-manager-plugin/common';
 import { getRetryAfterIntervalFromHeaders } from '../lib/http_response_retry_header';
 
 export type SlackConnectorType = ConnectorType<
@@ -175,7 +176,7 @@ async function slackExecutor(
 
     // special handling for 5xx
     if (status >= 500) {
-      return retryResult(actionId, err.message);
+      return retryResult(actionId, err.message, TaskErrorSource.FRAMEWORK);
     }
 
     // special handling for rate limiting
@@ -183,7 +184,7 @@ async function slackExecutor(
       return pipe(
         getRetryAfterIntervalFromHeaders(headers),
         map((retry) => retryResultSeconds(actionId, err.message, retry)),
-        getOrElse(() => retryResult(actionId, err.message))
+        getOrElse(() => retryResult(actionId, err.message, TaskErrorSource.USER))
       );
     }
 
@@ -245,7 +246,11 @@ function serviceErrorResult(
   };
 }
 
-function retryResult(actionId: string, message: string): ConnectorTypeExecutorResult<void> {
+function retryResult(
+  actionId: string,
+  serviceMessage: string,
+  errorSource: TaskErrorSource
+): ConnectorTypeExecutorResult<void> {
   const errMessage = i18n.translate(
     'xpack.stackConnectors.slack.errorPostingRetryLaterErrorMessage',
     {
@@ -257,6 +262,8 @@ function retryResult(actionId: string, message: string): ConnectorTypeExecutorRe
     message: errMessage,
     retry: true,
     actionId,
+    errorSource,
+    serviceMessage,
   };
 }
 
@@ -283,5 +290,6 @@ function retryResultSeconds(
     retry,
     actionId,
     serviceMessage: message,
+    errorSource: TaskErrorSource.USER,
   };
 }