[Response Ops][Alerting] Classify index pattern not found errors from ES query rule as user error (#189494)

Resolves https://github.com/elastic/response-ops-team/issues/220

## Summary

Classifies errors caused by users deleting data view used by the ES
query KQL rule as user errors.

## To Verify
1. Create a KQL ES query rule that uses a data view and let it run
successfully.
2. Delete the data view
3. See the rule execution fail and check the metrics from
https://localhost:5601/api/task_manager/metrics?reset=false classify the
error as a user error (in `metrics.task_run.value.overall.user_errors`)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.test.ts

@@ -6,6 +6,7 @@
  */
 
 import { OnlySearchSourceRuleParams } from '../types';
+import { searchSourceCommonMock } from '@kbn/data-plugin/common/search/search_source/mocks';
 import { createSearchSourceMock } from '@kbn/data-plugin/common/search/search_source/mocks';
 import { loggerMock } from '@kbn/logging-mocks';
 import {
@@ -13,6 +14,7 @@ import {
   generateLink,
   updateFilterReferences,
   getSmallerDataViewSpec,
+  fetchSearchSourceQuery,
 } from './fetch_search_source_query';
 import {
   createStubDataView,
@@ -24,6 +26,11 @@ import { Comparator } from '../../../../common/comparator_types';
 import { dataViewPluginMocks } from '@kbn/data-views-plugin/public/mocks';
 import { DiscoverAppLocatorParams } from '@kbn/discover-plugin/common';
 import { LocatorPublic } from '@kbn/share-plugin/common';
+import { SavedObjectsErrorHelpers } from '@kbn/core-saved-objects-server';
+import {
+  getErrorSource,
+  TaskErrorSource,
+} from '@kbn/task-manager-plugin/server/task_running/errors';
 
 const createDataView = () => {
   const id = 'test-id';
@@ -423,6 +430,54 @@ describe('fetchSearchSourceQuery', () => {
       `);
       expect(logger.warn).toHaveBeenCalledWith('Top hits size is capped at 100');
     });
+
+    it('should throw user error if data view is not found', async () => {
+      searchSourceCommonMock.createLazy.mockImplementationOnce(() => {
+        throw SavedObjectsErrorHelpers.createGenericNotFoundError('index-pattern', 'abc');
+      });
+
+      try {
+        await fetchSearchSourceQuery({
+          ruleId: 'abc',
+          params: defaultParams,
+          // @ts-expect-error
+          services: {
+            logger,
+            searchSourceClient: searchSourceCommonMock,
+          },
+          spacePrefix: '',
+          dateStart: new Date().toISOString(),
+          dateEnd: new Date().toISOString(),
+        });
+      } catch (err) {
+        expect(getErrorSource(err)).toBe(TaskErrorSource.USER);
+        expect(err.message).toBe('Saved object [index-pattern/abc] not found');
+      }
+    });
+
+    it('should re-throw error for generic errors', async () => {
+      searchSourceCommonMock.createLazy.mockImplementationOnce(() => {
+        throw new Error('fail');
+      });
+
+      try {
+        await fetchSearchSourceQuery({
+          ruleId: 'abc',
+          params: defaultParams,
+          // @ts-expect-error
+          services: {
+            logger,
+            searchSourceClient: searchSourceCommonMock,
+          },
+          spacePrefix: '',
+          dateStart: new Date().toISOString(),
+          dateEnd: new Date().toISOString(),
+        });
+      } catch (err) {
+        expect(getErrorSource(err)).not.toBeDefined();
+        expect(err.message).toBe('fail');
+      }
+    });
   });
 
   describe('generateLink', () => {

x-pack/plugins/stack_alerts/server/rule_types/es_query/lib/fetch_search_source_query.ts

@@ -22,8 +22,9 @@ import {
 import { isGroupAggregation } from '@kbn/triggers-actions-ui-plugin/common';
 import { SharePluginStart } from '@kbn/share-plugin/server';
 import { DiscoverAppLocatorParams } from '@kbn/discover-plugin/common';
-import { Logger } from '@kbn/core/server';
+import { Logger, SavedObjectsErrorHelpers } from '@kbn/core/server';
 import { LocatorPublic } from '@kbn/share-plugin/common';
+import { createTaskRunError, TaskErrorSource } from '@kbn/task-manager-plugin/server';
 import { OnlySearchSourceRuleParams } from '../types';
 import { getComparatorScript } from '../../../../common';
 
@@ -56,7 +57,16 @@ export async function fetchSearchSourceQuery({
   const { logger, searchSourceClient } = services;
   const isGroupAgg = isGroupAggregation(params.termField);
   const isCountAgg = isCountAggregation(params.aggType);
-  const initialSearchSource = await searchSourceClient.createLazy(params.searchConfiguration);
+
+  let initialSearchSource;
+  try {
+    initialSearchSource = await searchSourceClient.createLazy(params.searchConfiguration);
+  } catch (err) {
+    if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
+      throw createTaskRunError(err, TaskErrorSource.USER);
+    }
+    throw err;
+  }
 
   const index = initialSearchSource.getField('index') as DataView;
   const { searchSource, filterToExcludeHitsFromPreviousRun } = await updateSearchSource(

x-pack/plugins/stack_alerts/tsconfig.json

@@ -53,7 +53,8 @@
     "@kbn/search-types",
     "@kbn/alerting-comparators",
     "@kbn/task-manager-plugin",
-    "@kbn/core-logging-server-mocks"
+    "@kbn/core-logging-server-mocks",
+    "@kbn/core-saved-objects-server"
   ],
   "exclude": [
     "target/**/*",