[ResponseOps][Alerting] Do not return alerts from internally managed rule types (#223453)

## Summary

This PR introduces the concept of internally managed rule types. The
purpose of this PR is to hide alerts in the alerts table in the UI
produced by internally managed rule types. In following PRs, we will
enhance the framework to handle more cases when the product requirements
are clearer. If, in the future, the streams team wants to use the alerts
table to show stream alerts, we could introduce a new parameter in the
alerting API to allow alerts produced by internally managed rule types
to be returned.

Fixes: https://github.com/elastic/kibana/issues/221379

cc @kdelemme @dgieselaar 

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/platform/plugins/shared/alerting/README.md

@@ -14,24 +14,30 @@ Table of Contents
 	- [Terminology](#terminology)
 	- [Usage](#usage)
 	- [Alerting API Keys](#alerting-api-keys)
-	- [Plugin Status](#plugin-status)
 	- [Rule Types](#rule-types)
 		- [Methods](#methods)
-		- [Alerts as Data](#alerts-as-data)
 		- [Executor](#executor)
-		- [Action variables](#action-variables)
+		- [Alerts as Data](#alerts-as-data)
+		- [Action Variables](#action-variables)
+		- [useSavedObjectReferences Hooks](#usesavedobjectreferences-hooks)
 	- [Recovered Alerts](#recovered-alerts)
 	- [Licensing](#licensing)
 	- [Documentation](#documentation)
 	- [Tests](#tests)
 		- [Example](#example)
 	- [Role Based Access-Control](#role-based-access-control)
-	- [Alerting Navigation](#alert-navigation)
+		- [Subfeature privileges](#subfeature-privileges)
+		- [`read` privileges vs. `all` privileges](#read-privileges-vs-all-privileges)
+	- [Alert Navigation](#alert-navigation)
+		- [registerNavigation](#registernavigation)
+		- [registerDefaultNavigation](#registerdefaultnavigation)
+		- [Balancing both APIs side by side](#balancing-both-apis-side-by-side)
 	- [Internal HTTP APIs](#internal-http-apis)
 		- [`GET /internal/alerting/rule/{id}/state`: Get rule state](#get-internalalertingruleidstate-get-rule-state)
-		- [`GET /internal/alerting/rule/{id}/_alert_summary`: Get rule alert summary](#get-internalalertingruleidalertsummary-get-rule-alert-summary)
-		- [`POST /api/alerting/rule/{id}/_update_api_key`: Update rule API key](#post-internalalertingruleidupdateapikey-update-rule-api-key)
+		- [`GET /internal/alerting/rule/{id}/_alert_summary`: Get rule alert summary](#get-internalalertingruleid_alert_summary-get-rule-alert-summary)
+		- [`POST /api/alerting/rule/{id}/_update_api_key`: Update rule API key](#post-apialertingruleid_update_api_key-update-rule-api-key)
 	- [Alert Factory](#alert-factory)
+		- [When should I use `setContext`?](#when-should-i-use-setcontext)
 	- [Templating Actions](#templating-actions)
 		- [Examples](#examples)
 
@@ -102,6 +108,7 @@ The following table describes the properties of the `options` object.
 |alerts|(Optional) Specify options for writing alerts as data documents for this rule type. This feature is currently under development so this field is optional but we will eventually make this a requirement of all rule types. For full details, see the alerts as data section below.|IRuleTypeAlerts|
 |autoRecoverAlerts|(Optional) Whether the framework should determine if alerts have recovered between rule runs. If not specified, the default value of `true` is used. |boolean|
 |getViewInAppRelativeUrl|(Optional) When developing a rule type, you can choose to implement this hook for generating a link back to the Kibana application that can be used in alert actions. If not specified, a generic link back to the Rule Management app is generated.|Function|
+|internallyManaged|(Optional) Indicates that the rule type is managed internally by a Kibana plugin. Alerts of internally managed rule types are not returned by the APIs and thus not shown in the alerts table.|boolean|
 
 ### Executor
 

x-pack/platform/plugins/shared/alerting/server/rule_type_registry.ts

@@ -71,6 +71,7 @@ export interface RegistryRuleType
     | 'defaultScheduleInterval'
     | 'doesSetRecoveryContext'
     | 'alerts'
+    | 'internallyManaged'
   > {
   id: string;
   enabledInLicense: boolean;

x-pack/platform/plugins/shared/alerting/server/types.ts

@@ -350,6 +350,11 @@ export interface RuleType<
    */
   autoRecoverAlerts?: boolean;
   getViewInAppRelativeUrl?: GetViewInAppRelativeUrlFn<Params>;
+  /**
+   * Indicates that the rule type is managed internally by a Kibana plugin.
+   * Alerts of internally managed rule types are not returned by the APIs and thus not shown in the alerts table.
+   */
+  internallyManaged?: boolean;
 }
 export type UntypedRuleType = RuleType<
   RuleTypeParams,

x-pack/platform/plugins/shared/rule_registry/server/search_strategy/search_strategy.test.ts

@@ -819,4 +819,38 @@ describe('ruleRegistrySearchStrategyProvider()', () => {
       })
     );
   });
+
+  it('removes internally managed rule types', async () => {
+    const request: RuleRegistrySearchRequest = {
+      ruleTypeIds: ['.es-query', '.internally-managed', '.not-internally-managed'],
+      trackScores: true,
+    };
+
+    const options = {};
+    const deps = {
+      request: {},
+    };
+
+    getAuthorizedRuleTypesMock.mockResolvedValue([]);
+    getAlertIndicesAliasMock.mockReturnValue(['security-siem']);
+    alerting.listTypes.mockReturnValue(
+      // @ts-expect-error: rule type properties are not needed for the test
+      new Map([
+        ['.es-query', {}],
+        ['.internally-managed', { internallyManaged: true }],
+        ['.not-internally-managed', { internallyManaged: false }],
+      ])
+    );
+
+    const strategy = ruleRegistrySearchStrategyProvider(data, alerting, logger, security, spaces);
+
+    await lastValueFrom(
+      strategy.search(request, options, deps as unknown as SearchStrategyDependencies)
+    );
+
+    expect(authorizationMock.getAllAuthorizedRuleTypesFindOperation).toHaveBeenCalledWith({
+      authorizationEntity: 'alert',
+      ruleTypeIds: ['.es-query', '.not-internally-managed'],
+    });
+  });
 });

x-pack/platform/plugins/shared/rule_registry/server/search_strategy/search_strategy.ts

@@ -10,6 +10,7 @@ import { map, mergeMap, catchError, of } from 'rxjs';
 import type { estypes } from '@elastic/elasticsearch';
 import type { Logger } from '@kbn/core/server';
 import { from } from 'rxjs';
+import type { RegistryRuleType } from '@kbn/alerting-plugin/server/rule_type_registry';
 import { ENHANCED_ES_SEARCH_STRATEGY } from '@kbn/data-plugin/common';
 import type { ISearchStrategy, PluginStart } from '@kbn/data-plugin/server';
 import type { AlertingServerStart } from '@kbn/alerting-plugin/server';
@@ -61,8 +62,11 @@ export const ruleRegistrySearchStrategyProvider = (
 
       const registeredRuleTypes = alerting.listTypes();
 
+      const ruleTypesWithoutInternalRuleTypes =
+        getRuleTypesWithoutInternalRuleTypes(registeredRuleTypes);
+
       const [validRuleTypeIds, _] = partition(request.ruleTypeIds, (ruleTypeId) =>
-        registeredRuleTypes.has(ruleTypeId)
+        ruleTypesWithoutInternalRuleTypes.has(ruleTypeId)
       );
 
       if (isAnyRuleTypeESAuthorized && !isEachRuleTypeESAuthorized) {
@@ -235,3 +239,11 @@ export const ruleRegistrySearchStrategyProvider = (
     },
   };
 };
+
+const getRuleTypesWithoutInternalRuleTypes = (registeredRuleTypes: Map<string, RegistryRuleType>) =>
+  new Map(
+    Array.from(registeredRuleTypes).filter(
+      ([_id, ruleType]) =>
+        ruleType.internallyManaged == null || !Boolean(ruleType.internallyManaged)
+    )
+  );

x-pack/platform/plugins/shared/streams/server/lib/rules/esql/register.ts

@@ -55,5 +55,6 @@ export function esqlRuleType(): PersistenceAlertType<
       shouldWrite: false,
       isSpaceAware: false,
     },
+    internallyManaged: true,
   };
 }

x-pack/platform/test/alerting_api_integration/common/lib/alert_utils.ts

@@ -754,3 +754,15 @@ function getAlwaysFiringRuleWithSystemAction(reference: string) {
     ],
   };
 }
+
+export function getAlwaysFiringInternalRule() {
+  return {
+    enabled: true,
+    name: 'Internal Rule',
+    schedule: { interval: '1m' },
+    tags: [],
+    rule_type_id: 'test.internal-rule-type',
+    consumer: 'alertsFixture',
+    params: {},
+  };
+}

x-pack/platform/test/alerting_api_integration/common/plugins/alerts/server/rule_types.ts

@@ -1192,6 +1192,41 @@ function getSeverityRuleType() {
   return result;
 }
 
+const getInternalRuleType = () => {
+  const result: RuleType<{}, never, {}, {}, {}, 'default'> = {
+    id: 'test.internal-rule-type',
+    name: 'Test: Internal Rule Type',
+    actionGroups: [{ id: 'default', name: 'Default' }],
+    validate: {
+      params: schema.any(),
+    },
+    category: 'management',
+    producer: 'alertsFixture',
+    solution: 'stack',
+    defaultActionGroupId: 'default',
+    minimumLicenseRequired: 'basic',
+    isExportable: true,
+    internallyManaged: true,
+    async executor(ruleExecutorOptions) {
+      const { services } = ruleExecutorOptions;
+
+      services.alertsClient?.report({ id: '1', actionGroup: 'default' });
+      services.alertsClient?.report({ id: '2', actionGroup: 'default' });
+
+      return { state: {} };
+    },
+    alerts: {
+      context: 'observability.test.alerts',
+      mappings: {
+        fieldMap: {},
+      },
+      useLegacyAlerts: true,
+      shouldWrite: true,
+    },
+  };
+  return result;
+};
+
 async function sendSignal(
   logger: Logger,
   es: ElasticsearchClient,
@@ -1531,4 +1566,5 @@ export function defineRuleTypes(
   alerting.registerType(getPatternFiringAlertsAsDataRuleType());
   alerting.registerType(getWaitingRuleType(logger));
   alerting.registerType(getSeverityRuleType());
+  alerting.registerType(getInternalRuleType());
 }

x-pack/test/rule_registry/security_and_spaces/tests/basic/search_strategy.ts

@@ -6,8 +6,13 @@
  */
 import expect from '@kbn/expect';
 
-import { ALERT_START } from '@kbn/rule-data-utils';
+import { ALERT_RULE_TYPE_ID, ALERT_START } from '@kbn/rule-data-utils';
 import type { RuleRegistrySearchResponse } from '@kbn/rule-registry-plugin/common';
+import { ObjectRemover } from '@kbn/test-suites-xpack-platform/alerting_api_integration/common/lib';
+import { getAlwaysFiringInternalRule } from '@kbn/test-suites-xpack-platform/alerting_api_integration/common/lib/alert_utils';
+import { getEventLog } from '@kbn/test-suites-xpack-platform/alerting_api_integration/common/lib';
+import type { RetryService } from '@kbn/ftr-common-functional-services';
+import type { Client } from '@elastic/elasticsearch';
 import type { FtrProviderContext } from '../../../common/ftr_provider_context';
 import {
   obsOnlySpacesAll,
@@ -28,6 +33,9 @@ export default ({ getService }: FtrProviderContext) => {
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const secureSearch = getService('secureSearch');
   const kbnClient = getService('kibanaServer');
+  const es = getService('es');
+  const supertest = getService('supertest');
+  const retry = getService('retry');
 
   describe('ruleRegistryAlertsSearchStrategy', () => {
     let kibanaVersion: string;
@@ -983,6 +991,55 @@ export default ({ getService }: FtrProviderContext) => {
         expect(result.rawResponse.hits.total).to.eql(0);
       });
     });
+
+    describe('internal rule types', () => {
+      const alertAsDataIndex = '.internal.alerts-observability.test.alerts.alerts-default-000001';
+      const objectRemover = new ObjectRemover(supertest);
+      const rulePayload = getAlwaysFiringInternalRule();
+      let ruleId: string;
+
+      before(async () => {
+        await deleteAllAlertsFromIndex(alertAsDataIndex, es);
+      });
+
+      beforeEach(async () => {
+        const { body: createdRule1 } = await supertest
+          .post('/api/alerting/rule')
+          .set('kbn-xsrf', 'foo')
+          .send(rulePayload)
+          .expect(200);
+
+        ruleId = createdRule1.id;
+        objectRemover.add('default', createdRule1.id, 'rule', 'alerting');
+      });
+
+      afterEach(async () => {
+        await deleteAllAlertsFromIndex(alertAsDataIndex, es);
+        await objectRemover.removeAll();
+      });
+
+      it('should not return alerts from internal rule types', async () => {
+        await waitForRuleExecution(retry, getService, ruleId);
+        await waitForActiveAlerts(es, retry, alertAsDataIndex, rulePayload.rule_type_id);
+
+        const result = await secureSearch.send<RuleRegistrySearchResponse>({
+          supertestWithoutAuth,
+          auth: {
+            username: superUser.username,
+            password: superUser.password,
+          },
+          referer: 'test',
+          internalOrigin: 'Kibana',
+          options: {
+            ruleTypeIds: [rulePayload.rule_type_id],
+          },
+          strategy: 'privateRuleRegistryAlertsSearchStrategy',
+        });
+
+        expect(result.rawResponse.hits.total).to.eql(0);
+        expect(result.rawResponse.hits.hits.length).to.eql(0);
+      });
+    });
   });
 };
 
@@ -1001,3 +1058,51 @@ const validateRuleTypeIds = (result: RuleRegistrySearchResponse, ruleTypeIdsToVe
     )
   ).to.eql(true);
 };
+
+const waitForRuleExecution = async (
+  retry: RetryService,
+  getService: FtrProviderContext['getService'],
+  ruleId: string
+) => {
+  return await retry.try(async () => {
+    await getEventLog({
+      getService,
+      spaceId: 'default',
+      type: 'alert',
+      id: ruleId,
+      provider: 'alerting',
+      actions: new Map([['active-instance', { gte: 1 }]]),
+    });
+  });
+};
+
+const waitForActiveAlerts = async (
+  es: Client,
+  retry: RetryService,
+  alertAsDataIndex: string,
+  ruleTypeId: string
+) => {
+  await retry.try(async () => {
+    const {
+      hits: { hits: activeAlerts },
+    } = await es.search({
+      index: alertAsDataIndex,
+      query: { match_all: {} },
+    });
+
+    activeAlerts.forEach((activeAlert: any) => {
+      expect(activeAlert._source[ALERT_RULE_TYPE_ID]).eql(ruleTypeId);
+    });
+  });
+};
+
+const deleteAllAlertsFromIndex = async (index: string, es: Client) => {
+  await es.deleteByQuery({
+    index,
+    query: {
+      match_all: {},
+    },
+    conflicts: 'proceed',
+    ignore_unavailable: true,
+  });
+};