[Response Ops][Alerting] Enable framework alerts as data by default (#154076)

## Summary

Setting `xpack.alerting.enableFrameworkAlerts` to true by default. This
causes alerts-as-data resource installation to be handled by the
alerting plugin and not the rule registry. We're keeping the feature
flag in case we run into issues but eventually we'll clean up the code
to remove the feature flag and clean up the rule registry code that
relies on the feature flag. Changing this default setting early will
allow us to identify issues before the 8.8 FF where we can revert if
needed.

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-securitysolution-es-utils/src/delete_all_index/index.ts

@@ -11,6 +11,7 @@ import type { ElasticsearchClient } from '../elasticsearch_client';
 export const deleteAllIndex = async (
   esClient: ElasticsearchClient,
   pattern: string,
+  specifyAlias: boolean = false,
   maxAttempts = 5
 ): Promise<boolean> => {
   for (let attempt = 1; ; attempt++) {
@@ -22,9 +23,14 @@ export const deleteAllIndex = async (
 
     // resolve pattern to concrete index names
     const { body: resp } = await esClient.indices.getAlias(
-      {
-        index: pattern,
-      },
+      specifyAlias
+        ? {
+            name: pattern,
+            index: `${pattern}-*`,
+          }
+        : {
+            index: pattern,
+          },
       { ignore: [404], meta: true }
     );
 

x-pack/plugins/alerting/server/config.test.ts

@@ -13,7 +13,7 @@ describe('config validation', () => {
     expect(configSchema.validate(config)).toMatchInlineSnapshot(`
       Object {
         "cancelAlertsOnRuleTimeout": true,
-        "enableFrameworkAlerts": false,
+        "enableFrameworkAlerts": true,
         "healthCheck": Object {
           "interval": "60m",
         },

x-pack/plugins/alerting/server/config.ts

@@ -62,7 +62,7 @@ export const configSchema = schema.object({
   maxEphemeralActionsPerAlert: schema.number({
     defaultValue: DEFAULT_MAX_EPHEMERAL_ACTIONS_PER_ALERT,
   }),
-  enableFrameworkAlerts: schema.boolean({ defaultValue: false }),
+  enableFrameworkAlerts: schema.boolean({ defaultValue: true }),
   cancelAlertsOnRuleTimeout: schema.boolean({ defaultValue: true }),
   rules: rulesSchema,
 });

x-pack/plugins/security_solution/cypress/e2e/data_sources/sourcerer.cy.ts

@@ -11,7 +11,6 @@ import { HOSTS_URL, TIMELINES_URL } from '../../urls/navigation';
 import {
   addIndexToDefault,
   clickAlertCheckbox,
-  deleteAlertsIndex,
   deselectSourcererOptions,
   isDataViewSelection,
   isHostsStatValue,
@@ -23,9 +22,9 @@ import {
   openAdvancedSettings,
   openDataViewSelection,
   openSourcerer,
+  refreshUntilAlertsIndexExists,
   resetSourcerer,
   saveSourcerer,
-  waitForAlertsIndexToExist,
 } from '../../tasks/sourcerer';
 import { postDataView } from '../../tasks/common';
 import { openTimelineUsingToggle } from '../../tasks/security_main';
@@ -46,7 +45,6 @@ const dataViews = ['auditbeat-*,fakebeat-*', 'auditbeat-*,*beat*,siem-read*,.kib
 describe('Sourcerer', () => {
   before(() => {
     esArchiverResetKibana();
-    deleteAlertsIndex();
     dataViews.forEach((dataView: string) => postDataView(dataView));
   });
   describe('permissions', () => {
@@ -144,19 +142,13 @@ describe('Timeline scope', () => {
     visit(TIMELINES_URL);
   });
 
-  it('correctly loads SIEM data view before and after signals index exists', () => {
+  it('correctly loads SIEM data view', () => {
     openTimelineUsingToggle();
     openSourcerer('timeline');
     isDataViewSelection(siemDataViewTitle);
     openAdvancedSettings();
     isSourcererSelection(`auditbeat-*`);
-    isNotSourcererSelection(`${DEFAULT_ALERTS_INDEX}-default`);
-    isSourcererOptions(
-      [...DEFAULT_INDEX_PATTERN, `${DEFAULT_ALERTS_INDEX}-default`].filter(
-        (pattern) => pattern !== 'auditbeat-*'
-      )
-    );
-    waitForAlertsIndexToExist();
+    isSourcererSelection(`${DEFAULT_ALERTS_INDEX}-default`);
     isSourcererOptions(DEFAULT_INDEX_PATTERN.filter((pattern) => pattern !== 'auditbeat-*'));
     isNotSourcererOption(`${DEFAULT_ALERTS_INDEX}-default`);
   });
@@ -211,7 +203,7 @@ describe('Timeline scope', () => {
     });
     beforeEach(() => {
       visit(TIMELINES_URL);
-      waitForAlertsIndexToExist();
+      refreshUntilAlertsIndexExists();
     });
     it('Modifies timeline to alerts only, and switches to different saved timeline without issue', function () {
       openTimelineById(this.timelineId).then(() => {

x-pack/plugins/security_solution/cypress/tasks/sourcerer.ts

@@ -10,8 +10,6 @@ import { HOSTS_URL } from '../urls/navigation';
 import { waitForPage } from './login';
 import { openTimelineUsingToggle } from './security_main';
 import { DEFAULT_ALERTS_INDEX } from '../../common/constants';
-import { createRule } from './api_calls/rules';
-import { getNewRule } from '../objects/rule';
 
 export const openSourcerer = (sourcererScope?: string) => {
   if (sourcererScope != null && sourcererScope === 'timeline') {
@@ -115,28 +113,7 @@ export const addIndexToDefault = (index: string) => {
     });
 };
 
-export const deleteAlertsIndex = () => {
-  const alertsIndexUrl = `${Cypress.env(
-    'ELASTICSEARCH_URL'
-  )}/.internal.alerts-security.alerts-default-000001`;
-
-  cy.request({
-    url: alertsIndexUrl,
-    method: 'GET',
-    headers: { 'kbn-xsrf': 'cypress-creds' },
-    failOnStatusCode: false,
-  }).then((response) => {
-    if (response.status === 200) {
-      cy.request({
-        url: alertsIndexUrl,
-        method: 'DELETE',
-        headers: { 'kbn-xsrf': 'cypress-creds' },
-      });
-    }
-  });
-};
-
-const refreshUntilAlertsIndexExists = async () => {
+export const refreshUntilAlertsIndexExists = async () => {
   cy.waitUntil(
     () => {
       cy.reload();
@@ -153,11 +130,6 @@ const refreshUntilAlertsIndexExists = async () => {
   );
 };
 
-export const waitForAlertsIndexToExist = () => {
-  createRule(getNewRule({ rule_id: '1', max_signals: 100 }));
-  refreshUntilAlertsIndexExists();
-};
-
 export const deleteRuntimeField = (dataView: string, fieldName: string) => {
   const deleteRuntimeFieldPath = `/api/data_views/data_view/${dataView}/runtime_field/${fieldName}`;
 

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/delete_index_route.ts

@@ -57,7 +57,7 @@ export const deleteIndexRoute = (router: SecuritySolutionPluginRouter) => {
             body: `index: "${index}" does not exist`,
           });
         } else {
-          await deleteAllIndex(esClient, index);
+          await deleteAllIndex(esClient, index, true);
           const policyExists = await getPolicyExists(esClient, index);
           if (policyExists) {
             await deletePolicy(esClient, index);

x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/alerting/alerts.ts

@@ -47,7 +47,7 @@ export default function alertTests({ getService }: FtrProviderContext) {
     after(async () => {
       await esTestIndexTool.destroy();
       await es.indices.delete({ index: authorizationIndex });
-      await es.indices.delete({ index: alertAsDataIndex });
+      await es.deleteByQuery({ index: alertAsDataIndex, query: { match_all: {} } });
     });
 
     for (const scenario of UserAtSpaceScenarios) {

x-pack/test/api_integration/apis/maps/maps_telemetry.ts

@@ -30,8 +30,8 @@ export default function ({ getService }: FtrProviderContext) {
           return fieldStat.name === 'geo_point';
         }
       );
-      expect(geoPointFieldStats.count).to.be(7);
-      expect(geoPointFieldStats.index_count).to.be(6);
+      expect(geoPointFieldStats.count).to.be(39);
+      expect(geoPointFieldStats.index_count).to.be(10);
 
       const geoShapeFieldStats = apiResponse.cluster_stats.indices.mappings.field_types.find(
         (fieldStat: estypes.ClusterStatsFieldTypes) => {

x-pack/test/apm_api_integration/tests/alerts/error_count_threshold.spec.ts

@@ -69,7 +69,8 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       await synthtraceEsClient.clean();
       await supertest.delete(`/api/alerting/rule/${ruleId}`).set('kbn-xsrf', 'foo');
       await supertest.delete(`/api/actions/connector/${actionId}`).set('kbn-xsrf', 'foo');
-      await esDeleteAllIndices(['.alerts*', INDEX_NAME]);
+      await esDeleteAllIndices(INDEX_NAME);
+      await es.deleteByQuery({ index: '.alerts*', query: { match_all: {} } });
       await es.deleteByQuery({
         index: '.kibana-event-log-*',
         query: { term: { 'kibana.alert.rule.consumer': 'apm' } },

x-pack/test/apm_api_integration/tests/service_groups/service_group_count/service_group_count.spec.ts

@@ -21,7 +21,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
   const apmApiClient = getService('apmApiClient');
   const supertest = getService('supertest');
   const synthtraceEsClient = getService('synthtraceEsClient');
-  const esDeleteAllIndices = getService('esDeleteAllIndices');
   const esClient = getService('es');
   const log = getService('log');
   const start = Date.now() - 24 * 60 * 60 * 1000;
@@ -90,7 +89,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
       after(async () => {
         await supertest.delete(`/api/alerting/rule/${ruleId}`).set('kbn-xsrf', 'true');
-        await esDeleteAllIndices('.alerts*');
+        await esClient.deleteByQuery({ index: '.alerts*', query: { match_all: {} } });
       });
 
       it('returns the correct number of alerts', async () => {

x-pack/test/apm_api_integration/tests/services/service_alerts.spec.ts

@@ -16,7 +16,6 @@ export default function ServiceAlerts({ getService }: FtrProviderContext) {
   const apmApiClient = getService('apmApiClient');
   const supertest = getService('supertest');
   const synthtraceEsClient = getService('synthtraceEsClient');
-  const esDeleteAllIndices = getService('esDeleteAllIndices');
   const esClient = getService('es');
   const log = getService('log');
   const start = Date.now() - 24 * 60 * 60 * 1000;
@@ -122,7 +121,7 @@ export default function ServiceAlerts({ getService }: FtrProviderContext) {
 
       after(async () => {
         await supertest.delete(`/api/alerting/rule/${ruleId}`).set('kbn-xsrf', 'true');
-        await esDeleteAllIndices('.alerts*');
+        await esClient.deleteByQuery({ index: '.alerts*', query: { match_all: {} } });
       });
 
       it('returns the correct number of alerts', async () => {

x-pack/test/detection_engine_api_integration/basic/tests/query_signals.ts

@@ -22,23 +22,6 @@ export default ({ getService }: FtrProviderContext) => {
 
   describe('query_signals_route and find_alerts_route', () => {
     describe('validation checks', () => {
-      it('should not give errors when querying and the signals index does not exist yet', async () => {
-        const { body } = await supertest
-          .post(DETECTION_ENGINE_QUERY_SIGNALS_URL)
-          .set('kbn-xsrf', 'true')
-          .send(getSignalStatus())
-          .expect(200);
-
-        // remove any server generated items that are indeterministic
-        delete body.took;
-
-        expect(body).to.eql({
-          timed_out: false,
-          _shards: { total: 0, successful: 0, skipped: 0, failed: 0 },
-          hits: { total: { value: 0, relation: 'eq' }, max_score: 0, hits: [] },
-        });
-      });
-
       // This fails and should be investigated or removed if it no longer applies
       it.skip('should not give errors when querying and the signals index does exist and is empty', async () => {
         await createSignalsIndex(supertest, log);
@@ -144,23 +127,6 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('find_alerts_route', () => {
       describe('validation checks', () => {
-        it('should not give errors when querying and the signals index does not exist yet', async () => {
-          const { body } = await supertest
-            .post(ALERTS_AS_DATA_FIND_URL)
-            .set('kbn-xsrf', 'true')
-            .send({ ...getSignalStatus(), index: '.siem-signals-default' })
-            .expect(200);
-
-          // remove any server generated items that are indeterministic
-          delete body.took;
-
-          expect(body).to.eql({
-            timed_out: false,
-            _shards: { total: 0, successful: 0, skipped: 0, failed: 0 },
-            hits: { total: { value: 0, relation: 'eq' }, max_score: 0, hits: [] },
-          });
-        });
-
         // This fails and should be investigated or removed if it no longer applies
         it.skip('should not give errors when querying and the signals index does exist and is empty', async () => {
           await createSignalsIndex(supertest, log);

x-pack/test/detection_engine_api_integration/security_and_spaces/group10/finalize_signals_migrations.ts

@@ -48,6 +48,19 @@ export default ({ getService }: FtrProviderContext): void => {
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const log = getService('log');
 
+  const getSignalsMigrationStatus = async (query: any) => {
+    const { body } = await supertest
+      .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
+      .query(query)
+      .set('kbn-xsrf', 'true')
+      .expect(200);
+
+    const filteredIndices = body.indices.filter(
+      (index: any) => index?.index !== '.internal.alerts-security.alerts-default-000001'
+    );
+    return filteredIndices;
+  };
+
   describe('Finalizing signals migrations', () => {
     let legacySignalsIndexName: string;
     let outdatedSignalsIndexName: string;
@@ -93,12 +106,9 @@ export default ({ getService }: FtrProviderContext): void => {
     });
 
     it('replaces the original index alias with the migrated one', async () => {
-      const { body } = await supertest
-        .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
-        .query({ from: '2020-10-10' })
-        .set('kbn-xsrf', 'true')
-        .expect(200);
-      const statusResponses: StatusResponse[] = body.indices;
+      const statusResponses: StatusResponse[] = await getSignalsMigrationStatus({
+        from: '2020-10-10',
+      });
       const indicesBefore = statusResponses.map((index) => index.index);
 
       expect(indicesBefore).to.contain(createdMigration.index);
@@ -170,17 +180,11 @@ export default ({ getService }: FtrProviderContext): void => {
         log
       );
 
-      const { body: bodyAfter } = await supertest
-        .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
-        .query({ from: '2020-10-10' })
-        .set('kbn-xsrf', 'true')
-        .expect(200);
-
-      const statusAfter: StatusResponse[] = bodyAfter.indices;
-      expect(statusAfter.map((s) => s.index)).to.eql([
+      const indices = await getSignalsMigrationStatus({ from: '2020-10-10' });
+      expect(indices.map((s: any) => s.index)).to.eql([
         ...createdMigrations.map((c) => c.migration_index),
       ]);
-      expect(statusAfter.map((s) => s.is_outdated)).to.eql([false, false]);
+      expect(indices.map((s: any) => s.is_outdated)).to.eql([false, false]);
     });
 
     // This fails and should be investigated or removed if it no longer applies

x-pack/test/detection_engine_api_integration/security_and_spaces/group10/get_signals_migration_status.ts

@@ -20,6 +20,19 @@ export default ({ getService }: FtrProviderContext): void => {
   const supertestWithoutAuth = getService('supertestWithoutAuth');
   const log = getService('log');
 
+  const getSignalsMigrationStatus = async (query: any) => {
+    const { body } = await supertest
+      .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
+      .query(query)
+      .set('kbn-xsrf', 'true')
+      .expect(200);
+
+    const filteredIndices = body.indices.filter(
+      (index: any) => index?.index !== '.internal.alerts-security.alerts-default-000001'
+    );
+    return filteredIndices;
+  };
+
   describe('Signals migration status', () => {
     let legacySignalsIndexName: string;
     beforeEach(async () => {
@@ -35,24 +48,12 @@ export default ({ getService }: FtrProviderContext): void => {
     });
 
     it('returns no indexes if no signals exist in the specified range', async () => {
-      const { body } = await supertest
-        .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
-        .query({ from: '2020-10-20' })
-        .set('kbn-xsrf', 'true')
-        .expect(200);
-
-      expect(body.indices).to.eql([]);
+      const indices = await getSignalsMigrationStatus({ from: '2020-10-20' });
+      expect(indices).to.eql([]);
     });
 
     it('includes an index if its signals are within the specified range', async () => {
-      const {
-        body: { indices },
-      } = await supertest
-        .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
-        .query({ from: '2020-10-10' })
-        .set('kbn-xsrf', 'true')
-        .expect(200);
-
+      const indices = await getSignalsMigrationStatus({ from: '2020-10-10' });
       expect(indices).length(1);
       expect(indices[0].index).to.eql(legacySignalsIndexName);
     });
@@ -62,13 +63,8 @@ export default ({ getService }: FtrProviderContext): void => {
         await esArchiver.load('x-pack/test/functional/es_archives/signals/outdated_signals_index')
       );
 
-      const { body } = await supertest
-        .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL)
-        .query({ from: '2020-10-10' })
-        .set('kbn-xsrf', 'true')
-        .expect(200);
-
-      expect(body.indices).to.eql([
+      const indices = await getSignalsMigrationStatus({ from: '2020-10-10' });
+      expect(indices).to.eql([
         {
           index: legacySignalsIndexName,
           is_outdated: true,

x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/index.ts

@@ -13,10 +13,11 @@ export default ({ loadTestFile }: FtrProviderContext): void => {
     loadTestFile(require.resolve('./eql'));
     loadTestFile(require.resolve('./machine_learning'));
     loadTestFile(require.resolve('./new_terms'));
-    loadTestFile(require.resolve('./query'));
     loadTestFile(require.resolve('./saved_query'));
     loadTestFile(require.resolve('./threat_match'));
     loadTestFile(require.resolve('./threshold'));
     loadTestFile(require.resolve('./non_ecs_fields'));
+
+    loadTestFile(require.resolve('./query'));
   });
 };

x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/query.ts

@@ -71,6 +71,7 @@ export default ({ getService }: FtrProviderContext) => {
   const esArchiver = getService('esArchiver');
   const es = getService('es');
   const log = getService('log');
+  const esDeleteAllIndices = getService('esDeleteAllIndices');
 
   describe('Query type rules', () => {
     before(async () => {
@@ -79,6 +80,10 @@ export default ({ getService }: FtrProviderContext) => {
       await esArchiver.load('x-pack/test/functional/es_archives/signals/severity_risk_overrides');
     });
 
+    afterEach(async () => {
+      await esDeleteAllIndices('.preview.alerts*');
+    });
+
     after(async () => {
       await esArchiver.unload('x-pack/test/functional/es_archives/auditbeat/hosts');
       await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/alerts/8.1.0');

x-pack/test/functional/es_archives/rule_registry/alerts/data.json

@@ -128,7 +128,7 @@
 {
   "type": "doc",
   "value": {
-    "index": ".alerts-observability.logs.alerts-default",
+    "index": ".alerts-observability.logs.alerts-000001",
     "id": "123456789XYZ",
     "source": {
       "event.kind": "signal",
@@ -149,7 +149,7 @@
 {
   "type": "doc",
   "value": {
-    "index": ".alerts-observability.logs.alerts-default",
+    "index": ".alerts-observability.logs.alerts-000001",
     "id": "space1alertLogs",
     "source": {
       "event.kind": "signal",

x-pack/test/functional/es_archives/rule_registry/alerts/mappings.json

@@ -53,7 +53,7 @@
 {
   "type": "index",
   "value": {
-    "index": ".alerts-observability.logs.alerts-default",
+    "index": ".alerts-observability.logs.alerts-000001",
     "mappings": {
       "properties": {
         "message": {

x-pack/test/functional/es_archives/session_view/alerts/mappings.json

@@ -3,7 +3,6 @@
   "value": {
     "aliases": {
       ".alerts-security.alerts-default": {
-        "is_write_index": true
       },
       ".siem-signals-default": {
         "is_write_index": false

x-pack/test/rule_registry/security_and_spaces/tests/basic/get_browser_fields_by_feature_id.ts

@@ -56,7 +56,20 @@ export default ({ getService }: FtrProviderContext) => {
           'logs',
           'uptime',
         ]);
-        expect(Object.keys(browserFields)).to.eql(['base', 'event', 'kibana', 'message']);
+        expect(Object.keys(browserFields)).to.eql([
+          'base',
+          'agent',
+          'anomaly',
+          'ecs',
+          'error',
+          'event',
+          'kibana',
+          'message',
+          'monitor',
+          'observer',
+          'tls',
+          'url',
+        ]);
       });
 
       it(`${superUser.username} should NOT be able to get browser fields for siem featureId`, async () => {

x-pack/test/rule_registry/spaces_only/tests/basic/bootstrap.ts

@@ -1,42 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import expect from '@kbn/expect';
-import type { FtrProviderContext } from '../../../common/ftr_provider_context';
-import { obsOnlyRead } from '../../../common/lib/authentication/users';
-import { getAlertsTargetIndices } from '../../../common/lib/helpers';
-
-// eslint-disable-next-line import/no-default-export
-export default function registryRulesApiTest({ getService }: FtrProviderContext) {
-  const es = getService('es');
-
-  describe('Rule Registry API', () => {
-    describe('with read permissions', () => {
-      it('does not bootstrap the apm rule indices', async () => {
-        const { body: targetIndices } = await getAlertsTargetIndices(
-          getService,
-          obsOnlyRead,
-          'space1'
-        );
-        const errorOrUndefined = await es.indices
-          .get({
-            index: targetIndices[0],
-            expand_wildcards: 'open',
-            allow_no_indices: false,
-          })
-          .then(() => {})
-          .catch((error) => {
-            return error.toString();
-          });
-
-        expect(errorOrUndefined).not.to.be(undefined);
-
-        expect(errorOrUndefined).to.contain('index_not_found_exception');
-      });
-    });
-  });
-}

x-pack/test/rule_registry/spaces_only/tests/basic/index.ts

@@ -18,8 +18,5 @@ export default ({ loadTestFile, getService }: FtrProviderContext): void => {
     after(async () => {
       await deleteSpaces(getService);
     });
-
-    // Basic
-    loadTestFile(require.resolve('./bootstrap'));
   });
 };

x-pack/test/rule_registry/spaces_only/tests/trial/create_rule.ts

@@ -46,20 +46,6 @@ export default function registryRulesApiTest({ getService }: FtrProviderContext)
 
   describe('Rule Registry API', async () => {
     describe('with write permissions', () => {
-      it('does not bootstrap indices on plugin startup', async () => {
-        const { body: targetIndices } = await getAlertsTargetIndices(getService, obsOnly, SPACE_ID);
-        try {
-          const res = await es.indices.get({
-            index: targetIndices[0],
-            expand_wildcards: 'open',
-            allow_no_indices: true,
-          });
-          expect(res).to.be.empty();
-        } catch (exc) {
-          expect(exc.statusCode).to.eql(404);
-        }
-      });
-
       describe('when creating a rule', () => {
         let createResponse: {
           alert: Rule;