Add ".reindexed-v8-" prefix to the valid prefixes list (#204819)

As part of v9.0 readiness, we reindex the indices in v.8 but still has
data from v.7x.
As a result of the process, the reindexed indices get `.reindexed-v8-`
prefix.
This PR add that prefix to the valid prefixes list.


# To verify:

Run Kibana and ES in 7.17 (use `-E path.data=your-data-path` to keep the
data in your local)
Create some rules that generate alerts (Observability rules to have AAD)
Let them run for a while.
Stop Kibana and ES, switch to 8.x branch and run ES and Kibana again
Open the Upgrade Assistant.
It should show the `.internal.alerts-*` indices 
Click on them and start reindexing on the opened flyout.
Check that `.reindexed-v8-internal.alerts-*` index has been created
Let you rules run for a while again.
Your alerts should be updated and there shouldn't be any error on your
terminal.

---------

Co-authored-by: Ersin Erdal <ersin.erdal@elastic.co>
Co-authored-by: Ersin Erdal <92688503+ersin-erdal@users.noreply.github.com>

x-pack/platform/plugins/shared/alerting/server/alerts_service/alerts_service.test.ts

@@ -144,7 +144,10 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => {
 
   const indexPatterns = useDataStream
     ? [`.alerts-${context ? context : 'test'}.alerts-${namespace}`]
-    : [`.internal.alerts-${context ? context : 'test'}.alerts-${namespace}-*`];
+    : [
+        `.internal.alerts-${context ? context : 'test'}.alerts-${namespace}-*`,
+        `.reindexed-v8-internal.alerts-${context ? context : 'test'}.alerts-${namespace}-*`,
+      ];
   return {
     name: `.alerts-${context ? context : 'test'}.alerts-${namespace}-index-template`,
     body: {
@@ -564,7 +567,10 @@ describe('Alerts Service', () => {
               },
             });
             expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({
-              index: '.internal.alerts-test.alerts-default-*',
+              index: [
+                '.internal.alerts-test.alerts-default-*',
+                `.reindexed-v8-internal.alerts-test.alerts-default-*`,
+              ],
               name: '.alerts-test.alerts-*',
             });
           }
@@ -627,7 +633,10 @@ describe('Alerts Service', () => {
               },
             });
             expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({
-              index: '.internal.alerts-test.alerts-default-*',
+              index: [
+                '.internal.alerts-test.alerts-default-*',
+                `.reindexed-v8-internal.alerts-test.alerts-default-*`,
+              ],
               name: '.alerts-test.alerts-*',
             });
           }
@@ -686,7 +695,10 @@ describe('Alerts Service', () => {
               },
             });
             expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({
-              index: '.internal.alerts-test.alerts-default-*',
+              index: [
+                '.internal.alerts-test.alerts-default-*',
+                `.reindexed-v8-internal.alerts-test.alerts-default-*`,
+              ],
               name: '.alerts-test.alerts-*',
             });
           }
@@ -728,7 +740,10 @@ describe('Alerts Service', () => {
               },
             });
             expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(1, {
-              index: '.internal.alerts-test.alerts-default-*',
+              index: [
+                '.internal.alerts-test.alerts-default-*',
+                `.reindexed-v8-internal.alerts-test.alerts-default-*`,
+              ],
               name: '.alerts-test.alerts-*',
             });
           }
@@ -792,7 +807,10 @@ describe('Alerts Service', () => {
               },
             });
             expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(2, {
-              index: '.internal.alerts-test.alerts-another-namespace-*',
+              index: [
+                '.internal.alerts-test.alerts-another-namespace-*',
+                '.reindexed-v8-internal.alerts-test.alerts-another-namespace-*',
+              ],
               name: '.alerts-test.alerts-*',
             });
           }
@@ -826,7 +844,10 @@ describe('Alerts Service', () => {
             })
           );
           expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({
-            index: '.internal.alerts-test.alerts-default-*',
+            index: [
+              '.internal.alerts-test.alerts-default-*',
+              '.reindexed-v8-internal.alerts-test.alerts-default-*',
+            ],
             name: '.alerts-test.alerts-*',
           });
           expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2);
@@ -871,11 +892,12 @@ describe('Alerts Service', () => {
           const template = {
             name: `.alerts-empty.alerts-default-index-template`,
             body: {
-              index_patterns: [
-                useDataStreamForAlerts
-                  ? `.alerts-empty.alerts-default`
-                  : `.internal.alerts-empty.alerts-default-*`,
-              ],
+              index_patterns: useDataStreamForAlerts
+                ? [`.alerts-empty.alerts-default`]
+                : [
+                    `.internal.alerts-empty.alerts-default-*`,
+                    `.reindexed-v8-internal.alerts-empty.alerts-default-*`,
+                  ],
               composed_of: ['.alerts-framework-mappings'],
               ...(useDataStreamForAlerts ? { data_stream: { hidden: true } } : {}),
               priority: 7,
@@ -931,7 +953,10 @@ describe('Alerts Service', () => {
               },
             });
             expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({
-              index: '.internal.alerts-empty.alerts-default-*',
+              index: [
+                '.internal.alerts-empty.alerts-default-*',
+                '.reindexed-v8-internal.alerts-empty.alerts-default-*',
+              ],
               name: '.alerts-empty.alerts-*',
             });
           }

x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/create_or_update_index_template.ts

@@ -43,10 +43,12 @@ export const getIndexTemplate = ({
     namespace,
   };
 
-  const dataStreamFields = dataStreamAdapter.getIndexTemplateFields(
-    indexPatterns.alias,
-    indexPatterns.pattern
-  );
+  const patterns: string[] = [indexPatterns.pattern];
+  if (indexPatterns.reindexedPattern) {
+    patterns.push(indexPatterns.reindexedPattern);
+  }
+
+  const dataStreamFields = dataStreamAdapter.getIndexTemplateFields(indexPatterns.alias, patterns);
 
   const indexLifecycle = {
     name: ilmPolicyName,

x-pack/platform/plugins/shared/alerting/server/alerts_service/lib/data_stream_adapter.ts

@@ -16,7 +16,7 @@ import { retryTransientEsErrors } from './retry_transient_es_errors';
 
 export interface DataStreamAdapter {
   isUsingDataStreams(): boolean;
-  getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields;
+  getIndexTemplateFields(alias: string, patterns: string[]): IndexTemplateFields;
   createStream(opts: CreateConcreteWriteIndexOpts): Promise<void>;
 }
 
@@ -48,7 +48,7 @@ class DataStreamImplementation implements DataStreamAdapter {
     return true;
   }
 
-  getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields {
+  getIndexTemplateFields(alias: string, patterns: string[]): IndexTemplateFields {
     return {
       data_stream: { hidden: true },
       index_patterns: [alias],
@@ -66,9 +66,9 @@ class AliasImplementation implements DataStreamAdapter {
     return false;
   }
 
-  getIndexTemplateFields(alias: string, pattern: string): IndexTemplateFields {
+  getIndexTemplateFields(alias: string, patterns: string[]): IndexTemplateFields {
     return {
-      index_patterns: [pattern],
+      index_patterns: patterns,
       rollover_alias: alias,
     };
   }
@@ -134,10 +134,15 @@ async function createAliasStream(opts: CreateConcreteWriteIndexOpts): Promise<vo
   try {
     // Specify both the index pattern for the backing indices and their aliases
     // The alias prevents the request from finding other namespaces that could match the -* pattern
+    const patterns: string[] = [indexPatterns.pattern];
+    if (indexPatterns.reindexedPattern) {
+      patterns.push(indexPatterns.reindexedPattern);
+    }
+
     const response = await retryTransientEsErrors(
       () =>
         esClient.indices.getAlias({
-          index: indexPatterns.pattern,
+          index: patterns,
           name: indexPatterns.basePattern,
         }),
       { logger }
@@ -151,7 +156,7 @@ async function createAliasStream(opts: CreateConcreteWriteIndexOpts): Promise<vo
       }))
     );
 
-    logger.debug(
+    logger.info(
       () =>
         `Found ${concreteIndices.length} concrete indices for ${
           indexPatterns.name

x-pack/platform/plugins/shared/alerting/server/alerts_service/resource_installer_utils.test.ts

@@ -36,14 +36,17 @@ describe('getIndexTemplateAndPattern', () => {
     expect(getIndexTemplateAndPattern({ context: 'test' })).toEqual({
       template: '.alerts-test.alerts-default-index-template',
       pattern: '.internal.alerts-test.alerts-default-*',
+      reindexedPattern: '.reindexed-v8-internal.alerts-test.alerts-default-*',
       basePattern: '.alerts-test.alerts-*',
       alias: '.alerts-test.alerts-default',
       validPrefixes: [
         '.ds-.alerts-',
-        '.internal.alerts-',
         '.alerts-',
-        '.internal.preview.alerts-',
+        '.internal.alerts-',
+        '.reindexed-v8-internal.alerts-',
         '.preview.alerts-',
+        '.internal.preview.alerts-',
+        '.reindexed-v8-internal.preview.alerts-',
       ],
       name: '.internal.alerts-test.alerts-default-000001',
     });
@@ -53,14 +56,17 @@ describe('getIndexTemplateAndPattern', () => {
     expect(getIndexTemplateAndPattern({ context: 'test', namespace: 'special' })).toEqual({
       template: '.alerts-test.alerts-special-index-template',
       pattern: '.internal.alerts-test.alerts-special-*',
+      reindexedPattern: '.reindexed-v8-internal.alerts-test.alerts-special-*',
       basePattern: '.alerts-test.alerts-*',
       alias: '.alerts-test.alerts-special',
       validPrefixes: [
         '.ds-.alerts-',
-        '.internal.alerts-',
         '.alerts-',
-        '.internal.preview.alerts-',
+        '.internal.alerts-',
+        '.reindexed-v8-internal.alerts-',
         '.preview.alerts-',
+        '.internal.preview.alerts-',
+        '.reindexed-v8-internal.preview.alerts-',
       ],
       name: '.internal.alerts-test.alerts-special-000001',
     });
@@ -76,15 +82,18 @@ describe('getIndexTemplateAndPattern', () => {
     ).toEqual({
       template: '.alerts-test.alerts-special-index-template',
       pattern: '.internal.alerts-test.alerts-special-*',
+      reindexedPattern: '.reindexed-v8-internal.alerts-test.alerts-special-*',
       basePattern: '.alerts-test.alerts-*',
       alias: '.alerts-test.alerts-special',
       name: '.internal.alerts-test.alerts-special-000001',
       validPrefixes: [
         '.ds-.alerts-',
-        '.internal.alerts-',
         '.alerts-',
-        '.internal.preview.alerts-',
+        '.internal.alerts-',
+        '.reindexed-v8-internal.alerts-',
         '.preview.alerts-',
+        '.internal.preview.alerts-',
+        '.reindexed-v8-internal.preview.alerts-',
       ],
       secondaryAlias: `siem.signals-special`,
     });

x-pack/platform/plugins/shared/alerting/server/alerts_service/resource_installer_utils.ts

@@ -13,12 +13,15 @@ interface GetComponentTemplateNameOpts {
   context?: string;
   name?: string;
 }
+
 export const VALID_ALERT_INDEX_PREFIXES = [
   '.ds-.alerts-',
-  '.internal.alerts-',
   '.alerts-',
-  '.internal.preview.alerts-',
+  '.internal.alerts-',
+  '.reindexed-v8-internal.alerts-',
   '.preview.alerts-',
+  '.internal.preview.alerts-',
+  '.reindexed-v8-internal.preview.alerts-',
 ];
 
 export const getComponentTemplateName = ({ context, name }: GetComponentTemplateNameOpts = {}) =>
@@ -27,6 +30,7 @@ export const getComponentTemplateName = ({ context, name }: GetComponentTemplate
 export interface IIndexPatternString {
   template: string;
   pattern: string;
+  reindexedPattern?: string;
   alias: string;
   name: string;
   basePattern: string;
@@ -51,6 +55,7 @@ export const getIndexTemplateAndPattern = ({
   return {
     template: `.alerts-${patternWithNamespace}-index-template`,
     pattern: `.internal.alerts-${patternWithNamespace}-*`,
+    reindexedPattern: `.reindexed-v8-internal.alerts-${patternWithNamespace}-*`,
     basePattern: `.alerts-${pattern}-*`,
     name: `.internal.alerts-${patternWithNamespace}-000001`,
     alias: `.alerts-${patternWithNamespace}`,

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/install_resources.ts

@@ -147,6 +147,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
         expect(contextIndexTemplate.name).to.eql(indexTemplateName);
         expect(contextIndexTemplate.index_template.index_patterns).to.eql([
           '.internal.alerts-test.patternfiring.alerts-default-*',
+          '.reindexed-v8-internal.alerts-test.patternfiring.alerts-default-*',
         ]);
         expect(contextIndexTemplate.index_template.composed_of).to.eql([
           '.alerts-test.patternfiring.alerts-mappings',