[data views] data views + rollup index referenced by alias (#212592)

## Summary

Upgrading to 9.x involves reindexing indices created in 7.x, which does
include rollup indices. Reindexing means relying on aliases to preserve
existing index names. As it turns out, our existing code did not work
with rollups that referenced aliases, rather than indices. This is
because the index name is used as an object key even if it was retrieved
via alias.


Note - I need to verify this on 9.0 from scratch. I used upgraded data
and need to verify the steps to make this work when testing.

To test
1. Add sample data
2. Create a rollup job that references the sample data. 
3. Create a data view that references the rollup index. It may take a
few minutes for the rollup index to be populated.
4. Create an alias from the dev console, like such - 

```
POST _aliases
{
  "actions": [
    {
      "add": {
        "index": "rollup",
        "alias": "my-alias"
      }
    }
  ]
}
``` 
5. Create a rollup data view based in the alias you just created.

Part of https://github.com/elastic/kibana/issues/211850

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

src/platform/plugins/shared/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -142,7 +142,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
           params: {
             rollup_index: rollupIndex,
           },
-          aggs: rollupIndicesCapabilities[rollupIndex].aggs,
+          aggs: rollupCaps?.aggs,
         };
       }
 
@@ -176,6 +176,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
   const isLoadingSources = useObservable(dataViewEditorService.isLoadingSources$, true);
   const existingDataViewNames = useObservable(dataViewEditorService.dataViewNames$);
   const rollupIndex = useObservable(dataViewEditorService.rollupIndex$);
+  const rollupCaps = useObservable(dataViewEditorService.rollupCaps$);
   const rollupIndicesCapabilities = useObservable(dataViewEditorService.rollupIndicesCaps$, {});
 
   useDebounce(
@@ -194,7 +195,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
     dataViewEditorService.setType(type);
   }, [dataViewEditorService, type]);
 
-  const getRollupIndices = (rollupCaps: RollupIndicesCapsResponse) => Object.keys(rollupCaps);
+  const getRollupIndices = (rollupCapsRes: RollupIndicesCapsResponse) => Object.keys(rollupCapsRes);
 
   const onTypeChange = useCallback(
     (newType: INDEX_PATTERN_TYPE) => {

src/platform/plugins/shared/data_view_editor/public/components/form_fields/title_field.tsx

@@ -80,8 +80,15 @@ const createMatchesIndicesValidator = ({
     }
 
     // A rollup index pattern needs to match one and only one rollup index.
-    const rollupIndexMatches = matchedIndices.exactMatchedIndices.filter((matchedIndex) =>
+    const rollupIndexMatches = matchedIndices.exactMatchedIndices.filter(
-      rollupIndices.includes(matchedIndex.name)
+      (matchedIndex) =>
+        rollupIndices.includes(matchedIndex.name) ||
+        // matched item is alias
+        (matchedIndex.item.indices?.length === 1 &&
+          rollupIndices.includes(matchedIndex.item.indices[0])) ||
+        // matched item is an index referenced by an alias
+        (matchedIndex.item.aliases?.length === 1 &&
+          rollupIndices.includes(matchedIndex.item.aliases[0]))
     );
 
     if (!rollupIndexMatches.length) {

src/platform/plugins/shared/data_view_editor/public/data_view_editor_service.ts

@@ -27,7 +27,12 @@ import {
   DataViewField,
 } from '@kbn/data-views-plugin/public';
 
-import { RollupIndicesCapsResponse, MatchedIndicesSet, TimestampOption } from './types';
+import {
+  RollupIndicesCapsResponse,
+  RollupIndiciesCapability,
+  MatchedIndicesSet,
+  TimestampOption,
+} from './types';
 import { getMatchedIndices, ensureMinimumTime, extractTimeFields, removeSpaces } from './lib';
 import { GetFieldsOptions } from './shared_imports';
 
@@ -70,6 +75,7 @@ interface DataViewEditorState {
   loadingTimestampFields: boolean;
   timestampFieldOptions: TimestampOption[];
   rollupIndexName?: string | null;
+  rollupCaps?: RollupIndiciesCapability;
 }
 
 const defaultDataViewEditorState: DataViewEditorState = {
@@ -119,6 +125,7 @@ export class DataViewEditorService {
     this.loadingTimestampFields$ = stateSelector((state) => state.loadingTimestampFields);
     this.timestampFieldOptions$ = stateSelector((state) => state.timestampFieldOptions);
     this.rollupIndex$ = stateSelector((state) => state.rollupIndexName);
+    this.rollupCaps$ = stateSelector((state) => state.rollupCaps);
 
     // when list of matched indices is updated always update timestamp fields
     this.loadTimestampFieldsSub = this.matchedIndices$.subscribe(() => this.loadTimestampFields());
@@ -162,6 +169,8 @@ export class DataViewEditorService {
 
   // current matched rollup index
   rollupIndex$: Observable<string | undefined | null>;
+  // current matched rollup capabilities
+  rollupCaps$: Observable<RollupIndiciesCapability | undefined>;
   // alernates between value and undefined so validation can treat new value as thought its a promise
   private rollupIndexForProvider$ = new Subject<string | undefined | null>();
 
@@ -244,11 +253,27 @@ export class DataViewEditorService {
     // verify we're looking at the current result
     if (currentLoadingMatchedIndicesIdx === this.currentLoadingMatchedIndices) {
       if (type === INDEX_PATTERN_TYPE.ROLLUP) {
-        const rollupIndices = exactMatched.filter((index) => isRollupIndex(index.name));
+        const rollupIndices = exactMatched.filter(
+          (index) =>
+            isRollupIndex(index.name) ||
+            // if its an alias
+            (index.item.indices?.length === 1 && isRollupIndex(index.item.indices[0])) ||
+            // if its an index referenced by an alias
+            (index.item.aliases?.length === 1 && isRollupIndex(index.item.aliases[0]))
+        );
+
         newRollupIndexName = rollupIndices.length === 1 ? rollupIndices[0].name : null;
-        this.updateState({ rollupIndexName: newRollupIndexName });
+        const newRollupCaps = await this.rollupCapsResponse.then((response) => {
+          return (
+            response[newRollupIndexName || ''] ||
+            // if its an alias
+            response[rollupIndices[0]?.item.indices?.[0] || '']
+          );
+        });
+
+        this.updateState({ rollupIndexName: newRollupIndexName, rollupCaps: newRollupCaps });
       } else {
-        this.updateState({ rollupIndexName: null });
+        this.updateState({ rollupIndexName: null, rollupCaps: undefined });
       }
 
       this.updateState({ matchedIndices });

src/platform/plugins/shared/data_views/public/services/get_indices.test.ts

@@ -21,7 +21,7 @@ export const successfulResolveResponse = {
   aliases: [
     {
       name: 'f-alias',
-      indices: ['freeze-index', 'my-index'],
+      indices: ['my-index'],
     },
   ],
   data_streams: [
@@ -69,6 +69,16 @@ describe('getIndices', () => {
     expect(result[2].name).toBe('remoteCluster1:bar-01');
   });
 
+  it('should work with rollup indices based on aliases', async () => {
+    const isRollupIdx = (indexName: string) => indexName === 'my-index';
+    const result = await getIndices({
+      http,
+      pattern: 'kibana',
+      isRollupIndex: isRollupIdx,
+    });
+    expect(result[0].tags[1].key).toBe('rollup');
+  });
+
   it('should ignore ccs query-all', async () => {
     expect((await getIndices({ http, pattern: '*:', isRollupIndex })).length).toBe(0);
   });

src/platform/plugins/shared/data_views/public/services/get_indices.ts

@@ -119,6 +119,9 @@ export const responseToItemArray = (
     const isFrozen = (index.attributes || []).includes(ResolveIndexResponseItemIndexAttrs.FROZEN);
 
     tags.push(...getTags(index.name));
+    index.aliases?.forEach((alias) => {
+      tags.push(...getTags(alias));
+    });
     if (isFrozen) {
       tags.push({ name: frozenLabel, key: 'frozen', color: 'danger' });
     }
@@ -130,11 +133,15 @@ export const responseToItemArray = (
     });
   });
   (response.aliases || []).forEach((alias) => {
-    source.push({
+    const item = {
       name: alias.name,
       tags: [{ key: 'alias', name: aliasLabel, color: 'default' }],
       item: alias,
-    });
+    };
+    // we only need to check the first index to see if its a rollup since there can only be one alias match
+    item.tags.push(...getTags(alias.indices[0]));
+    item.tags.push(...getTags(alias.name));
+    source.push(item);
   });
   (response.data_streams || []).forEach((dataStream) => {
     source.push({

src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.test.ts

@@ -71,6 +71,19 @@ describe('Index Pattern Fetcher - server', () => {
     expect(esClient.rollup.getRollupIndexCaps).toHaveBeenCalledTimes(1);
   });
 
+  it("works with index aliases - when rollup response doesn't have index as key", async () => {
+    esClient.rollup.getRollupIndexCaps.mockResponse(
+      rollupResponse as unknown as estypes.RollupGetRollupIndexCapsResponse
+    );
+    indexPatterns = new IndexPatternsFetcher(esClient, optionalParams);
+    await indexPatterns.getFieldsForWildcard({
+      pattern: patternList,
+      type: DataViewType.ROLLUP,
+      rollupIndex: 'foo',
+    });
+    expect(esClient.rollup.getRollupIndexCaps).toHaveBeenCalledTimes(1);
+  });
+
   it("doesn't call rollup api when given rollup data view and rollups are disabled", async () => {
     esClient.rollup.getRollupIndexCaps.mockResponse(
       rollupResponse as unknown as estypes.RollupGetRollupIndexCapsResponse

src/platform/plugins/shared/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -122,11 +122,15 @@ export class IndexPatternsFetcher {
 
     if (this.rollupsEnabled && type === DataViewType.ROLLUP && rollupIndex) {
       const rollupFields: FieldDescriptor[] = [];
-      const capabilityCheck = getCapabilitiesForRollupIndices(
+      const capabilities = getCapabilitiesForRollupIndices(
         await this.elasticsearchClient.rollup.getRollupIndexCaps({
           index: rollupIndex,
         })
-      )[rollupIndex];
+      );
+
+      const capabilityCheck =
+        // use the rollup index name BUT if its an alias, we'll take the first one
+        capabilities[rollupIndex] || capabilities[Object.keys(capabilities)[0]];
 
       if (capabilityCheck.error) {
         throw new Error(capabilityCheck.error);

x-pack/platform/plugins/private/rollup/server/rollup_data_enricher.ts

@@ -7,6 +7,7 @@
 
 import { IScopedClusterClient } from '@kbn/core/server';
 import { Index } from '@kbn/index-management-plugin/server';
+import { isArray } from 'lodash';
 
 export const rollupDataEnricher = async (indicesList: Index[], client: IScopedClusterClient) => {
   if (!indicesList || !indicesList.length) {
@@ -19,7 +20,10 @@ export const rollupDataEnricher = async (indicesList: Index[], client: IScopedCl
     });
 
     return indicesList.map((index) => {
-      const isRollupIndex = !!rollupJobData[index.name];
+      let isRollupIndex = !!rollupJobData[index.name];
+      if (!isRollupIndex && isArray(index.aliases)) {
+        isRollupIndex = index.aliases.some((alias) => !!rollupJobData[alias]);
+      }
       return {
         ...index,
         isRollupIndex,

x-pack/test/functional/apps/rollup_job/hybrid_index_pattern.js

@@ -113,6 +113,30 @@ export default function ({ getService, getPageObjects }) {
       expect(fields).to.eql(['@timestamp', '_id', '_ignored', '_index', '_score', '_source']);
     });
 
+    it('create hybrid index pattern - with alias to rollup index', async () => {
+      const rollupAlias = 'rollup-alias';
+      await es.indices.putAlias({
+        index: rollupTargetIndexName,
+        name: rollupAlias,
+      });
+      await PageObjects.common.navigateToApp('settings');
+      await PageObjects.settings.createIndexPattern(rollupAlias, '@timestamp', false);
+
+      await PageObjects.settings.clickKibanaIndexPatterns();
+      const indexPatternNames = await PageObjects.settings.getAllIndexPatternNames();
+      //The assertion is going to check that the string has the right name and that the text Rollup
+      //is included (since there is a Rollup tag).
+      const filteredIndexPatternNames = indexPatternNames.filter(
+        (i) => i.includes(rollupIndexPatternName) && i.includes('Rollup')
+      );
+      expect(filteredIndexPatternNames.length).to.be(1);
+
+      // ensure all fields are available
+      await PageObjects.settings.clickIndexPatternByName(rollupAlias);
+      const fields = await PageObjects.settings.getFieldNames();
+      expect(fields).to.eql(['@timestamp', '_id', '_ignored', '_index', '_score', '_source']);
+    });
+
     after(async () => {
       // Delete the rollup job.
       await es.rollup.deleteJob({ id: rollupJobName });