Added hidden filter to data streams tab (#85028)

* Added hidden filter to data streams tab

* Fix i18n import

* Fixed tests

* hidden ds pr feedback

* Added includeHidden query to data streams list

* Changed how badge group renders data streams badges

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts

@@ -19,7 +19,7 @@ export interface DataStreamsTabTestBed extends TestBed<TestSubjects> {
     goToDataStreamsList: () => void;
     clickEmptyPromptIndexTemplateLink: () => void;
     clickIncludeStatsSwitch: () => void;
-    clickIncludeManagedSwitch: () => void;
+    toggleViewFilterAt: (index: number) => void;
     clickReloadButton: () => void;
     clickNameAt: (index: number) => void;
     clickIndicesAt: (index: number) => void;
@@ -82,9 +82,16 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
     find('includeStatsSwitch').simulate('click');
   };
 
-  const clickIncludeManagedSwitch = () => {
-    const { find } = testBed;
-    find('includeManagedSwitch').simulate('click');
+  const toggleViewFilterAt = (index: number) => {
+    const { find, component } = testBed;
+    act(() => {
+      find('viewButton').simulate('click');
+    });
+    component.update();
+    act(() => {
+      find('filterItem').at(index).simulate('click');
+    });
+    component.update();
   };
 
   const clickReloadButton = () => {
@@ -197,7 +204,7 @@ export const setup = async (overridingDependencies: any = {}): Promise<DataStrea
       goToDataStreamsList,
       clickEmptyPromptIndexTemplateLink,
       clickIncludeStatsSwitch,
-      clickIncludeManagedSwitch,
+      toggleViewFilterAt,
       clickReloadButton,
       clickNameAt,
       clickIndicesAt,
@@ -235,6 +242,7 @@ export const createDataStreamPayload = (dataStream: Partial<DataStream>): DataSt
   privileges: {
     delete_index: true,
   },
+  hidden: false,
   ...dataStream,
 });
 

x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts

@@ -19,6 +19,8 @@ import {
   createNonDataStreamIndex,
 } from './data_streams_tab.helpers';
 
+const nonBreakingSpace = ' ';
+
 describe('Data Streams tab', () => {
   const { server, httpRequestsMockHelpers } = setupEnvironment();
   let testBed: DataStreamsTabTestBed;
@@ -82,6 +84,25 @@ describe('Data Streams tab', () => {
       // Assert against the text because the href won't be available, due to dependency upon our core mock.
       expect(findEmptyPromptIndexTemplateLink().text()).toBe('Fleet');
     });
+
+    test('when hidden data streams are filtered by default, the table is rendered empty', async () => {
+      const hiddenDataStream = createDataStreamPayload({
+        name: 'hidden-data-stream',
+        hidden: true,
+      });
+      httpRequestsMockHelpers.setLoadDataStreamsResponse([hiddenDataStream]);
+
+      testBed = await setup({
+        plugins: {},
+      });
+
+      await act(async () => {
+        testBed.actions.goToDataStreamsList();
+      });
+
+      testBed.component.update();
+      expect(testBed.find('dataStreamTable').text()).toContain('No data streams found');
+    });
   });
 
   describe('when there are data streams', () => {
@@ -397,7 +418,6 @@ describe('Data Streams tab', () => {
   });
 
   describe('managed data streams', () => {
-    const nonBreakingSpace = ' ';
     beforeEach(async () => {
       const managedDataStream = createDataStreamPayload({
         name: 'managed-data-stream',
@@ -429,8 +449,8 @@ describe('Data Streams tab', () => {
       ]);
     });
 
-    test('turning off "Include managed" switch hides managed data streams', async () => {
-      const { exists, actions, component, table } = testBed;
+    test('turning off "managed" filter hides managed data streams', async () => {
+      const { actions, table } = testBed;
       let { tableCellsValues } = table.getMetaData('dataStreamTable');
 
       expect(tableCellsValues).toEqual([
@@ -438,18 +458,43 @@ describe('Data Streams tab', () => {
         ['', 'non-managed-data-stream', 'green', '1', 'Delete'],
       ]);
 
-      expect(exists('includeManagedSwitch')).toBe(true);
-
-      await act(async () => {
-        actions.clickIncludeManagedSwitch();
-      });
-      component.update();
+      actions.toggleViewFilterAt(0);
 
       ({ tableCellsValues } = table.getMetaData('dataStreamTable'));
       expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]);
     });
   });
 
+  describe('hidden data streams', () => {
+    beforeEach(async () => {
+      const hiddenDataStream = createDataStreamPayload({
+        name: 'hidden-data-stream',
+        hidden: true,
+      });
+      httpRequestsMockHelpers.setLoadDataStreamsResponse([hiddenDataStream]);
+
+      testBed = await setup({
+        history: createMemoryHistory(),
+      });
+      await act(async () => {
+        testBed.actions.goToDataStreamsList();
+      });
+      testBed.component.update();
+    });
+
+    test('show hidden data streams when filter is toggled', () => {
+      const { table, actions } = testBed;
+
+      actions.toggleViewFilterAt(1);
+
+      const { tableCellsValues } = table.getMetaData('dataStreamTable');
+
+      expect(tableCellsValues).toEqual([
+        ['', `hidden-data-stream${nonBreakingSpace}Hidden`, 'green', '1', 'Delete'],
+      ]);
+    });
+  });
+
   describe('data stream privileges', () => {
     describe('delete', () => {
       const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers;

x-pack/plugins/index_management/common/lib/data_stream_serialization.ts

@@ -19,6 +19,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS
     maximum_timestamp: maxTimeStamp,
     _meta,
     privileges,
+    hidden,
   } = dataStreamFromEs;
 
   return {
@@ -39,6 +40,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS
     maxTimeStamp,
     _meta,
     privileges,
+    hidden,
   };
 }
 

x-pack/plugins/index_management/common/types/data_streams.ts

@@ -38,6 +38,7 @@ export interface DataStreamFromEs {
   store_size?: string;
   maximum_timestamp?: number;
   privileges: PrivilegesFromEs;
+  hidden: boolean;
 }
 
 export interface DataStreamIndexFromEs {
@@ -59,6 +60,7 @@ export interface DataStream {
   maxTimeStamp?: number;
   _meta?: Meta;
   privileges: Privileges;
+  hidden: boolean;
 }
 
 export interface DataStreamIndex {

x-pack/plugins/index_management/public/application/lib/data_streams.tsx

@@ -6,10 +6,34 @@
 
 import { DataStream } from '../../../common';
 
-export const isManagedByIngestManager = (dataStream: DataStream): boolean => {
+export const isFleetManaged = (dataStream: DataStream): boolean => {
+  // TODO check if the wording will change to 'fleet'
   return Boolean(dataStream._meta?.managed && dataStream._meta?.managed_by === 'ingest-manager');
 };
 
-export const filterDataStreams = (dataStreams: DataStream[]): DataStream[] => {
-  return dataStreams.filter((dataStream: DataStream) => !isManagedByIngestManager(dataStream));
+export const filterDataStreams = (
+  dataStreams: DataStream[],
+  visibleTypes: string[]
+): DataStream[] => {
+  return dataStreams.filter((dataStream: DataStream) => {
+    // include all data streams that are neither hidden nor managed
+    if (!dataStream.hidden && !isFleetManaged(dataStream)) {
+      return true;
+    }
+    if (dataStream.hidden && visibleTypes.includes('hidden')) {
+      return true;
+    }
+    return isFleetManaged(dataStream) && visibleTypes.includes('managed');
+  });
+};
+
+export const isSelectedDataStreamHidden = (
+  dataStreams: DataStream[],
+  selectedDataStreamName?: string
+): boolean => {
+  return (
+    !!selectedDataStreamName &&
+    !!dataStreams.find((dataStream: DataStream) => dataStream.name === selectedDataStreamName)
+      ?.hidden
+  );
 };

x-pack/plugins/index_management/public/application/sections/home/components/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { FilterListButton, Filters } from './filter_list_button';

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_badges.tsx

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiBadge, EuiBadgeGroup } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { DataStream } from '../../../../../common';
+import { isFleetManaged } from '../../../lib/data_streams';
+
+interface Props {
+  dataStream: DataStream;
+}
+
+export const DataStreamsBadges: React.FunctionComponent<Props> = ({ dataStream }) => {
+  const badges = [];
+  if (isFleetManaged(dataStream)) {
+    badges.push(
+      <EuiBadge color="hollow">
+        <FormattedMessage
+          id="xpack.idxMgmt.dataStreamList.table.managedDataStreamBadge"
+          defaultMessage="Managed"
+        />
+      </EuiBadge>
+    );
+  }
+  if (dataStream.hidden) {
+    badges.push(
+      <EuiBadge color="hollow">
+        <FormattedMessage
+          id="xpack.idxMgmt.dataStreamList.table.hiddenDataStreamBadge"
+          defaultMessage="Hidden"
+        />
+      </EuiBadge>
+    );
+  }
+  return badges.length > 0 ? (
+    <>
+      &nbsp;
+      <EuiBadgeGroup>{badges}</EuiBadgeGroup>
+    </>
+  ) : null;
+};

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx

@@ -32,6 +32,7 @@ import { useUrlGenerator } from '../../../../services/use_url_generator';
 import { getIndexListUri, getTemplateDetailsLink } from '../../../../services/routing';
 import { ILM_PAGES_POLICY_EDIT, ILM_URL_GENERATOR_ID } from '../../../../constants';
 import { useAppContext } from '../../../../app_context';
+import { DataStreamsBadges } from '../data_stream_badges';
 
 interface DetailsListProps {
   details: Array<{
@@ -269,6 +270,7 @@ export const DataStreamDetailPanel: React.FunctionComponent<Props> = ({
           <EuiTitle size="m">
             <h2 id="dataStreamDetailPanelTitle" data-test-subj="dataStreamDetailPanelTitle">
               {dataStreamName}
+              {dataStream && <DataStreamsBadges dataStream={dataStream} />}
             </h2>
           </EuiTitle>
         </EuiFlyoutHeader>

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_list.tsx

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { useState } from 'react';
+import React, { useMemo, useState } from 'react';
 import { RouteComponentProps } from 'react-router-dom';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { i18n } from '@kbn/i18n';
@@ -32,8 +32,10 @@ import { documentationService } from '../../../services/documentation';
 import { Section } from '../home';
 import { DataStreamTable } from './data_stream_table';
 import { DataStreamDetailPanel } from './data_stream_detail_panel';
-import { filterDataStreams } from '../../../lib/data_streams';
+import { filterDataStreams, isSelectedDataStreamHidden } from '../../../lib/data_streams';
+import { FilterListButton, Filters } from '../components';
 
+export type DataStreamFilterName = 'managed' | 'hidden';
 interface MatchParams {
   dataStreamName?: string;
 }
@@ -45,7 +47,7 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
   location: { search },
   history,
 }) => {
-  const { isDeepLink } = extractQueryParams(search);
+  const { isDeepLink, includeHidden } = extractQueryParams(search);
   const decodedDataStreamName = attemptToURIDecode(dataStreamName);
 
   const {
@@ -54,11 +56,111 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
   } = useAppContext();
 
   const [isIncludeStatsChecked, setIsIncludeStatsChecked] = useState(false);
-  const [isIncludeManagedChecked, setIsIncludeManagedChecked] = useState(true);
   const { error, isLoading, data: dataStreams, resendRequest: reload } = useLoadDataStreams({
     includeStats: isIncludeStatsChecked,
   });
 
+  const [filters, setFilters] = useState<Filters<DataStreamFilterName>>({
+    managed: {
+      name: i18n.translate('xpack.idxMgmt.dataStreamList.viewManagedLabel', {
+        defaultMessage: 'Fleet-managed data streams',
+      }),
+      checked: 'on',
+    },
+    hidden: {
+      name: i18n.translate('xpack.idxMgmt.dataStreamList.viewHiddenLabel', {
+        defaultMessage: 'Hidden data streams',
+      }),
+      checked: includeHidden ? 'on' : 'off',
+    },
+  });
+
+  const activateHiddenFilter = (shouldBeActive: boolean) => {
+    if (shouldBeActive && filters.hidden.checked === 'off') {
+      setFilters({
+        ...filters,
+        hidden: {
+          ...filters.hidden,
+          checked: 'on',
+        },
+      });
+    }
+  };
+
+  const filteredDataStreams = useMemo(() => {
+    if (!dataStreams) {
+      // If dataStreams are not fetched, return empty array.
+      return [];
+    }
+
+    const visibleTypes = Object.entries(filters)
+      .filter(([name, _filter]) => _filter.checked === 'on')
+      .map(([name]) => name);
+
+    return filterDataStreams(dataStreams, visibleTypes);
+  }, [dataStreams, filters]);
+
+  const renderHeader = () => {
+    return (
+      <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
+        <EuiFlexItem>
+          <EuiText color="subdued">
+            <FormattedMessage
+              id="xpack.idxMgmt.dataStreamList.dataStreamsDescription"
+              defaultMessage="Data streams store time-series data across multiple indices. {learnMoreLink}"
+              values={{
+                learnMoreLink: (
+                  <EuiLink
+                    href={documentationService.getDataStreamsDocumentationLink()}
+                    target="_blank"
+                    external
+                  >
+                    {i18n.translate('xpack.idxMgmt.dataStreamListDescription.learnMoreLinkText', {
+                      defaultMessage: 'Learn more.',
+                    })}
+                  </EuiLink>
+                ),
+              }}
+            />
+          </EuiText>
+        </EuiFlexItem>
+
+        <EuiFlexItem grow={false}>
+          <EuiFlexGroup gutterSize="s">
+            <EuiFlexItem grow={false}>
+              <EuiSwitch
+                label={i18n.translate(
+                  'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel',
+                  {
+                    defaultMessage: 'Include stats',
+                  }
+                )}
+                checked={isIncludeStatsChecked}
+                onChange={(e) => setIsIncludeStatsChecked(e.target.checked)}
+                data-test-subj="includeStatsSwitch"
+              />
+            </EuiFlexItem>
+
+            <EuiFlexItem grow={false}>
+              <EuiIconTip
+                content={i18n.translate(
+                  'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
+                  {
+                    defaultMessage: 'Including stats can increase reload times',
+                  }
+                )}
+                position="top"
+              />
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiFlexItem>
+        <EuiFlexItem grow={false}>
+          <FilterListButton<DataStreamFilterName> filters={filters} onChange={setFilters} />
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    );
+  };
+
   let content;
 
   if (isLoading) {
@@ -150,94 +252,10 @@ export const DataStreamList: React.FunctionComponent<RouteComponentProps<MatchPa
       />
     );
   } else if (Array.isArray(dataStreams) && dataStreams.length > 0) {
-    const filteredDataStreams = isIncludeManagedChecked
-      ? dataStreams
-      : filterDataStreams(dataStreams);
+    activateHiddenFilter(isSelectedDataStreamHidden(dataStreams, decodedDataStreamName));
     content = (
       <>
-        <EuiFlexGroup alignItems="center" justifyContent="spaceBetween">
-          <EuiFlexItem>
-            <EuiText color="subdued">
-              <FormattedMessage
-                id="xpack.idxMgmt.dataStreamList.dataStreamsDescription"
-                defaultMessage="Data streams store time-series data across multiple indices. {learnMoreLink}"
-                values={{
-                  learnMoreLink: (
-                    <EuiLink
-                      href={documentationService.getDataStreamsDocumentationLink()}
-                      target="_blank"
-                      external
-                    >
-                      {i18n.translate('xpack.idxMgmt.dataStreamListDescription.learnMoreLinkText', {
-                        defaultMessage: 'Learn more.',
-                      })}
-                    </EuiLink>
-                  ),
-                }}
-              />
-            </EuiText>
-          </EuiFlexItem>
-
-          <EuiFlexItem grow={false}>
-            <EuiFlexGroup gutterSize="s">
-              <EuiFlexItem grow={false}>
-                <EuiSwitch
-                  label={i18n.translate(
-                    'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchLabel',
-                    {
-                      defaultMessage: 'Include stats',
-                    }
-                  )}
-                  checked={isIncludeStatsChecked}
-                  onChange={(e) => setIsIncludeStatsChecked(e.target.checked)}
-                  data-test-subj="includeStatsSwitch"
-                />
-              </EuiFlexItem>
-
-              <EuiFlexItem grow={false}>
-                <EuiIconTip
-                  content={i18n.translate(
-                    'xpack.idxMgmt.dataStreamListControls.includeStatsSwitchToolTip',
-                    {
-                      defaultMessage: 'Including stats can increase reload times',
-                    }
-                  )}
-                  position="top"
-                />
-              </EuiFlexItem>
-            </EuiFlexGroup>
-          </EuiFlexItem>
-          <EuiFlexItem grow={false}>
-            <EuiFlexGroup gutterSize="s">
-              <EuiFlexItem grow={false}>
-                <EuiSwitch
-                  label={i18n.translate(
-                    'xpack.idxMgmt.dataStreamListControls.includeManagedSwitchLabel',
-                    {
-                      defaultMessage: 'Include Fleet-managed streams',
-                    }
-                  )}
-                  checked={isIncludeManagedChecked}
-                  onChange={(e) => setIsIncludeManagedChecked(e.target.checked)}
-                  data-test-subj="includeManagedSwitch"
-                />
-              </EuiFlexItem>
-
-              <EuiFlexItem grow={false}>
-                <EuiIconTip
-                  content={i18n.translate(
-                    'xpack.idxMgmt.dataStreamListControls.includeManagedSwitchToolTip',
-                    {
-                      defaultMessage: 'Display data streams managed by Fleet',
-                    }
-                  )}
-                  position="top"
-                />
-              </EuiFlexItem>
-            </EuiFlexGroup>
-          </EuiFlexItem>
-        </EuiFlexGroup>
-
+        {renderHeader()}
         <EuiSpacer size="l" />
 
         <DataStreamTable

x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx

@@ -7,23 +7,16 @@
 import React, { useState, Fragment } from 'react';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
-import {
-  EuiInMemoryTable,
-  EuiBasicTableColumn,
-  EuiButton,
-  EuiLink,
-  EuiBadge,
-  EuiToolTip,
-} from '@elastic/eui';
+import { EuiInMemoryTable, EuiBasicTableColumn, EuiButton, EuiLink } from '@elastic/eui';
 import { ScopedHistory } from 'kibana/public';
 
 import { DataStream } from '../../../../../../common/types';
 import { UseRequestResponse, reactRouterNavigate } from '../../../../../shared_imports';
 import { getDataStreamDetailsLink, getIndexListUri } from '../../../../services/routing';
-import { isManagedByIngestManager } from '../../../../lib/data_streams';
 import { DataHealth } from '../../../../components';
 import { DeleteDataStreamConfirmationModal } from '../delete_data_stream_confirmation_modal';
 import { humanizeTimeStamp } from '../humanize_time_stamp';
+import { DataStreamsBadges } from '../data_stream_badges';
 
 interface Props {
   dataStreams?: DataStream[];
@@ -61,26 +54,7 @@ export const DataStreamTable: React.FunctionComponent<Props> = ({
           >
             {name}
           </EuiLink>
-          {isManagedByIngestManager(dataStream) ? (
-            <Fragment>
-              &nbsp;
-              <EuiToolTip
-                content={i18n.translate(
-                  'xpack.idxMgmt.dataStreamList.table.managedDataStreamTooltip',
-                  {
-                    defaultMessage: 'Created and managed by Fleet',
-                  }
-                )}
-              >
-                <EuiBadge color="hollow">
-                  <FormattedMessage
-                    id="xpack.idxMgmt.dataStreamList.table.managedDataStreamBadge"
-                    defaultMessage="Managed"
-                  />
-                </EuiBadge>
-              </EuiToolTip>
-            </Fragment>
-          ) : null}
+          <DataStreamsBadges dataStream={dataStream} />
         </Fragment>
       );
     },

x-pack/plugins/index_management/public/application/sections/home/template_list/components/index.ts

@@ -4,6 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export * from './filter_list_button';
-
 export * from './template_type_indicator';

x-pack/plugins/index_management/public/application/sections/home/template_list/template_list.tsx

@@ -36,7 +36,7 @@ import { getIsLegacyFromQueryParams } from '../../../lib/index_templates';
 import { TemplateTable } from './template_table';
 import { TemplateDetails } from './template_details';
 import { LegacyTemplateTable } from './legacy_templates/template_table';
-import { FilterListButton, Filters } from './components';
+import { FilterListButton, Filters } from '../components';
 import { attemptToURIDecode } from '../../../../shared_imports';
 
 type FilterName = 'managed' | 'cloudManaged' | 'system';

x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts

@@ -65,12 +65,24 @@ const enhanceDataStreams = ({
   });
 };
 
+const getDataStreams = (client: ElasticsearchClient, name = '*') => {
+  // TODO update when elasticsearch client has update requestParams for 'indices.getDataStream'
+  return client.transport.request({
+    path: `/_data_stream/${encodeURIComponent(name)}`,
+    method: 'GET',
+    querystring: {
+      expand_wildcards: 'all',
+    },
+  });
+};
+
 const getDataStreamsStats = (client: ElasticsearchClient, name = '*') => {
   return client.transport.request({
     path: `/_data_stream/${encodeURIComponent(name)}/_stats`,
     method: 'GET',
     querystring: {
       human: true,
+      expand_wildcards: 'all',
     },
   });
 };
@@ -107,7 +119,7 @@ export function registerGetAllRoute({
       try {
         let {
           body: { data_streams: dataStreams },
-        } = await asCurrentUser.indices.getDataStream();
+        } = await getDataStreams(asCurrentUser);
 
         let dataStreamsStats;
         let dataStreamsPrivileges;
@@ -165,7 +177,7 @@ export function registerGetOneRoute({
             body: { data_streams: dataStreamsStats },
           },
         ] = await Promise.all([
-          asCurrentUser.indices.getDataStream({ name }),
+          getDataStreams(asCurrentUser, name),
           getDataStreamsStats(asCurrentUser, name),
         ]);
 

x-pack/test/api_integration/apis/management/index_management/data_streams.ts

@@ -14,6 +14,7 @@ import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../ftr_provider_context';
 // @ts-ignore
 import { API_BASE_PATH } from './constants';
+import { DataStream } from '../../../../../plugins/index_management/common';
 
 export default function ({ getService }: FtrProviderContext) {
   const supertest = getService('supertest');
@@ -66,32 +67,41 @@ export default function ({ getService }: FtrProviderContext) {
       before(async () => await createDataStream(testDataStreamName));
       after(async () => await deleteDataStream(testDataStreamName));
 
-      it('returns an array of all data streams', async () => {
+      it('returns an array of data streams', async () => {
         const { body: dataStreams } = await supertest
           .get(`${API_BASE_PATH}/data_streams`)
           .set('kbn-xsrf', 'xxx')
           .expect(200);
 
+        expect(dataStreams).to.be.an('array');
+
+        // returned array can contain automatically created data streams
+        const testDataStream = dataStreams.find(
+          (dataStream: DataStream) => dataStream.name === testDataStreamName
+        );
+
+        expect(testDataStream).to.be.ok();
+
         // ES determines these values so we'll just echo them back.
-        const { name: indexName, uuid } = dataStreams[0].indices[0];
-        expect(dataStreams).to.eql([
-          {
-            name: testDataStreamName,
-            privileges: {
-              delete_index: true,
-            },
-            timeStampField: { name: '@timestamp' },
-            indices: [
-              {
-                name: indexName,
-                uuid,
-              },
-            ],
-            generation: 1,
-            health: 'yellow',
-            indexTemplateName: testDataStreamName,
+        const { name: indexName, uuid } = testDataStream!.indices[0];
+
+        expect(testDataStream).to.eql({
+          name: testDataStreamName,
+          privileges: {
+            delete_index: true,
           },
-        ]);
+          timeStampField: { name: '@timestamp' },
+          indices: [
+            {
+              name: indexName,
+              uuid,
+            },
+          ],
+          generation: 1,
+          health: 'yellow',
+          indexTemplateName: testDataStreamName,
+          hidden: false,
+        });
       });
 
       it('includes stats when provided the includeStats query parameter', async () => {
@@ -100,12 +110,21 @@ export default function ({ getService }: FtrProviderContext) {
           .set('kbn-xsrf', 'xxx')
           .expect(200);
 
+        expect(dataStreams).to.be.an('array');
+
+        // returned array can contain automatically created data streams
+        const testDataStream = dataStreams.find(
+          (dataStream: DataStream) => dataStream.name === testDataStreamName
+        );
+
+        expect(testDataStream).to.be.ok();
+
         // ES determines these values so we'll just echo them back.
-        const { name: indexName, uuid } = dataStreams[0].indices[0];
-        const { storageSize, ...dataStreamWithoutStorageSize } = dataStreams[0];
+
+        const { name: indexName, uuid } = testDataStream!.indices[0];
+        const { storageSize, ...dataStreamWithoutStorageSize } = testDataStream!;
         assertDataStreamStorageSizeExists(storageSize);
 
-        expect(dataStreams.length).to.be(1);
         expect(dataStreamWithoutStorageSize).to.eql({
           name: testDataStreamName,
           privileges: {
@@ -122,6 +141,7 @@ export default function ({ getService }: FtrProviderContext) {
           health: 'yellow',
           indexTemplateName: testDataStreamName,
           maxTimeStamp: 0,
+          hidden: false,
         });
       });
 
@@ -152,6 +172,7 @@ export default function ({ getService }: FtrProviderContext) {
           health: 'yellow',
           indexTemplateName: testDataStreamName,
           maxTimeStamp: 0,
+          hidden: false,
         });
       });
     });