[Enterprise Search] Add polling for index data to Search Index view in Content app (#137083)

* Clean-up IndexViewLogic

* Add polling for index data to Search Index view

* Stop flashing UX

* Stop further flashing UX

* Fix TotalStats flashing on poll

* Fix tests

* Fix clearFetchIndexTimeoutId -> clearFetchIndexTimeout

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.test.ts

@@ -6,8 +6,8 @@
  */
 
 import { LogicMounter, mockFlashMessageHelpers } from '../../../__mocks__/kea_logic';
-
 import { apiIndex, connectorIndex } from '../../__mocks__/view_index.mock';
+import './_mocks_/index_name_logic.mock';
 
 import { nextTick } from '@kbn/test-jest-helpers';
 
@@ -21,10 +21,11 @@ import { IngestionMethod, IngestionStatus } from '../../types';
 
 import { indexToViewIndex } from '../../utils/indices';
 
-import { IndexViewLogic } from './index_view_logic';
+import { IndexViewLogic, IndexViewValues } from './index_view_logic';
 
-const DEFAULT_VALUES = {
+const DEFAULT_VALUES: IndexViewValues = {
   data: undefined,
+  fetchIndexTimeoutId: null,
   index: undefined,
   ingestionMethod: IngestionMethod.API,
   ingestionStatus: IngestionStatus.CONNECTED,
@@ -32,7 +33,7 @@ const DEFAULT_VALUES = {
   isWaitingForSync: false,
   lastUpdated: null,
   localSyncNowValue: false,
-  syncStatus: undefined,
+  syncStatus: null,
 };
 
 const CONNECTOR_VALUES = {
@@ -62,12 +63,17 @@ describe('IndexViewLogic', () => {
   });
 
   describe('actions', () => {
-    describe('fetchIndex.apiSuccess', () => {
+    describe('FetchIndexApiLogic.apiSuccess', () => {
+      beforeEach(() => {
+        IndexViewLogic.actions.createNewFetchIndexTimeout = jest.fn();
+      });
+
       it('should update values', () => {
         FetchIndexApiLogic.actions.apiSuccess({
           ...connectorIndex,
           connector: { ...connectorIndex.connector!, sync_now: true },
         });
+
         expect(IndexViewLogic.values).toEqual({
           ...CONNECTOR_VALUES,
           data: {
@@ -79,54 +85,77 @@ describe('IndexViewLogic', () => {
             connector: { ...CONNECTOR_VALUES.index.connector, sync_now: true },
           },
           isWaitingForSync: true,
+          localSyncNowValue: true,
           syncStatus: SyncStatus.COMPLETED,
         });
       });
+
       it('should update values with no connector', () => {
         FetchIndexApiLogic.actions.apiSuccess(apiIndex);
+
         expect(IndexViewLogic.values).toEqual({
           ...DEFAULT_VALUES,
           data: apiIndex,
           index: apiIndex,
         });
       });
+
+      it('should call createNewFetchIndexTimeout', () => {
+        FetchIndexApiLogic.actions.apiSuccess(apiIndex);
+
+        expect(IndexViewLogic.actions.createNewFetchIndexTimeout).toHaveBeenCalled();
+      });
     });
+
+    describe('fetchIndex.apiError', () => {
+      beforeEach(() => {
+        IndexViewLogic.actions.createNewFetchIndexTimeout = jest.fn();
+      });
+
+      it('should call createNewFetchIndexTimeout', () => {
+        FetchIndexApiLogic.actions.apiError({} as HttpError);
+
+        expect(IndexViewLogic.actions.createNewFetchIndexTimeout).toHaveBeenCalled();
+      });
+    });
+
     describe('startSync', () => {
-      it('should call makeRequest', async () => {
+      it('should call makeStartSyncRequest', async () => {
+        // TODO: replace with mounting connectorIndex to FetchIndexApiDirectly to avoid
+        // needing to mock out actions unrelated to test called by listeners
+        IndexViewLogic.actions.createNewFetchIndexTimeout = jest.fn();
         FetchIndexApiLogic.actions.apiSuccess(connectorIndex);
-        IndexViewLogic.actions.makeRequest = jest.fn();
+        IndexViewLogic.actions.makeStartSyncRequest = jest.fn();
+
         IndexViewLogic.actions.startSync();
         await nextTick();
-        expect(IndexViewLogic.actions.makeRequest).toHaveBeenCalledWith({
+
+        expect(IndexViewLogic.actions.makeStartSyncRequest).toHaveBeenCalledWith({
           connectorId: '2',
         });
-        expect(IndexViewLogic.values).toEqual({
-          ...CONNECTOR_VALUES,
-          syncStatus: SyncStatus.COMPLETED,
-        });
       });
     });
-    describe('apiSuccess', () => {
+
+    describe('StartSyncApiLogic.apiSuccess', () => {
       it('should set localSyncNow to true', async () => {
-        FetchIndexApiLogic.actions.apiSuccess(connectorIndex);
-        StartSyncApiLogic.actions.apiSuccess({});
-        expect(IndexViewLogic.values).toEqual({
-          ...CONNECTOR_VALUES,
-          isWaitingForSync: true,
-          localSyncNowValue: true,
-          syncStatus: SyncStatus.COMPLETED,
+        mount({
+          localSyncNowValue: false,
         });
+        StartSyncApiLogic.actions.apiSuccess({});
+
+        expect(IndexViewLogic.values.localSyncNowValue).toEqual(true);
       });
     });
   });
 
   describe('listeners', () => {
-    it('calls clearFlashMessages on makeRequest', () => {
-      IndexViewLogic.actions.makeRequest({ connectorId: 'connectorId' });
+    it('calls clearFlashMessages on makeStartSyncRequest', () => {
+      IndexViewLogic.actions.makeStartSyncRequest({ connectorId: 'connectorId' });
       expect(mockFlashMessageHelpers.clearFlashMessages).toHaveBeenCalledTimes(1);
     });
+
     it('calls flashAPIErrors on apiError', () => {
-      IndexViewLogic.actions.apiError({} as HttpError);
+      IndexViewLogic.actions.startSyncApiError({} as HttpError);
       expect(mockFlashMessageHelpers.flashAPIErrors).toHaveBeenCalledTimes(1);
       expect(mockFlashMessageHelpers.flashAPIErrors).toHaveBeenCalledWith({});
     });

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/index_view_logic.ts

@@ -31,15 +31,32 @@ import {
   isConnectorIndex,
 } from '../../utils/indices';
 
-export type IndicesActions = Pick<
-  Actions<StartSyncArgs, {}>,
-  'makeRequest' | 'apiSuccess' | 'apiError'
-> & {
-  fetchIndexSuccess: Actions<FetchIndexApiParams, FetchIndexApiResponse>['apiSuccess'];
+import { IndexNameLogic } from './index_name_logic';
+
+const FETCH_INDEX_POLLING_DURATION = 5000; // 5 seconds
+const FETCH_INDEX_POLLING_DURATION_ON_FAILURE = 30000; // 30 seconds
+
+type FetchIndexApiValues = Actions<FetchIndexApiParams, FetchIndexApiResponse>;
+type StartSyncApiValues = Actions<StartSyncArgs, {}>;
+
+export interface IndexViewActions {
+  clearFetchIndexTimeout(): void;
+  createNewFetchIndexTimeout(duration: number): { duration: number };
+  fetchIndexApiSuccess: FetchIndexApiValues['apiSuccess'];
+  makeFetchIndexRequest: FetchIndexApiValues['makeRequest'];
+  makeStartSyncRequest: StartSyncApiValues['makeRequest'];
+  resetFetchIndexApi: FetchIndexApiValues['apiReset'];
+  setFetchIndexTimeoutId(timeoutId: NodeJS.Timeout): { timeoutId: NodeJS.Timeout };
+  startFetchIndexPoll(): void;
   startSync(): void;
-};
-export interface IndicesValues {
+  startSyncApiError: StartSyncApiValues['apiError'];
+  startSyncApiSuccess: StartSyncApiValues['apiSuccess'];
+  stopFetchIndexPoll(): void;
+}
+
+export interface IndexViewValues {
   data: typeof FetchIndexApiLogic.values.data;
+  fetchIndexTimeoutId: NodeJS.Timeout | null;
   index: ElasticsearchViewIndex | undefined;
   ingestionMethod: IngestionMethod;
   ingestionStatus: IngestionStatus;
@@ -47,41 +64,95 @@ export interface IndicesValues {
   isWaitingForSync: boolean;
   lastUpdated: string | null;
   localSyncNowValue: boolean; // holds local value after update so UI updates correctly
-  syncStatus: SyncStatus;
+  syncStatus: SyncStatus | null;
 }
 
-export const IndexViewLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>({
+export const IndexViewLogic = kea<MakeLogicType<IndexViewValues, IndexViewActions>>({
   actions: {
+    clearFetchIndexTimeout: true,
+    createNewFetchIndexTimeout: (duration) => ({ duration }),
+    setFetchIndexTimeoutId: (timeoutId) => ({ timeoutId }),
+    startFetchIndexPoll: true,
     startSync: true,
+    stopFetchIndexPoll: true,
   },
   connect: {
-    actions: [StartSyncApiLogic, ['makeRequest', 'apiSuccess', 'apiError']],
+    actions: [
+      StartSyncApiLogic,
+      [
+        'apiError as startSyncApiError',
+        'apiSuccess as startSyncApiSuccess',
+        'makeRequest as makeStartSyncRequest',
+      ],
+      FetchIndexApiLogic,
+      [
+        'apiError as fetchIndexApiError',
+        'apiReset as resetFetchIndexApi',
+        'apiSuccess as fetchIndexApiSuccess',
+        'makeRequest as makeFetchIndexRequest',
+      ],
+    ],
     values: [FetchIndexApiLogic, ['data']],
   },
   listeners: ({ actions, values }) => ({
-    apiError: (e) => flashAPIErrors(e),
-    apiSuccess: () => {
+    createNewFetchIndexTimeout: ({ duration }) => {
+      if (values.fetchIndexTimeoutId) {
+        clearTimeout(values.fetchIndexTimeoutId);
+      }
+      const { indexName } = IndexNameLogic.values;
+      const timeoutId = setTimeout(() => {
+        actions.makeFetchIndexRequest({ indexName });
+      }, duration);
+      actions.setFetchIndexTimeoutId(timeoutId);
+    },
+    fetchIndexApiError: () => {
+      actions.createNewFetchIndexTimeout(FETCH_INDEX_POLLING_DURATION_ON_FAILURE);
+    },
+    fetchIndexApiSuccess: () => {
+      actions.createNewFetchIndexTimeout(FETCH_INDEX_POLLING_DURATION);
+    },
+    makeStartSyncRequest: () => clearFlashMessages(),
+    startFetchIndexPoll: () => {
+      const { indexName } = IndexNameLogic.values;
+      // we rely on listeners for fetchIndexApiError and fetchIndexApiSuccess to handle reccuring polling
+      actions.makeFetchIndexRequest({ indexName });
+    },
+    startSync: () => {
+      if (isConnectorIndex(values.data)) {
+        actions.makeStartSyncRequest({ connectorId: values.data?.connector?.id });
+      }
+    },
+    startSyncApiError: (e) => flashAPIErrors(e),
+    startSyncApiSuccess: () => {
       flashSuccessToast(
         i18n.translate('xpack.enterpriseSearch.content.searchIndex.index.syncSuccess.message', {
           defaultMessage: 'Successfully scheduled a sync, waiting for a connector to pick it up',
         })
       );
     },
-    makeRequest: () => clearFlashMessages(),
-    startSync: () => {
-      if (isConnectorIndex(values.data)) {
-        actions.makeRequest({ connectorId: values.data?.connector?.id });
+    stopFetchIndexPoll: () => {
+      if (values.fetchIndexTimeoutId) {
+        clearTimeout(values.fetchIndexTimeoutId);
       }
+      actions.clearFetchIndexTimeout();
+      actions.resetFetchIndexApi();
     },
   }),
-  path: ['enterprise_search', 'content', 'view_index_logic'],
+  path: ['enterprise_search', 'content', 'index_view_logic'],
   reducers: {
+    fetchIndexTimeoutId: [
+      null,
+      {
+        clearFetchIndexTimeout: () => null,
+        setFetchIndexTimeoutId: (_, { timeoutId }) => timeoutId,
+      },
+    ],
     localSyncNowValue: [
       false,
       {
-        apiSuccess: () => true,
-        fetchIndexSuccess: (_, index) =>
+        fetchIndexApiSuccess: (_, index) =>
           isConnectorIndex(index) ? index.connector.sync_now : false,
+        startSyncApiSuccess: () => true,
       },
     ],
   },
@@ -91,13 +162,13 @@ export const IndexViewLogic = kea<MakeLogicType<IndicesValues, IndicesActions>>(
     ingestionStatus: [() => [selectors.data], (data) => getIngestionStatus(data)],
     isSyncing: [
       () => [selectors.syncStatus],
-      (syncStatus) => syncStatus === SyncStatus.IN_PROGRESS,
+      (syncStatus: SyncStatus) => syncStatus === SyncStatus.IN_PROGRESS,
     ],
     isWaitingForSync: [
       () => [selectors.data, selectors.localSyncNowValue],
       (data, localSyncNowValue) => data?.connector?.sync_now || localSyncNowValue,
     ],
     lastUpdated: [() => [selectors.data], (data) => getLastUpdated(data)],
-    syncStatus: [() => [selectors.data], (data) => data?.connector?.last_sync_status],
+    syncStatus: [() => [selectors.data], (data) => data?.connector?.last_sync_status ?? null],
   }),
 });

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/overview.logic.ts

@@ -38,10 +38,10 @@ interface OverviewLogicValues {
   data: typeof FetchIndexApiLogic.values.data;
   indexData: typeof FetchIndexApiLogic.values.data;
   isClientsPopoverOpen: boolean;
+  isError: boolean;
   isGenerateModalOpen: boolean;
   isLoading: boolean;
   isManageKeysPopoverOpen: boolean;
-  isSuccess: boolean;
   status: typeof FetchIndexApiLogic.values.status;
 }
 
@@ -102,7 +102,11 @@ export const OverviewLogic = kea<MakeLogicType<OverviewLogicValues, OverviewLogi
         apiKeyStatus === Status.SUCCESS ? apiKeyData.apiKey.encoded : '',
     ],
     indexData: [() => [selectors.data], (data) => data],
-    isLoading: [() => [selectors.status], (status) => status === Status.LOADING],
-    isSuccess: [() => [selectors.status], (status) => status === Status.SUCCESS],
+    isLoading: [
+      () => [selectors.status, selectors.data],
+      (status, data) =>
+        status === Status.IDLE || (typeof data === 'undefined' && status === Status.LOADING),
+    ],
+    isError: [() => [selectors.status], (status) => status === Status.ERROR],
   }),
 });

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/search_index.tsx

@@ -36,6 +36,7 @@ import { SearchIndexDomainManagement } from './crawler/domain_management/domain_
 import { SearchIndexDocuments } from './documents';
 import { SearchIndexIndexMappings } from './index_mappings';
 import { IndexNameLogic } from './index_name_logic';
+import { IndexViewLogic } from './index_view_logic';
 import { SearchIndexOverview } from './overview';
 
 export enum SearchIndexTabId {
@@ -51,8 +52,8 @@ export enum SearchIndexTabId {
 }
 
 export const SearchIndex: React.FC = () => {
-  const { makeRequest, apiReset } = useActions(FetchIndexApiLogic);
   const { data: indexData, status: indexApiStatus } = useValues(FetchIndexApiLogic);
+  const { startFetchIndexPoll, stopFetchIndexPoll } = useActions(IndexViewLogic);
   const { isCalloutVisible } = useValues(IndexCreatedCalloutLogic);
   const { tabId = SearchIndexTabId.OVERVIEW } = useParams<{
     tabId?: string;
@@ -61,8 +62,8 @@ export const SearchIndex: React.FC = () => {
   const { indexName } = useValues(IndexNameLogic);
 
   useEffect(() => {
-    makeRequest({ indexName });
-    return apiReset;
+    startFetchIndexPoll();
+    return stopFetchIndexPoll;
   }, [indexName]);
 
   const ALL_INDICES_TABS: EuiTabbedContentTab[] = [
@@ -146,7 +147,10 @@ export const SearchIndex: React.FC = () => {
     <EnterpriseSearchContentPageTemplate
       pageChrome={[...baseBreadcrumbs, indexName]}
       pageViewTelemetry={tabId}
-      isLoading={indexApiStatus === Status.LOADING || indexApiStatus === Status.IDLE}
+      isLoading={
+        indexApiStatus === Status.IDLE ||
+        (typeof indexData === 'undefined' && indexApiStatus === Status.LOADING)
+      }
       pageHeader={{
         pageTitle: indexName,
         rightSideItems: getHeaderActions(indexData),

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/search_index/total_stats.tsx

@@ -22,9 +22,10 @@ interface TotalStatsProps {
 }
 
 export const TotalStats: React.FC<TotalStatsProps> = ({ ingestionType, additionalItems = [] }) => {
-  const { indexData, isSuccess } = useValues(OverviewLogic);
+  const { indexData, isError, isLoading } = useValues(OverviewLogic);
   const documentCount = indexData?.total.docs.count ?? 0;
-  const isLoading = !isSuccess;
+  const hideStats = isLoading || isError;
+
   const stats: EuiStatProps[] = [
     {
       description: i18n.translate(
@@ -33,7 +34,7 @@ export const TotalStats: React.FC<TotalStatsProps> = ({ ingestionType, additiona
           defaultMessage: 'Ingestion type',
         }
       ),
-      isLoading,
+      isLoading: hideStats,
       title: ingestionType,
     },
     {
@@ -43,7 +44,7 @@ export const TotalStats: React.FC<TotalStatsProps> = ({ ingestionType, additiona
           defaultMessage: 'Document count',
         }
       ),
-      isLoading,
+      isLoading: hideStats,
       title: documentCount,
     },
     ...additionalItems,