[8.x] [Search] [Onboarding] Update document count (#196674) (#197201)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Search] [Onboarding] Update document count
(#196674)](https://github.com/elastic/kibana/pull/196674)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Joe
McElroy","email":"joseph.mcelroy@elastic.co"},"sourceCommit":{"committedDate":"2024-10-22T10:10:47Z","message":"[Search]
[Onboarding] Update document count (#196674)\n\n## Summary\r\n\r\nThis
uses the document list count within the quick stats. Before it
used\r\nthe document count from es3 billing and takes a while to
update.\r\n\r\n\r\nhttps://github.com/user-attachments/assets/9c29c88a-628f-4c63-99e5-d892a835e973\r\n\r\n###
Checklist\r\n\r\nDelete any items that are not applicable to this
PR.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n\r\n---------\r\n\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"5c51e783e5aee29bae1087403b75e2236b9ec0e8","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Search","backport:prev-minor"],"title":"[Search]
[Onboarding] Update document
count","number":196674,"url":"https://github.com/elastic/kibana/pull/196674","mergeCommit":{"message":"[Search]
[Onboarding] Update document count (#196674)\n\n## Summary\r\n\r\nThis
uses the document list count within the quick stats. Before it
used\r\nthe document count from es3 billing and takes a while to
update.\r\n\r\n\r\nhttps://github.com/user-attachments/assets/9c29c88a-628f-4c63-99e5-d892a835e973\r\n\r\n###
Checklist\r\n\r\nDelete any items that are not applicable to this
PR.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n\r\n---------\r\n\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"5c51e783e5aee29bae1087403b75e2236b9ec0e8"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/196674","number":196674,"mergeCommit":{"message":"[Search]
[Onboarding] Update document count (#196674)\n\n## Summary\r\n\r\nThis
uses the document list count within the quick stats. Before it
used\r\nthe document count from es3 billing and takes a while to
update.\r\n\r\n\r\nhttps://github.com/user-attachments/assets/9c29c88a-628f-4c63-99e5-d892a835e973\r\n\r\n###
Checklist\r\n\r\nDelete any items that are not applicable to this
PR.\r\n\r\n- [x] [Unit or
functional\r\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\r\nwere
updated or added to match the most common scenarios\r\n- [x] [Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests changed\r\n\r\n---------\r\n\r\nCo-authored-by:
Elastic Machine
<elasticmachine@users.noreply.github.com>","sha":"5c51e783e5aee29bae1087403b75e2236b9ec0e8"}}]}]
BACKPORT-->

Co-authored-by: Joe McElroy <joseph.mcelroy@elastic.co>

packages/kbn-search-index-documents/lib/fetch_search_results.test.ts

@@ -88,6 +88,7 @@ describe('fetchSearchResults lib function', () => {
       index: indexName,
       q: query,
       size: DEFAULT_DOCS_PER_PAGE,
+      track_total_hits: false,
     });
   });
 
@@ -109,6 +110,7 @@ describe('fetchSearchResults lib function', () => {
       index: indexName,
       q: '\\"yellow banana\\"',
       size: DEFAULT_DOCS_PER_PAGE,
+      track_total_hits: false,
     });
   });
 
@@ -123,6 +125,7 @@ describe('fetchSearchResults lib function', () => {
       from: DEFAULT_FROM_VALUE,
       index: indexName,
       size: DEFAULT_DOCS_PER_PAGE,
+      track_total_hits: false,
     });
   });
 
@@ -150,6 +153,42 @@ describe('fetchSearchResults lib function', () => {
       index: indexName,
       q: query,
       size: DEFAULT_DOCS_PER_PAGE,
+      track_total_hits: false,
+    });
+  });
+
+  it('should send track_total_hits true when specified', async () => {
+    mockClient.search.mockImplementationOnce(() =>
+      Promise.resolve({
+        ...mockSearchResponseWithHits,
+        hits: {
+          ...mockSearchResponseWithHits.hits,
+          total: {
+            ...mockSearchResponseWithHits.hits.total,
+            value: 0,
+          },
+          hits: [],
+        },
+      })
+    );
+
+    await expect(
+      fetchSearchResults(
+        mockClient as unknown as ElasticsearchClient,
+        indexName,
+        query,
+        0,
+        25,
+        true
+      )
+    ).resolves.toEqual(emptySearchResultsResponse);
+
+    expect(mockClient.search).toHaveBeenCalledWith({
+      from: DEFAULT_FROM_VALUE,
+      index: indexName,
+      q: query,
+      size: DEFAULT_DOCS_PER_PAGE,
+      track_total_hits: true,
     });
   });
 });

packages/kbn-search-index-documents/lib/fetch_search_results.ts

@@ -18,7 +18,8 @@ export const fetchSearchResults = async (
   indexName: string,
   query?: string,
   from: number = 0,
-  size: number = DEFAULT_DOCS_PER_PAGE
+  size: number = DEFAULT_DOCS_PER_PAGE,
+  trackTotalHits: boolean = false
 ): Promise<Paginate<SearchHit>> => {
   const result = await fetchWithPagination(
     async () =>
@@ -27,6 +28,7 @@ export const fetchSearchResults = async (
         index: indexName,
         size,
         ...(!!query ? { q: escapeLuceneChars(query) } : {}),
+        track_total_hits: trackTotalHits,
       }),
     from,
     size

x-pack/plugins/search_indices/public/components/indices/details_page.tsx

@@ -190,7 +190,7 @@ export const SearchIndexDetailsPage = () => {
   }, [isShowingDeleteModal]);
   const { euiTheme } = useEuiTheme();
 
-  if (isInitialLoading || isMappingsInitialLoading) {
+  if (isInitialLoading || isMappingsInitialLoading || indexDocumentsIsInitialLoading) {
     return (
       <SectionLoading>
         {i18n.translate('xpack.searchIndices.loadingDescription', {
@@ -209,7 +209,7 @@ export const SearchIndexDetailsPage = () => {
       panelled
       bottomBorder
     >
-      {isIndexError || isMappingsError || !index || !mappings ? (
+      {isIndexError || isMappingsError || !index || !mappings || !indexDocuments ? (
         <IndexloadingError
           error={indexError}
           navigateToIndexListPage={navigateToIndexListPage}
@@ -297,7 +297,7 @@ export const SearchIndexDetailsPage = () => {
                 </EuiFlexItem>
                 <EuiFlexItem>
                   <EuiFlexGroup>
-                    <QuickStats index={index} mappings={mappings} />
+                    <QuickStats indexDocuments={indexDocuments} index={index} mappings={mappings} />
                   </EuiFlexGroup>
                 </EuiFlexItem>
               </EuiFlexGroup>

x-pack/plugins/search_indices/public/components/quick_stats/quick_stats.tsx

@@ -22,10 +22,12 @@ import { Mappings } from '../../types';
 import { countVectorBasedTypesFromMappings } from './mappings_convertor';
 import { QuickStat } from './quick_stat';
 import { useKibana } from '../../hooks/use_kibana';
+import { IndexDocuments } from '../../hooks/api/use_document_search';
 
 export interface QuickStatsProps {
   index: Index;
   mappings: Mappings;
+  indexDocuments: IndexDocuments;
 }
 
 export const SetupAISearchButton: React.FC = () => {
@@ -60,12 +62,13 @@ export const SetupAISearchButton: React.FC = () => {
   );
 };
 
-export const QuickStats: React.FC<QuickStatsProps> = ({ index, mappings }) => {
+export const QuickStats: React.FC<QuickStatsProps> = ({ index, mappings, indexDocuments }) => {
   const [open, setOpen] = useState<boolean>(false);
   const { euiTheme } = useEuiTheme();
   const mappingStats = useMemo(() => countVectorBasedTypesFromMappings(mappings), [mappings]);
   const vectorFieldCount =
     mappingStats.sparse_vector + mappingStats.dense_vector + mappingStats.semantic_text;
+  const docCount = indexDocuments?.results._meta.page.total ?? 0;
 
   return (
     <EuiPanel
@@ -89,13 +92,13 @@ export const QuickStats: React.FC<QuickStatsProps> = ({ index, mappings }) => {
               defaultMessage: 'Document count',
             })}
             data-test-subj="QuickStatsDocumentCount"
-            secondaryTitle={<EuiI18nNumber value={index.documents ?? 0} />}
+            secondaryTitle={<EuiI18nNumber value={docCount ?? 0} />}
             stats={[
               {
                 title: i18n.translate('xpack.searchIndices.quickStats.documents.totalTitle', {
                   defaultMessage: 'Total',
                 }),
-                description: <EuiI18nNumber value={index.documents ?? 0} />,
+                description: <EuiI18nNumber value={docCount ?? 0} />,
               },
               {
                 title: i18n.translate('xpack.searchIndices.quickStats.documents.indexSize', {

x-pack/plugins/search_indices/public/hooks/api/use_delete_document.ts

@@ -40,12 +40,18 @@ export const useDeleteDocument = (indexName: string) => {
         queryClient.setQueryData(
           [QueryKeys.SearchDocuments, indexName],
           (snapshot: IndexDocuments | undefined) => {
-            const oldData = snapshot ?? { results: { data: [] } };
+            const oldData = snapshot ?? { results: { data: [], _meta: { page: { total: 0 } } } };
             return {
               ...oldData,
               results: {
                 ...oldData.results,
                 data: oldData.results.data.filter((doc: SearchHit) => doc._id !== id),
+                _meta: {
+                  page: {
+                    ...oldData.results._meta.page,
+                    total: oldData.results._meta.page.total - 1,
+                  },
+                },
               },
             } as IndexDocuments;
           }

x-pack/plugins/search_indices/public/hooks/api/use_document_search.ts

@@ -36,6 +36,7 @@ export const useIndexDocumentSearch = (indexName: string) => {
       http.post<IndexDocuments>(`/internal/serverless_search/indices/${indexName}/search`, {
         body: JSON.stringify({
           searchQuery: '',
+          trackTotalHits: true,
         }),
         query: {
           page: 0,

x-pack/plugins/serverless_search/server/routes/indices_routes.ts

@@ -107,6 +107,7 @@ export const registerIndicesRoutes = ({ logger, router }: RouteDependencies) =>
           searchQuery: schema.string({
             defaultValue: '',
           }),
+          trackTotalHits: schema.boolean({ defaultValue: false }),
         }),
         params: schema.object({
           index_name: schema.string(),
@@ -126,8 +127,16 @@ export const registerIndicesRoutes = ({ logger, router }: RouteDependencies) =>
       const searchQuery = request.body.searchQuery;
       const { page = 0, size = DEFAULT_DOCS_PER_PAGE } = request.query;
       const from = page * size;
+      const trackTotalHits = request.body.trackTotalHits;
 
-      const searchResults = await fetchSearchResults(client, indexName, searchQuery, from, size);
+      const searchResults = await fetchSearchResults(
+        client,
+        indexName,
+        searchQuery,
+        from,
+        size,
+        trackTotalHits
+      );
 
       return response.ok({
         body: {

x-pack/test_serverless/functional/page_objects/svl_search_index_detail_page.ts

@@ -42,6 +42,15 @@ export function SvlSearchIndexDetailPageProvider({ getService }: FtrProviderCont
       await quickStatsDocumentElem.click();
       expect(await quickStatsDocumentElem.getVisibleText()).to.contain('Index Size\n0b');
     },
+
+    async expectQuickStatsToHaveDocumentCount(count: number) {
+      const quickStatsElem = await testSubjects.find('quickStats');
+      const quickStatsDocumentElem = await quickStatsElem.findByTestSubject(
+        'QuickStatsDocumentCount'
+      );
+      expect(await quickStatsDocumentElem.getVisibleText()).to.contain(`Document count\n${count}`);
+    },
+
     async expectQuickStatsAIMappings() {
       await testSubjects.existOrFail('quickStats', { timeout: 2000 });
       const quickStatsElem = await testSubjects.find('quickStats');

x-pack/test_serverless/functional/test_suites/search/search_index_detail.ts

@@ -127,6 +127,9 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
         it('should have index documents', async () => {
           await pageObjects.svlSearchIndexDetailPage.expectHasIndexDocuments();
         });
+        it('should have one document in quick stats', async () => {
+          await pageObjects.svlSearchIndexDetailPage.expectQuickStatsToHaveDocumentCount(1);
+        });
         it('should have with data tabs', async () => {
           await pageObjects.svlSearchIndexDetailPage.expectWithDataTabsExists();
           await pageObjects.svlSearchIndexDetailPage.expectShouldDefaultToDataTab();
@@ -145,6 +148,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
           await pageObjects.svlSearchIndexDetailPage.withDataChangeTabs('dataTab');
           await pageObjects.svlSearchIndexDetailPage.clickFirstDocumentDeleteAction();
           await pageObjects.svlSearchIndexDetailPage.expectAddDocumentCodeExamples();
+          await pageObjects.svlSearchIndexDetailPage.expectQuickStatsToHaveDocumentCount(0);
         });
       });