[Enterprise Search] Search Apps. - fetch indices one-by-one (#156571)

## Summary

fetches a search application's indices' stats one at a time.

if even one index is not available the stats api returns an error[^1].
while fetching them all together is probably more efficient we have to
get them one-by-one just in case one isn't available.

### Checklist


- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

[^1]: <details><summary>stats errors response</summary>
    <pre>
    {
      "error": {
        "root_cause": [
          {
            "type": "index_not_found_exception",
            "reason": "no such index [sloane-books-001]",
            "resource.type": "index_or_alias",
            "resource.id": "sloane-books-001",
            "index_uuid": "_na_",
            "index": "sloane-books-001"
          }
        ],
        "type": "index_not_found_exception",
        "reason": "no such index [sloane-books-001]",
        "resource.type": "index_or_alias",
        "resource.id": "sloane-books-001",
        "index_uuid": "_na_",
        "index": "sloane-books-001"
      },
      "status": 404
    }
    </pre>
</details>

x-pack/plugins/enterprise_search/common/types/engines.ts

@@ -26,7 +26,7 @@ export interface EnterpriseSearchEngineDetails {
 }
 
 export interface EnterpriseSearchEngineIndex {
-  count: number;
+  count: number | null;
   health: HealthStatus | 'unknown';
   name: string;
 }

x-pack/plugins/enterprise_search/server/lib/engines/available_indices.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
+
+export const availableIndices = async (
+  client: IScopedClusterClient,
+  indices: string[]
+): Promise<string[]> => {
+  if (await client.asCurrentUser.indices.exists({ index: indices })) return indices;
+
+  const indicesAndExists: Array<[string, boolean]> = await Promise.all(
+    indices.map(async (index) => [index, await client.asCurrentUser.indices.exists({ index })])
+  );
+  return indicesAndExists.flatMap(([index, exists]) => (exists ? [index] : []));
+};

x-pack/plugins/enterprise_search/server/lib/engines/fetch_indices_stats.test.ts

@@ -12,13 +12,14 @@ describe('fetchIndicesStats lib function', () => {
   const mockClient = {
     asCurrentUser: {
       indices: {
+        exists: jest.fn(),
         stats: jest.fn(),
       },
     },
     asInternalUser: {},
   };
   const indices = ['test-index-name-1', 'test-index-name-2', 'test-index-name-3'];
-  const indicesStats = {
+  const indexStats = {
     indices: {
       'test-index-name-1': {
         health: 'GREEN',
@@ -96,15 +97,11 @@ describe('fetchIndicesStats lib function', () => {
   });
 
   it('should return hydrated indices', async () => {
-    mockClient.asCurrentUser.indices.stats.mockImplementationOnce(() => indicesStats);
+    mockClient.asCurrentUser.indices.exists.mockImplementationOnce(() => true);
+    mockClient.asCurrentUser.indices.stats.mockImplementationOnce(() => indexStats);
 
     await expect(
       fetchIndicesStats(mockClient as unknown as IScopedClusterClient, indices)
     ).resolves.toEqual(fetchIndicesStatsResponse);
-
-    expect(mockClient.asCurrentUser.indices.stats).toHaveBeenCalledWith({
-      index: indices,
-      metric: ['docs'],
-    });
   });
 });

x-pack/plugins/enterprise_search/server/lib/engines/fetch_indices_stats.ts

@@ -9,21 +9,23 @@ import { IScopedClusterClient } from '@kbn/core-elasticsearch-server/src/client/
 
 import { EnterpriseSearchEngineIndex } from '../../../common/types/engines';
 
-export const fetchIndicesStats = async (client: IScopedClusterClient, indices: string[]) => {
-  const { indices: indicesStats = {} } = await client.asCurrentUser.indices.stats({
-    index: indices,
+import { availableIndices } from './available_indices';
+
+export const fetchIndicesStats = async (
+  client: IScopedClusterClient,
+  indices: string[]
+): Promise<EnterpriseSearchEngineIndex[]> => {
+  const indicesStats = await client.asCurrentUser.indices.stats({
+    index: await availableIndices(client, indices),
     metric: ['docs'],
   });
 
-  const indicesWithStats = indices.map((indexName: string) => {
-    const indexStats = indicesStats[indexName];
-    const hydratedIndex: EnterpriseSearchEngineIndex = {
-      count: indexStats?.primaries?.docs?.count ?? 0,
+  return indices.map((index) => {
+    const indexStats = indicesStats.indices?.[index];
+    return {
+      count: indexStats?.primaries?.docs?.count ?? null,
       health: indexStats?.health ?? 'unknown',
-      name: indexName,
+      name: index,
     };
-    return hydratedIndex;
   });
-
-  return indicesWithStats;
 };

x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.test.ts

@@ -16,6 +16,7 @@ describe('engines field_capabilities', () => {
   const mockClient = {
     asCurrentUser: {
       fieldCaps: jest.fn(),
+      indices: { exists: jest.fn() },
     },
     asInternalUser: {},
   };
@@ -44,6 +45,7 @@ describe('engines field_capabilities', () => {
         indices: ['index-001'],
       };
 
+      mockClient.asCurrentUser.indices.exists.mockResolvedValueOnce(true);
       mockClient.asCurrentUser.fieldCaps.mockResolvedValueOnce(fieldCapsResponse);
       await expect(
         fetchEngineFieldCapabilities(mockClient as unknown as IScopedClusterClient, mockEngine)

x-pack/plugins/enterprise_search/server/lib/engines/field_capabilities.ts

@@ -14,6 +14,8 @@ import {
   SchemaField,
 } from '../../../common/types/engines';
 
+import { availableIndices } from './available_indices';
+
 export const fetchEngineFieldCapabilities = async (
   client: IScopedClusterClient,
   engine: EnterpriseSearchEngine
@@ -21,9 +23,9 @@ export const fetchEngineFieldCapabilities = async (
   const { name, updated_at_millis, indices } = engine;
   const fieldCapabilities = await client.asCurrentUser.fieldCaps({
     fields: '*',
-    include_unmapped: true,
-    index: indices,
     filters: '-metadata',
+    include_unmapped: true,
+    index: await availableIndices(client, indices),
   });
   const fields = parseFieldsCapabilities(fieldCapabilities);
   return {