Appends the saved objects documents count to the CoreUsageData service (#124308)

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/development/core/server/kibana-plugin-core-server.md

@@ -244,7 +244,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  --- | --- |
 |  [APP\_WRAPPER\_CLASS](./kibana-plugin-core-server.app_wrapper_class.md) | The class name for top level \*and\* nested application wrappers to ensure proper layout |
 |  [kibanaResponseFactory](./kibana-plugin-core-server.kibanaresponsefactory.md) | Set of helpers used to create <code>KibanaResponse</code> to form HTTP response on an incoming request. Should be returned as a result of [RequestHandler](./kibana-plugin-core-server.requesthandler.md) execution. |
-|  [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md) | Merges two saved object migration maps. |
+|  [mergeSavedObjectMigrationMaps](./kibana-plugin-core-server.mergesavedobjectmigrationmaps.md) | Merges two saved object migration maps.<!-- -->If there is a migration for a given version on only one of the maps, that migration function will be used:<!-- -->mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '4.5.6': g }<!-- -->) -<!-- -->&gt; { '1.2.3': f, '4.5.6': g }<!-- -->If there is a migration for a given version on both maps, the migrations will be composed:<!-- -->mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '1.2.3': g }<!-- -->) -<!-- -->&gt; { '1.2.3': (doc, context) =<!-- -->&gt; f(g(doc, context), context) } |
 |  [pollEsNodesVersion](./kibana-plugin-core-server.pollesnodesversion.md) |  |
 |  [ServiceStatusLevels](./kibana-plugin-core-server.servicestatuslevels.md) | The current "level" of availability of a service. |
 |  [validBodyOutput](./kibana-plugin-core-server.validbodyoutput.md) | The set of valid body.output |

docs/development/core/server/kibana-plugin-core-server.mergesavedobjectmigrationmaps.md

@@ -6,6 +6,14 @@
 
 Merges two saved object migration maps.
 
+If there is a migration for a given version on only one of the maps, that migration function will be used:
+
+mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '4.5.6': g }<!-- -->) -<!-- -->&gt; { '1.2.3': f, '4.5.6': g }
+
+If there is a migration for a given version on both maps, the migrations will be composed:
+
+mergeSavedObjectMigrationMaps(<!-- -->{ '1.2.3': f }<!-- -->, { '1.2.3': g }<!-- -->) -<!-- -->&gt; { '1.2.3': (doc, context) =<!-- -->&gt; f(g(doc, context), context) }
+
 <b>Signature:</b>
 
 ```typescript

src/core/server/core_usage_data/core_usage_data_service.mock.ts

@@ -140,6 +140,7 @@ const createStartContractMock = () => {
                 alias: 'test_index',
                 primaryStoreSizeBytes: 1,
                 storeSizeBytes: 1,
+                savedObjectsDocsCount: 1,
               },
             ],
             legacyUrlAliases: {

src/core/server/core_usage_data/core_usage_data_service.test.ts

@@ -218,6 +218,11 @@ describe('CoreUsageDataService', () => {
             },
           ],
         } as any);
+        elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({
+          body: {
+            count: '15',
+          },
+        } as any);
         elasticsearch.client.asInternalUser.cat.indices.mockResolvedValueOnce({
           body: [
             {
@@ -229,6 +234,11 @@ describe('CoreUsageDataService', () => {
             },
           ],
         } as any);
+        elasticsearch.client.asInternalUser.count.mockResolvedValueOnce({
+          body: {
+            count: '10',
+          },
+        } as any);
         elasticsearch.client.asInternalUser.search.mockResolvedValueOnce({
           body: {
             hits: { total: { value: 6 } },
@@ -384,6 +394,7 @@ describe('CoreUsageDataService', () => {
                     "docsCount": 10,
                     "docsDeleted": 10,
                     "primaryStoreSizeBytes": 2000,
+                    "savedObjectsDocsCount": "15",
                     "storeSizeBytes": 1000,
                   },
                   Object {
@@ -391,6 +402,7 @@ describe('CoreUsageDataService', () => {
                     "docsCount": 20,
                     "docsDeleted": 20,
                     "primaryStoreSizeBytes": 4000,
+                    "savedObjectsDocsCount": "10",
                     "storeSizeBytes": 2000,
                   },
                 ],

src/core/server/core_usage_data/core_usage_data_service.ts

@@ -131,11 +131,11 @@ export class CoreUsageDataService
             return acc.add(index);
           }, new Set<string>())
           .values()
-      ).map((index) => {
+      ).map(async (index) => {
         // The _cat/indices API returns the _index_ and doesn't return a way
         // to map back from the index to the alias. So we have to make an API
-        // call for every alias
+        // call for every alias. The document count is the lucene document count.
-        return elasticsearch.client.asInternalUser.cat
+        const catIndicesResults = await elasticsearch.client.asInternalUser.cat
           .indices<any[]>({
             index,
             format: 'JSON',
@@ -143,6 +143,7 @@ export class CoreUsageDataService
           })
           .then(({ body }) => {
             const stats = body[0];
+
             return {
               alias: kibanaOrTaskManagerIndex(index, kibanaIndex),
               docsCount: stats['docs.count'] ? parseInt(stats['docs.count'], 10) : 0,
@@ -153,6 +154,27 @@ export class CoreUsageDataService
                 : 0,
             };
           });
+        // We use the GET <index>/_count API to get the number of saved objects
+        // to monitor if the cluster will hit the scalling limit of saved object migrations
+        const savedObjectsCounts = await elasticsearch.client.asInternalUser
+          .count({
+            index,
+          })
+          .then(({ body }) => {
+            return {
+              savedObjectsDocsCount: body.count ? body.count : 0,
+            };
+          });
+        this.logger.debug(
+          `Lucene documents count ${catIndicesResults.docsCount} from index ${catIndicesResults.alias}`
+        );
+        this.logger.debug(
+          `Saved objects documents count ${savedObjectsCounts.savedObjectsDocsCount} from index ${catIndicesResults.alias}`
+        );
+        return {
+          ...catIndicesResults,
+          ...savedObjectsCounts,
+        };
       })
     );
   }

src/core/server/core_usage_data/types.ts

@@ -177,6 +177,7 @@ export interface CoreServicesUsageData {
       docsDeleted: number;
       storeSizeBytes: number;
       primaryStoreSizeBytes: number;
+      savedObjectsDocsCount: number;
     }[];
     legacyUrlAliases: {
       activeCount: number;

src/core/server/server.api.md

@@ -426,6 +426,7 @@ export interface CoreServicesUsageData {
             docsDeleted: number;
             storeSizeBytes: number;
             primaryStoreSizeBytes: number;
+            savedObjectsDocsCount: number;
         }[];
         legacyUrlAliases: {
             activeCount: number;

src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts

@@ -355,14 +355,14 @@ export function getCoreUsageCollector(
                 type: 'long',
                 _meta: {
                   description:
-                    'The number of documents in the index, including hidden nested documents.',
+                    'The number of lucene documents in the index, including hidden nested documents.',
                 },
               },
               docsDeleted: {
                 type: 'long',
                 _meta: {
                   description:
-                    'The number of deleted documents in the index, including hidden nested documents.',
+                    'The number of deleted lucene documents in the index, including hidden nested documents.',
                 },
               },
               alias: {
@@ -382,6 +382,12 @@ export function getCoreUsageCollector(
                   description: 'The size in bytes of the index, for primaries and replicas.',
                 },
               },
+              savedObjectsDocsCount: {
+                type: 'long',
+                _meta: {
+                  description: 'The number of saved objects documents in the index.',
+                },
+              },
             },
           },
           legacyUrlAliases: {

src/plugins/telemetry/schema/oss_plugins.json

@@ -6248,13 +6248,13 @@
                       "docsCount": {
                         "type": "long",
                         "_meta": {
-                          "description": "The number of documents in the index, including hidden nested documents."
+                          "description": "The number of lucene documents in the index, including hidden nested documents."
                         }
                       },
                       "docsDeleted": {
                         "type": "long",
                         "_meta": {
-                          "description": "The number of deleted documents in the index, including hidden nested documents."
+                          "description": "The number of deleted lucene documents in the index, including hidden nested documents."
                         }
                       },
                       "alias": {
@@ -6274,6 +6274,12 @@
                         "_meta": {
                           "description": "The size in bytes of the index, for primaries and replicas."
                         }
+                      },
+                      "savedObjectsDocsCount": {
+                        "type": "long",
+                        "_meta": {
+                          "description": "The number of saved objects documents in the index."
+                        }
                       }
                     }
                   }