Make dashboard listing load faster (#201401)

## Summary

close https://github.com/elastic/kibana-team/issues/1239
close https://github.com/elastic/kibana/issues/193109


This PR is an easy improvement to make dashboard listing load faster for
deployments with a lot of dashboards. See the investigation:
https://github.com/elastic/kibana-team/issues/1239#issuecomment-2491145140

For our overview cluster by excluding not needed references we reduce
the **gzipped** response size from **~550kb -> ~75kb**

Longer term we should implement proper server-side pagination

src/plugins/dashboard/public/dashboard_listing/hooks/use_dashboard_listing_table.tsx

@@ -213,6 +213,10 @@ export const useDashboardListingTable = ({
           size: listingLimit,
           hasReference: references,
           hasNoReference: referencesToExclude,
+          options: {
+            // include only tags references in the response to save bandwidth
+            includeReferences: ['tag'],
+          },
         })
         .then(({ total, hits }) => {
           const searchEndTime = window.performance.now();

src/plugins/dashboard/server/content_management/dashboard_storage.ts

@@ -341,7 +341,10 @@ export class DashboardStorage {
     const soResponse = await soClient.find<DashboardSavedObjectAttributes>(soQuery);
     const hits = soResponse.saved_objects
       .map((so) => {
-        const { item } = savedObjectToItem(so, false, soQuery.fields);
+        const { item } = savedObjectToItem(so, false, {
+          allowedAttributes: soQuery.fields,
+          allowedReferences: optionsToLatest?.includeReferences,
+        });
         return item;
       })
       // Ignore any saved objects that failed to convert to items.

src/plugins/dashboard/server/content_management/v3/cm_services.ts

@@ -437,6 +437,7 @@ export const dashboardSearchOptionsSchema = schema.maybe(
     {
       onlyTitle: schema.maybe(schema.boolean()),
       fields: schema.maybe(schema.arrayOf(schema.string())),
+      includeReferences: schema.maybe(schema.arrayOf(schema.oneOf([schema.literal('tag')]))),
       kuery: schema.maybe(schema.string()),
       cursor: schema.maybe(schema.number()),
       limit: schema.maybe(schema.number()),

src/plugins/dashboard/server/content_management/v3/transform_utils.test.ts

@@ -432,7 +432,9 @@ describe('savedObjectToItem', () => {
       },
     };
 
-    const { item, error } = savedObjectToItem(input, true, ['title', 'description']);
+    const { item, error } = savedObjectToItem(input, true, {
+      allowedAttributes: ['title', 'description'],
+    });
     expect(error).toBeNull();
     expect(item).toEqual({
       ...commonSavedObject,
@@ -457,6 +459,60 @@ describe('savedObjectToItem', () => {
     expect(item).toBeNull();
     expect(error).not.toBe(null);
   });
+
+  it('should include only requested references', () => {
+    const input = {
+      ...commonSavedObject,
+      references: [
+        {
+          type: 'tag',
+          id: 'tag1',
+          name: 'tag-ref-tag1',
+        },
+        {
+          type: 'index-pattern',
+          id: 'index-pattern1',
+          name: 'index-pattern-ref-index-pattern1',
+        },
+      ],
+      attributes: {
+        title: 'title',
+        description: 'my description',
+        timeRestore: false,
+      },
+    };
+
+    {
+      const { item } = savedObjectToItem(input, true, {
+        allowedAttributes: ['title', 'description'],
+      });
+      expect(item?.references).toEqual(input.references);
+    }
+
+    {
+      const { item } = savedObjectToItem(input, true, {
+        allowedAttributes: ['title', 'description'],
+        allowedReferences: ['tag'],
+      });
+      expect(item?.references).toEqual([input.references[0]]);
+    }
+
+    {
+      const { item } = savedObjectToItem(input, true, {
+        allowedAttributes: ['title', 'description'],
+        allowedReferences: [],
+      });
+      expect(item?.references).toEqual([]);
+    }
+
+    {
+      const { item } = savedObjectToItem({ ...input, references: undefined }, true, {
+        allowedAttributes: ['title', 'description'],
+        allowedReferences: [],
+      });
+      expect(item?.references).toBeUndefined();
+    }
+  });
 });
 
 describe('getResultV3ToV2', () => {

src/plugins/dashboard/server/content_management/v3/transform_utils.ts

@@ -304,24 +304,35 @@ type PartialSavedObject<T> = Omit<SavedObject<Partial<T>>, 'references'> & {
   references: SavedObjectReference[] | undefined;
 };
 
+export interface SavedObjectToItemOptions {
+  /**
+   * attributes to include in the output item
+   */
+  allowedAttributes?: string[];
+  /**
+   * references to include in the output item
+   */
+  allowedReferences?: string[];
+}
+
 export function savedObjectToItem(
   savedObject: SavedObject<DashboardSavedObjectAttributes>,
   partial: false,
-  allowedAttributes?: string[]
+  opts?: SavedObjectToItemOptions
 ): SavedObjectToItemReturn<DashboardItem>;
 
 export function savedObjectToItem(
   savedObject: PartialSavedObject<DashboardSavedObjectAttributes>,
   partial: true,
-  allowedAttributes?: string[]
+  opts?: SavedObjectToItemOptions
 ): SavedObjectToItemReturn<PartialDashboardItem>;
 
 export function savedObjectToItem(
   savedObject:
     | SavedObject<DashboardSavedObjectAttributes>
     | PartialSavedObject<DashboardSavedObjectAttributes>,
-  partial: boolean,
-  allowedAttributes?: string[]
+  partial: boolean /* partial arg is used to enforce the correct savedObject type */,
+  { allowedAttributes, allowedReferences }: SavedObjectToItemOptions = {}
 ): SavedObjectToItemReturn<DashboardItem | PartialDashboardItem> {
   const {
     id,
@@ -342,6 +353,12 @@ export function savedObjectToItem(
     const attributesOut = allowedAttributes
       ? pick(dashboardAttributesOut(attributes), allowedAttributes)
       : dashboardAttributesOut(attributes);
+
+    // if includeReferences is provided, only include references of those types
+    const referencesOut = allowedReferences
+      ? references?.filter((reference) => allowedReferences.includes(reference.type))
+      : references;
+
     return {
       item: {
         id,
@@ -353,7 +370,7 @@ export function savedObjectToItem(
         attributes: attributesOut,
         error,
         namespaces,
-        references,
+        references: referencesOut,
         version,
         managed,
       },

x-pack/test/api_integration/apis/content_management/dashboard_search.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+import { sampleDashboard } from './helpers';
+
+export default function ({ getService }: FtrProviderContext) {
+  const kibanaServer = getService('kibanaServer');
+  const supertest = getService('supertest');
+
+  describe('search dashboards', function () {
+    const createPayload = {
+      ...sampleDashboard,
+      options: {
+        ...sampleDashboard.options,
+        references: [
+          {
+            type: 'tag',
+            id: 'tag1',
+            name: 'tag-ref-tag1',
+          },
+          {
+            type: 'index-pattern',
+            id: 'index-pattern1',
+            name: 'index-pattern-ref-index-pattern1',
+          },
+        ],
+      },
+    };
+    before(async () => {
+      await kibanaServer.savedObjects.clean({
+        types: ['dashboard'],
+      });
+
+      await supertest
+        .post('/api/content_management/rpc/create')
+        .set('kbn-xsrf', 'true')
+        .send(createPayload)
+        .expect(200);
+    });
+
+    it('can specify references to return', async () => {
+      const searchPayload = {
+        contentTypeId: 'dashboard',
+        version: 3,
+        query: {},
+        options: {},
+      };
+
+      {
+        const { body } = await supertest
+          .post('/api/content_management/rpc/search')
+          .set('kbn-xsrf', 'true')
+          .send(searchPayload)
+          .expect(200);
+
+        expect(body.result.result.hits[0].references).to.eql(createPayload.options.references);
+      }
+
+      {
+        const { body } = await supertest
+          .post('/api/content_management/rpc/search')
+          .set('kbn-xsrf', 'true')
+          .send({ ...searchPayload, options: { includeReferences: ['tag'] } })
+          .expect(200);
+
+        expect(body.result.result.hits[0].references).to.eql([createPayload.options.references[0]]);
+      }
+    });
+  });
+}

x-pack/test/api_integration/apis/content_management/index.ts

@@ -12,5 +12,6 @@ export default function ({ loadTestFile, getService }: FtrProviderContext) {
     loadTestFile(require.resolve('./created_by'));
     loadTestFile(require.resolve('./updated_by'));
     loadTestFile(require.resolve('./favorites'));
+    loadTestFile(require.resolve('./dashboard_search'));
   });
 }