[Fleet] Add cache-control headers to key /epm endpoints in Fleet API (#130921)

* Add cache-control header to /categories endpoint

* Add cache-control header + includeInstallStatus parameter to /packages endpoint

* Add cache-control header parameter to filepath endpoint

* Fix installation status type

* Fix getLimitedPackages call

* Fix cypress tests

* Fix checks + integration test

* Swap includeInstallStatus -> excludeInstallStatus query parameter

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -184,7 +184,15 @@ paths:
               schema:
                 $ref: '#/components/schemas/get_packages_response'
       operationId: list-all-packages
-    parameters: []
+    parameters:
+      - in: query
+        name: includeInstallStatus
+        schema:
+          type: boolean
+          default: false
+        description: >-
+          Whether to include the install status of each package. Defaults to
+          false to allow for caching of package requests.
   /epm/packages/_bulk:
     post:
       summary: Packages - Bulk install

x-pack/plugins/fleet/common/openapi/paths/epm@packages.yaml

@@ -9,4 +9,10 @@ get:
           schema:
             $ref: ../components/schemas/get_packages_response.yaml
   operationId: list-all-packages
-parameters: []
+parameters:
+  - in: query
+    name: includeInstallStatus
+    schema:
+      type: boolean
+      default: false
+    description: Whether to include the install status of each package. Defaults to false to allow for caching of package requests.

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -427,12 +427,17 @@ export interface PackageUsageStats {
 }
 
 export type Installable<T> =
+  | InstallStatusExcluded<T>
   | InstalledRegistry<T>
   | Installing<T>
   | NotInstalled<T>
   | InstallFailed<T>
   | InstalledBundled<T>;
 
+export type InstallStatusExcluded<T = {}> = T & {
+  status: undefined;
+};
+
 export type InstalledRegistry<T = {}> = T & {
   status: InstallationStatus['Installed'];
   savedObject: SavedObject<Installation>;

x-pack/plugins/fleet/common/types/rest_spec/epm.ts

@@ -32,6 +32,7 @@ export interface GetPackagesRequest {
   query: {
     category?: string;
     experimental?: boolean;
+    excludeInstallStatus?: boolean;
   };
 }
 

x-pack/plugins/fleet/cypress/integration/integrations_real.spec.ts

@@ -50,7 +50,19 @@ describe('Add Integration - Real API', () => {
   });
 
   function addAndVerifyIntegration() {
-    cy.intercept('GET', '/api/fleet/epm/packages?*').as('packages');
+    cy.intercept(
+      '/api/fleet/epm/packages?*',
+      {
+        middleware: true,
+      },
+      (req) => {
+        req.on('before:response', (res) => {
+          // force all API responses to not be cached
+          res.headers['cache-control'] = 'no-store';
+        });
+      }
+    ).as('packages');
+
     navigateTo(INTEGRATIONS);
     cy.wait('@packages');
     cy.get('.euiLoadingSpinner').should('not.exist');
@@ -75,7 +87,20 @@ describe('Add Integration - Real API', () => {
         .map((policy: any) => policy.id);
 
       cy.visit(`/app/fleet/policies/${agentPolicyId}`);
-      cy.intercept('GET', '/api/fleet/epm/packages?*').as('packages');
+
+      cy.intercept(
+        '/api/fleet/epm/packages?*',
+        {
+          middleware: true,
+        },
+        (req) => {
+          req.on('before:response', (res) => {
+            // force all API responses to not be cached
+            res.headers['cache-control'] = 'no-store';
+          });
+        }
+      ).as('packages');
+
       cy.getBySel(ADD_PACKAGE_POLICY_BTN).click();
       cy.wait('@packages');
       cy.get('.euiLoadingSpinner').should('not.exist');

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/home/available_packages.tsx

@@ -214,6 +214,7 @@ export const AvailablePackages: React.FC = memo(() => {
     error: eprPackageLoadingError,
   } = useGetPackages({
     category: '',
+    excludeInstallStatus: true,
   });
   const eprIntegrationList = useMemo(
     () => packageListToIntegrationsList(eprPackages?.items || []),

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -10,7 +10,7 @@ import path from 'path';
 import type { TypeOf } from '@kbn/config-schema';
 import mime from 'mime-types';
 import semverValid from 'semver/functions/valid';
-import type { ResponseHeaders, KnownHeaders } from '@kbn/core/server';
+import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/core/server';
 
 import type {
   GetInfoResponse,
@@ -62,6 +62,10 @@ import { getAsset } from '../../services/epm/archive/storage';
 import { getPackageUsageStats } from '../../services/epm/packages/get';
 import { updatePackage } from '../../services/epm/packages/update';
 
+const CACHE_CONTROL_10_MINUTES_HEADER: HttpResponseOptions['headers'] = {
+  'cache-control': 'max-age=600',
+};
+
 export const getCategoriesHandler: FleetRequestHandler<
   undefined,
   TypeOf<typeof GetCategoriesRequestSchema.query>
@@ -72,7 +76,7 @@ export const getCategoriesHandler: FleetRequestHandler<
       items: res,
       response: res,
     };
-    return response.ok({ body });
+    return response.ok({ body, headers: { ...CACHE_CONTROL_10_MINUTES_HEADER } });
   } catch (error) {
     return defaultIngestErrorHandler({ error, response });
   }
@@ -94,6 +98,9 @@ export const getListHandler: FleetRequestHandler<
     };
     return response.ok({
       body,
+      // Only cache responses where the installation status is excluded, otherwise the request
+      // needs up-to-date information on whether the package is installed so we can't cache it
+      headers: request.query.excludeInstallStatus ? { ...CACHE_CONTROL_10_MINUTES_HEADER } : {},
     });
   } catch (error) {
     return defaultIngestErrorHandler({ error, response });
@@ -164,13 +171,13 @@ export const getFileHandler: FleetRequestHandler<
         body: buffer,
         statusCode: 200,
         headers: {
-          'cache-control': 'max-age=10, public',
+          ...CACHE_CONTROL_10_MINUTES_HEADER,
           'content-type': contentType,
         },
       });
     } else {
       const registryResponse = await getFile(pkgName, pkgVersion, filePath);
-      const headersToProxy: KnownHeaders[] = ['content-type', 'cache-control'];
+      const headersToProxy: KnownHeaders[] = ['content-type'];
       const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => {
         const value = registryResponse.headers.get(knownHeader);
         if (value !== null) {
@@ -182,7 +189,7 @@ export const getFileHandler: FleetRequestHandler<
       return response.custom({
         body: registryResponse.body,
         statusCode: registryResponse.status,
-        headers: proxiedHeaders,
+        headers: { ...CACHE_CONTROL_10_MINUTES_HEADER, ...proxiedHeaders },
       });
     }
   } catch (error) {

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -45,9 +45,10 @@ export async function getCategories(options: GetCategoriesRequest['query']) {
 export async function getPackages(
   options: {
     savedObjectsClient: SavedObjectsClientContract;
+    excludeInstallStatus?: boolean;
   } & Registry.SearchParams
 ) {
-  const { savedObjectsClient, experimental, category } = options;
+  const { savedObjectsClient, experimental, category, excludeInstallStatus = false } = options;
   const registryItems = await Registry.fetchList({ category, experimental }).then((items) => {
     return items.map((item) =>
       Object.assign({}, item, { title: item.title || nameAsTitle(item.name) }, { id: item.name })
@@ -63,7 +64,23 @@ export async function getPackages(
       )
     )
     .sort(sortByName);
-  return packageList;
+
+  if (!excludeInstallStatus) {
+    return packageList;
+  }
+
+  // Exclude the `installStatus` value if the `excludeInstallStatus` query parameter is set to true
+  // to better facilitate response caching
+  const packageListWithoutStatus = packageList.map((pkg) => {
+    const newPkg = {
+      ...pkg,
+      status: undefined,
+    };
+
+    return newPkg;
+  });
+
+  return packageListWithoutStatus;
 }
 
 // Get package names for packages which cannot have more than one package policy on an agent policy
@@ -71,7 +88,10 @@ export async function getLimitedPackages(options: {
   savedObjectsClient: SavedObjectsClientContract;
 }): Promise<string[]> {
   const { savedObjectsClient } = options;
-  const allPackages = await getPackages({ savedObjectsClient, experimental: true });
+  const allPackages = await getPackages({
+    savedObjectsClient,
+    experimental: true,
+  });
   const installedPackages = allPackages.filter(
     (pkg) => pkg.status === installationStatuses.Installed
   );

x-pack/plugins/fleet/server/types/rest_spec/epm.ts

@@ -18,6 +18,7 @@ export const GetPackagesRequestSchema = {
   query: schema.object({
     category: schema.maybe(schema.string()),
     experimental: schema.maybe(schema.boolean()),
+    excludeInstallStatus: schema.maybe(schema.boolean({ defaultValue: false })),
   }),
 };