[Logs onboarding] Expose agent latest available version from fleet (#166811)

Relates to https://github.com/elastic/kibana/issues/165657.

In https://github.com/elastic/kibana/pull/166150 a method that returns
the latest available version was introduced in fleet.

We would like to use this method as a consumers of fleet plugin
([observability_onboarding](https://github.com/elastic/kibana/tree/main/x-pack/plugins/observability_onboarding)).
Additionally we would like to decide from outside fleet wether we want
to include currentVersion or not, this is specially useful for us since
we download the elastic agent executable from
https://artifacts.elastic.co/downloads/beats/elastic-agent/ where
snapshots are not available.

x-pack/plugins/fleet/server/routes/agent/handlers.ts

@@ -354,7 +354,7 @@ function isStringArray(arr: unknown | string[]): arr is string[] {
 
 export const getAvailableVersionsHandler: RequestHandler = async (context, request, response) => {
   try {
-    const availableVersions = await AgentService.getAvailableVersions();
+    const availableVersions = await AgentService.getAvailableVersions({});
     const body: GetAvailableVersionsResponse = { items: availableVersions };
     return response.ok({ body });
   } catch (error) {

x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts

@@ -297,9 +297,12 @@ export const getFullAgentPolicy: FleetRequestHandler<
 
   if (request.query.kubernetes === true) {
     try {
+      const agentVersion =
+        await fleetContext.agentClient.asInternalUser.getLatestAgentAvailableVersion();
       const fullAgentConfigMap = await agentPolicyService.getFullAgentConfigMap(
         soClient,
         request.params.agentPolicyId,
+        agentVersion,
         { standalone: request.query.standalone === true }
       );
       if (fullAgentConfigMap) {
@@ -356,9 +359,12 @@ export const downloadFullAgentPolicy: FleetRequestHandler<
 
   if (request.query.kubernetes === true) {
     try {
+      const agentVersion =
+        await fleetContext.agentClient.asInternalUser.getLatestAgentAvailableVersion();
       const fullAgentConfigMap = await agentPolicyService.getFullAgentConfigMap(
         soClient,
         request.params.agentPolicyId,
+        agentVersion,
         { standalone: request.query.standalone === true }
       );
       if (fullAgentConfigMap) {
@@ -411,10 +417,18 @@ export const getK8sManifest: FleetRequestHandler<
   undefined,
   TypeOf<typeof GetK8sManifestRequestSchema.query>
 > = async (context, request, response) => {
+  const fleetContext = await context.fleet;
+
   try {
     const fleetServer = request.query.fleetServer ?? '';
     const token = request.query.enrolToken ?? '';
-    const fullAgentManifest = await agentPolicyService.getFullAgentManifest(fleetServer, token);
+    const agentVersion =
+      await fleetContext.agentClient.asInternalUser.getLatestAgentAvailableVersion();
+    const fullAgentManifest = await agentPolicyService.getFullAgentManifest(
+      fleetServer,
+      token,
+      agentVersion
+    );
     if (fullAgentManifest) {
       const body: GetFullAgentManifestResponse = {
         item: fullAgentManifest,
@@ -437,10 +451,18 @@ export const downloadK8sManifest: FleetRequestHandler<
   undefined,
   TypeOf<typeof GetK8sManifestRequestSchema.query>
 > = async (context, request, response) => {
+  const fleetContext = await context.fleet;
+
   try {
     const fleetServer = request.query.fleetServer ?? '';
     const token = request.query.enrolToken ?? '';
-    const fullAgentManifest = await agentPolicyService.getFullAgentManifest(fleetServer, token);
+    const agentVersion =
+      await fleetContext.agentClient.asInternalUser.getLatestAgentAvailableVersion();
+    const fullAgentManifest = await agentPolicyService.getFullAgentManifest(
+      fleetServer,
+      token,
+      agentVersion
+    );
     if (fullAgentManifest) {
       const body = fullAgentManifest;
       const headers: ResponseHeaders = {

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -83,7 +83,7 @@ import {
 } from './elastic_agent_manifest';
 
 import { bulkInstallPackages } from './epm/packages';
-import { getAgentsByKuery, getLatestAvailableVersion } from './agents';
+import { getAgentsByKuery } from './agents';
 import { packagePolicyService } from './package_policy';
 import { incrementPackagePolicyCopyName } from './package_policies';
 import { outputService } from './output';
@@ -1029,6 +1029,7 @@ class AgentPolicyService {
   public async getFullAgentConfigMap(
     soClient: SavedObjectsClientContract,
     id: string,
+    agentVersion: string,
     options?: { standalone: boolean }
   ): Promise<string | null> {
     const fullAgentPolicy = await getFullAgentPolicy(soClient, id, options);
@@ -1048,7 +1049,6 @@ class AgentPolicyService {
         },
       };
 
-      const agentVersion = await getLatestAvailableVersion();
       const configMapYaml = fullAgentConfigMapToYaml(fullAgentConfigMap, safeDump);
       const updateManifestVersion = elasticAgentStandaloneManifest.replace('VERSION', agentVersion);
       const fixedAgentYML = configMapYaml.replace('agent.yml:', 'agent.yml: |-');
@@ -1060,9 +1060,9 @@ class AgentPolicyService {
 
   public async getFullAgentManifest(
     fleetServer: string,
-    enrolToken: string
+    enrolToken: string,
+    agentVersion: string
   ): Promise<string | null> {
-    const agentVersion = await getLatestAvailableVersion();
     const updateManifestVersion = elasticAgentManagedManifest.replace('VERSION', agentVersion);
     let updateManifest = updateManifestVersion;
     if (fleetServer !== '') {

x-pack/plugins/fleet/server/services/agents/agent_service.mock.ts

@@ -12,6 +12,7 @@ const createClientMock = (): jest.Mocked<AgentClient> => ({
   getAgentStatusById: jest.fn(),
   getAgentStatusForAgentPolicy: jest.fn(),
   listAgents: jest.fn(),
+  getLatestAgentAvailableVersion: jest.fn(),
 });
 
 const createServiceMock = (): jest.Mocked<AgentService> => ({

x-pack/plugins/fleet/server/services/agents/agent_service.test.ts

@@ -8,6 +8,7 @@
 jest.mock('../security');
 jest.mock('./crud');
 jest.mock('./status');
+jest.mock('./versions');
 
 import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
 import {
@@ -25,12 +26,14 @@ import type { AgentClient } from './agent_service';
 import { AgentServiceImpl } from './agent_service';
 import { getAgentsByKuery, getAgentById } from './crud';
 import { getAgentStatusById, getAgentStatusForAgentPolicy } from './status';
+import { getLatestAvailableVersion } from './versions';
 
 const mockGetAuthzFromRequest = getAuthzFromRequest as jest.Mock<Promise<FleetAuthz>>;
 const mockGetAgentsByKuery = getAgentsByKuery as jest.Mock;
 const mockGetAgentById = getAgentById as jest.Mock;
 const mockGetAgentStatusById = getAgentStatusById as jest.Mock;
 const mockGetAgentStatusForAgentPolicy = getAgentStatusForAgentPolicy as jest.Mock;
+const mockGetLatestAvailableVersion = getLatestAvailableVersion as jest.Mock;
 
 describe('AgentService', () => {
   beforeEach(() => {
@@ -100,6 +103,14 @@ describe('AgentService', () => {
           )
         );
       });
+
+      it('rejects on getLatestAgentAvailableVersion', async () => {
+        await expect(agentClient.getLatestAgentAvailableVersion()).rejects.toThrowError(
+          new FleetUnauthorizedError(
+            `User does not have adequate permissions to access Fleet agents.`
+          )
+        );
+      });
     });
 
     describe('with required privilege', () => {
@@ -188,4 +199,12 @@ function expectApisToCallServicesSuccessfully(
       'foo-filter'
     );
   });
+
+  test('client.getLatestAgentAvailableVersion calls getLatestAvailableVersion and returns results', async () => {
+    mockGetLatestAvailableVersion.mockResolvedValue('getLatestAvailableVersion success');
+    await expect(agentClient.getLatestAgentAvailableVersion()).resolves.toEqual(
+      'getLatestAvailableVersion success'
+    );
+    expect(mockGetLatestAvailableVersion).toHaveBeenCalledTimes(1);
+  });
 }

x-pack/plugins/fleet/server/services/agents/agent_service.ts

@@ -22,6 +22,7 @@ import { FleetUnauthorizedError } from '../../errors';
 
 import { getAgentsByKuery, getAgentById } from './crud';
 import { getAgentStatusById, getAgentStatusForAgentPolicy } from './status';
+import { getLatestAvailableVersion } from './versions';
 
 /**
  * A service for interacting with Agent data. See {@link AgentClient} for more information.
@@ -78,6 +79,11 @@ export interface AgentClient {
     page: number;
     perPage: number;
   }>;
+
+  /**
+   * Return the latest agent available version
+   */
+  getLatestAgentAvailableVersion(includeCurrentVersion?: boolean): Promise<string>;
 }
 
 /**
@@ -119,6 +125,11 @@ class AgentClientImpl implements AgentClient {
     );
   }
 
+  public async getLatestAgentAvailableVersion(includeCurrentVersion?: boolean) {
+    await this.#runPreflight();
+    return getLatestAvailableVersion(includeCurrentVersion);
+  }
+
   #runPreflight = async () => {
     if (this.preflightCheck) {
       return this.preflightCheck();

x-pack/plugins/fleet/server/services/agents/versions.test.ts

@@ -30,7 +30,7 @@ describe('getAvailableVersions', () => {
     mockKibanaVersion = '300.0.0';
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
 
-    const res = await getAvailableVersions(false);
+    const res = await getAvailableVersions({ cached: false });
 
     expect(res).toEqual(['300.0.0', '8.1.0', '8.0.0', '7.17.0']);
   });
@@ -39,7 +39,7 @@ describe('getAvailableVersions', () => {
     mockKibanaVersion = '300.0.0-SNAPSHOT';
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
 
-    const res = await getAvailableVersions(false);
+    const res = await getAvailableVersions({ cached: false });
     expect(res).toEqual(['300.0.0-SNAPSHOT', '8.1.0', '8.0.0', '7.17.0']);
   });
 
@@ -52,7 +52,16 @@ describe('getAvailableVersions', () => {
     };
     mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
 
-    const res = await getAvailableVersions(false);
+    const res = await getAvailableVersions({ cached: false });
+
+    expect(res).toEqual(['8.1.0', '8.0.0', '7.17.0']);
+  });
+
+  it('should not include the current version if includeCurrentVersion = false', async () => {
+    mockKibanaVersion = '300.0.0-SNAPSHOT';
+    mockedReadFile.mockResolvedValue(`["8.1.0", "8.0.0", "7.17.0", "7.16.0"]`);
+
+    const res = await getAvailableVersions({ cached: false, includeCurrentVersion: false });
 
     expect(res).toEqual(['8.1.0', '8.0.0', '7.17.0']);
   });

x-pack/plugins/fleet/server/services/agents/versions.ts

@@ -21,12 +21,21 @@ const AGENT_VERSION_BUILD_FILE = 'x-pack/plugins/fleet/target/agent_versions_lis
 
 let availableVersions: string[] | undefined;
 
-export const getLatestAvailableVersion = async (): Promise<string> => {
-  const versions = await getAvailableVersions();
+export const getLatestAvailableVersion = async (
+  includeCurrentVersion?: boolean
+): Promise<string> => {
+  const versions = await getAvailableVersions({ includeCurrentVersion });
+
   return versions[0];
 };
 
-export const getAvailableVersions = async (cached = true): Promise<string[]> => {
+export const getAvailableVersions = async ({
+  cached = true,
+  includeCurrentVersion,
+}: {
+  cached?: boolean;
+  includeCurrentVersion?: boolean;
+}): Promise<string[]> => {
   // Use cached value to avoid reading from disk each time
   if (cached && availableVersions) {
     return availableVersions;
@@ -51,7 +60,10 @@ export const getAvailableVersions = async (cached = true): Promise<string[]> =>
       .sort((a: any, b: any) => (semverGt(a, b) ? -1 : 1));
     versionsToDisplay = uniq(versions) as string[];
 
-    if (!config?.internal?.onlyAllowAgentUpgradeToKnownVersions) {
+    const appendCurrentVersion =
+      includeCurrentVersion ?? !config?.internal?.onlyAllowAgentUpgradeToKnownVersions;
+
+    if (appendCurrentVersion) {
       // Add current version if not already present
       const hasCurrentVersion = versionsToDisplay.some((v) => v === kibanaVersion);