[APM] Add pagination to source map API (#145959)

Closes https://github.com/elastic/kibana/issues/145884

Adds ability to paginate source map result via `page` and `perPage`.

The request `GET /api/apm/sourcemaps?page=2&perPage=10`, will return:

```json
{ "artifacts": [], "total": 0 }
```

x-pack/plugins/apm/server/routes/fleet/merge_package_policy_with_apm.ts

@@ -12,7 +12,10 @@ import {
   getPackagePolicyWithAgentConfigurations,
   PackagePolicy,
 } from './register_fleet_policy_callbacks';
-import { getPackagePolicyWithSourceMap, listArtifacts } from './source_maps';
+import {
+  getPackagePolicyWithSourceMap,
+  listSourceMapArtifacts,
+} from './source_maps';
 
 export async function mergePackagePolicyWithApm({
   packagePolicy,
@@ -24,7 +27,7 @@ export async function mergePackagePolicyWithApm({
   fleetPluginStart: NonNullable<APMPluginStartDependencies['fleet']>;
 }) {
   const agentConfigurations = await listConfigurations(internalESClient);
-  const artifacts = await listArtifacts({ fleetPluginStart });
+  const { artifacts } = await listSourceMapArtifacts({ fleetPluginStart });
   return getPackagePolicyWithAgentConfigurations(
     getPackagePolicyWithSourceMap({ packagePolicy, artifacts }),
     agentConfigurations

x-pack/plugins/apm/server/routes/fleet/source_maps.ts

@@ -32,37 +32,44 @@ export type FleetPluginStart = NonNullable<APMPluginStartDependencies['fleet']>;
 
 const doUnzip = promisify(unzip);
 
-function decodeArtifacts(artifacts: Artifact[]): Promise<ArtifactSourceMap[]> {
-  return Promise.all(
-    artifacts.map(async (artifact) => {
-      const body = await doUnzip(Buffer.from(artifact.body, 'base64'));
-      return {
-        ...artifact,
-        body: JSON.parse(body.toString()) as ApmArtifactBody,
-      };
-    })
-  );
+async function unzipArtifactBody(
+  artifact: Artifact
+): Promise<ArtifactSourceMap> {
+  const body = await doUnzip(Buffer.from(artifact.body, 'base64'));
+
+  return {
+    ...artifact,
+    body: JSON.parse(body.toString()) as ApmArtifactBody,
+  };
 }
 
 function getApmArtifactClient(fleetPluginStart: FleetPluginStart) {
   return fleetPluginStart.createArtifactsClient('apm');
 }
 
-export async function listArtifacts({
+export async function listSourceMapArtifacts({
   fleetPluginStart,
+  perPage = 20,
+  page = 1,
 }: {
   fleetPluginStart: FleetPluginStart;
+  perPage?: number;
+  page?: number;
 }) {
   const apmArtifactClient = getApmArtifactClient(fleetPluginStart);
-  const fleetArtifactsResponse = await apmArtifactClient.listArtifacts({
+  const artifactsResponse = await apmArtifactClient.listArtifacts({
     kuery: 'type: sourcemap',
-    perPage: 20,
-    page: 1,
+    perPage,
+    page,
     sortOrder: 'desc',
     sortField: 'created',
   });
 
-  return decodeArtifacts(fleetArtifactsResponse.items);
+  const artifacts = await Promise.all(
+    artifactsResponse.items.map(unzipArtifactBody)
+  );
+
+  return { artifacts, total: artifactsResponse.total };
 }
 
 export async function createApmArtifact({
@@ -141,8 +148,7 @@ export async function updateSourceMapsOnFleetPolicies({
   savedObjectsClient: SavedObjectsClientContract;
   elasticsearchClient: ElasticsearchClient;
 }) {
-  const artifacts = await listArtifacts({ fleetPluginStart });
-
+  const { artifacts } = await listSourceMapArtifacts({ fleetPluginStart });
   const apmFleetPolicies = await getApmPackagePolicies({
     core,
     fleetPluginStart,

x-pack/plugins/apm/server/routes/source_maps/route.ts

@@ -7,12 +7,12 @@
 import Boom from '@hapi/boom';
 import * as t from 'io-ts';
 import { SavedObjectsClientContract } from '@kbn/core/server';
-import { jsonRt } from '@kbn/io-ts-utils';
+import { jsonRt, toNumberRt } from '@kbn/io-ts-utils';
 import { Artifact } from '@kbn/fleet-plugin/server';
 import {
   createApmArtifact,
   deleteApmArtifact,
-  listArtifacts,
+  listSourceMapArtifacts,
   updateSourceMapsOnFleetPolicies,
   getCleanedBundleFilePath,
   ArtifactSourceMap,
@@ -40,14 +40,28 @@ export type SourceMap = t.TypeOf<typeof sourceMapRt>;
 const listSourceMapRoute = createApmServerRoute({
   endpoint: 'GET /api/apm/sourcemaps',
   options: { tags: ['access:apm'] },
+  params: t.partial({
+    query: t.partial({
+      page: toNumberRt,
+      perPage: toNumberRt,
+    }),
+  }),
   async handler({
+    params,
     plugins,
-  }): Promise<{ artifacts: ArtifactSourceMap[] } | undefined> {
+  }): Promise<{ artifacts: ArtifactSourceMap[]; total: number } | undefined> {
+    const { page, perPage } = params.query;
+
     try {
       const fleetPluginStart = await plugins.fleet?.start();
       if (fleetPluginStart) {
-        const artifacts = await listArtifacts({ fleetPluginStart });
-        return { artifacts };
+        const { artifacts, total } = await listSourceMapArtifacts({
+          fleetPluginStart,
+          page,
+          perPage,
+        });
+
+        return { artifacts, total };
       }
     } catch (e) {
       throw Boom.internal(

x-pack/test/apm_api_integration/tests/sourcemaps/sourcemaps.ts

@@ -7,7 +7,7 @@
 import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api';
 import type { SourceMap } from '@kbn/apm-plugin/server/routes/source_maps/route';
 import expect from '@kbn/expect';
-import { times } from 'lodash';
+import { first, last, times } from 'lodash';
 import { FtrProviderContext } from '../../common/ftr_provider_context';
 
 export default function ApiTest({ getService }: FtrProviderContext) {
@@ -47,11 +47,14 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     });
   }
 
-  async function listSourcemaps() {
+  async function listSourcemaps({ page, perPage }: { page?: number; perPage?: number } = {}) {
+    const query = page && perPage ? { page, perPage } : {};
+
     const response = await apmApiClient.readUser({
       endpoint: 'GET /api/apm/sourcemaps',
+      params: { query },
     });
-    return response.body.artifacts;
+    return response.body;
   }
 
   registry.when('source maps', { config: 'basic', archives: [] }, () => {
@@ -66,7 +69,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
       it('can upload a source map', async () => {
         resp = await uploadSourcemap({
-          serviceName: 'foo',
+          serviceName: 'my_service',
           serviceVersion: '1.0.0',
           bundleFilePath: 'bar',
           sourcemap: {
@@ -79,13 +82,13 @@ export default function ApiTest({ getService }: FtrProviderContext) {
       });
     });
 
-    describe('list source maps', () => {
+    describe('list source maps', async () => {
       const uploadedSourcemapIds: string[] = [];
       before(async () => {
-        const sourcemapCount = times(2);
+        const sourcemapCount = times(15);
         for (const i of sourcemapCount) {
           const sourcemap = await uploadSourcemap({
-            serviceName: 'foo',
+            serviceName: 'my_service',
             serviceVersion: `1.0.${i}`,
             bundleFilePath: 'bar',
             sourcemap: {
@@ -95,7 +98,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
             },
           });
           uploadedSourcemapIds.push(sourcemap.id);
-          await sleep(100);
         }
       });
 
@@ -103,17 +105,41 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         await Promise.all(uploadedSourcemapIds.map((id) => deleteSourcemap(id)));
       });
 
+      describe('pagination', () => {
+        it('can retrieve the first page', async () => {
+          const firstPageItems = await listSourcemaps({ page: 1, perPage: 5 });
+          expect(first(firstPageItems.artifacts)?.identifier).to.eql('my_service-1.0.14');
+          expect(last(firstPageItems.artifacts)?.identifier).to.eql('my_service-1.0.10');
+          expect(firstPageItems.artifacts.length).to.be(5);
+          expect(firstPageItems.total).to.be(15);
+        });
+
+        it('can retrieve the second page', async () => {
+          const secondPageItems = await listSourcemaps({ page: 2, perPage: 5 });
+          expect(first(secondPageItems.artifacts)?.identifier).to.eql('my_service-1.0.9');
+          expect(last(secondPageItems.artifacts)?.identifier).to.eql('my_service-1.0.5');
+          expect(secondPageItems.artifacts.length).to.be(5);
+          expect(secondPageItems.total).to.be(15);
+        });
+
+        it('can retrieve the third page', async () => {
+          const thirdPageItems = await listSourcemaps({ page: 3, perPage: 5 });
+          expect(first(thirdPageItems.artifacts)?.identifier).to.eql('my_service-1.0.4');
+          expect(last(thirdPageItems.artifacts)?.identifier).to.eql('my_service-1.0.0');
+          expect(thirdPageItems.artifacts.length).to.be(5);
+          expect(thirdPageItems.total).to.be(15);
+        });
+      });
+
       it('can list source maps', async () => {
         const sourcemaps = await listSourcemaps();
-        expect(sourcemaps).to.not.empty();
+        expect(sourcemaps.artifacts.length).to.be(15);
+        expect(sourcemaps.total).to.be(15);
       });
 
       it('returns newest source maps first', async () => {
-        const response = await apmApiClient.readUser({
-          endpoint: 'GET /api/apm/sourcemaps',
-        });
-
-        const timestamps = response.body.artifacts.map((a) => new Date(a.created).getTime());
+        const { artifacts } = await listSourcemaps();
+        const timestamps = artifacts.map((a) => new Date(a.created).getTime());
         expect(timestamps[0]).to.be.greaterThan(timestamps[1]);
       });
     });
@@ -121,7 +147,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
     describe('delete source maps', () => {
       it('can delete a source map', async () => {
         const sourcemap = await uploadSourcemap({
-          serviceName: 'foo',
+          serviceName: 'my_service',
           serviceVersion: '1.0.0',
           bundleFilePath: 'bar',
           sourcemap: {
@@ -132,13 +158,10 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         });
 
         await deleteSourcemap(sourcemap.id);
-        const sourcemaps = await listSourcemaps();
-        expect(sourcemaps).to.be.empty();
+        const { artifacts, total } = await listSourcemaps();
+        expect(artifacts).to.be.empty();
+        expect(total).to.be(0);
       });
     });
   });
 }
-
-function sleep(ms: number) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}