[asset manager] get assets method (#172051)

## Summary

Closes https://github.com/elastic/kibana/issues/169444

Add `/assets` endpoint with corresponding public/server client methods.
The method currently returns `host` and `service` asset type.

### Testing
- connect to cluster with apm and metrics data
- hit `/api/asset-manager/assets?from=now-30s&to=now&stringFilters=...`
- response contains both service and host assets, sorted by desc
timestamp

---------

Co-authored-by: Milton Hultgren <miltonhultgren@gmail.com>

x-pack/plugins/asset_manager/common/types_api.ts

@@ -170,6 +170,7 @@ export const assetFiltersSingleKindRT = rt.exact(
     type: rt.union([assetTypeRT, rt.array(assetTypeRT)]),
     ean: rt.union([rt.string, rt.array(rt.string)]),
     id: rt.string,
+    parentEan: rt.string,
     ['cloud.provider']: rt.string,
     ['cloud.region']: rt.string,
     ['orchestrator.cluster.name']: rt.string,
@@ -178,9 +179,10 @@ export const assetFiltersSingleKindRT = rt.exact(
 
 export type SingleKindAssetFilters = rt.TypeOf<typeof assetFiltersSingleKindRT>;
 
+const supportedKindRT = rt.union([rt.literal('host'), rt.literal('service')]);
 export const assetFiltersRT = rt.intersection([
   assetFiltersSingleKindRT,
-  rt.partial({ kind: rt.union([assetKindRT, rt.array(assetKindRT)]) }),
+  rt.partial({ kind: rt.union([supportedKindRT, rt.array(supportedKindRT)]) }),
 ]);
 
 export type AssetFilters = rt.TypeOf<typeof assetFiltersRT>;
@@ -248,7 +250,6 @@ export const getServiceAssetsQueryOptionsRT = rt.intersection([
     from: assetDateRT,
     to: assetDateRT,
     size: sizeRT,
-    parent: rt.string,
     stringFilters: rt.string,
     filters: assetFiltersSingleKindRT,
   }),
@@ -277,3 +278,21 @@ export const getPodAssetsResponseRT = rt.type({
   pods: rt.array(assetRT),
 });
 export type GetPodAssetsResponse = rt.TypeOf<typeof getPodAssetsResponseRT>;
+
+/**
+ * Assets
+ */
+export const getAssetsQueryOptionsRT = rt.intersection([
+  rt.strict({ from: assetDateRT }),
+  rt.partial({
+    to: assetDateRT,
+    size: sizeRT,
+    stringFilters: rt.string,
+    filters: assetFiltersRT,
+  }),
+]);
+export type GetAssetsQueryOptions = rt.TypeOf<typeof getAssetsQueryOptionsRT>;
+export const getAssetsResponseRT = rt.type({
+  assets: rt.array(assetRT),
+});
+export type GetAssetsResponse = rt.TypeOf<typeof getAssetsResponseRT>;

x-pack/plugins/asset_manager/common/types_client.ts

@@ -20,8 +20,5 @@ export interface SharedAssetsOptionsPublic<F = AssetFilters> {
 export type GetHostsOptionsPublic = SharedAssetsOptionsPublic<SingleKindAssetFilters>;
 export type GetContainersOptionsPublic = SharedAssetsOptionsPublic<SingleKindAssetFilters>;
 export type GetPodsOptionsPublic = SharedAssetsOptionsPublic<SingleKindAssetFilters>;
-
-export interface GetServicesOptionsPublic
-  extends SharedAssetsOptionsPublic<SingleKindAssetFilters> {
-  parent?: string;
-}
+export type GetServicesOptionsPublic = SharedAssetsOptionsPublic<SingleKindAssetFilters>;
+export type GetAssetsOptionsPublic = SharedAssetsOptionsPublic<AssetFilters>;

x-pack/plugins/asset_manager/public/lib/public_assets_client.test.ts

@@ -110,7 +110,7 @@ describe('Public assets client', () => {
 
     it('should include provided filters, but in string form', async () => {
       const client = new PublicAssetsClient(http);
-      const filters = { id: '*id-1*' };
+      const filters = { id: '*id-1*', parentEan: 'container:123' };
       await client.getServices({ from: 'x', filters });
       expect(http.get).toBeCalledWith(routePaths.GET_SERVICES, {
         query: {
@@ -120,14 +120,6 @@ describe('Public assets client', () => {
       });
     });
 
-    it('should include specified "parent" parameter in http.get query', async () => {
-      const client = new PublicAssetsClient(http);
-      await client.getServices({ from: 'x', to: 'y', parent: 'container:123' });
-      expect(http.get).toBeCalledWith(routePaths.GET_SERVICES, {
-        query: { from: 'x', to: 'y', parent: 'container:123' },
-      });
-    });
-
     it('should return the direct results of http.get', async () => {
       const client = new PublicAssetsClient(http);
       http.get.mockResolvedValueOnce('my services');

x-pack/plugins/asset_manager/public/lib/public_assets_client.ts

@@ -11,14 +11,22 @@ import {
   GetHostsOptionsPublic,
   GetServicesOptionsPublic,
   GetPodsOptionsPublic,
+  GetAssetsOptionsPublic,
 } from '../../common/types_client';
 import {
   GetContainerAssetsResponse,
   GetHostAssetsResponse,
   GetServiceAssetsResponse,
   GetPodAssetsResponse,
+  GetAssetsResponse,
 } from '../../common/types_api';
-import { GET_CONTAINERS, GET_HOSTS, GET_SERVICES, GET_PODS } from '../../common/constants_routes';
+import {
+  GET_CONTAINERS,
+  GET_HOSTS,
+  GET_SERVICES,
+  GET_PODS,
+  GET_ASSETS,
+} from '../../common/constants_routes';
 import { IPublicAssetsClient } from '../types';
 
 export class PublicAssetsClient implements IPublicAssetsClient {
@@ -71,4 +79,16 @@ export class PublicAssetsClient implements IPublicAssetsClient {
 
     return results;
   }
+
+  async getAssets(options: GetAssetsOptionsPublic) {
+    const { filters, ...otherOptions } = options;
+    const results = await this.http.get<GetAssetsResponse>(GET_ASSETS, {
+      query: {
+        stringFilters: JSON.stringify(filters),
+        ...otherOptions,
+      },
+    });
+
+    return results;
+  }
 }

x-pack/plugins/asset_manager/server/lib/accessors/hosts/get_hosts.test.ts

@@ -135,8 +135,8 @@ describe('getHosts', () => {
           },
         },
         {
-          term: {
-            'host.hostname': mockHostName,
+          terms: {
+            'host.hostname': [mockHostName],
           },
         },
       ])

x-pack/plugins/asset_manager/server/lib/accessors/hosts/get_hosts.ts

@@ -29,19 +29,22 @@ export async function getHosts(options: GetHostsOptionsInjected): Promise<{ host
   const filters: QueryDslQueryContainer[] = [];
 
   if (options.filters?.ean) {
-    const ean = Array.isArray(options.filters.ean) ? options.filters.ean[0] : options.filters.ean;
-    const { kind, id } = parseEan(ean);
+    const eans = Array.isArray(options.filters.ean) ? options.filters.ean : [options.filters.ean];
+    const hostnames = eans
+      .map(parseEan)
+      .filter(({ kind }) => kind === 'host')
+      .map(({ id }) => id);
 
     // if EAN filter isn't targeting a host asset, we don't need to do this query
-    if (kind !== 'host') {
+    if (hostnames.length === 0) {
       return {
         hosts: [],
       };
     }
 
     filters.push({
-      term: {
-        'host.hostname': id,
+      terms: {
+        'host.hostname': hostnames,
       },
     });
   }

x-pack/plugins/asset_manager/server/lib/accessors/services/get_services.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
 import { Asset } from '../../../../common/types_api';
 import { collectServices } from '../../collectors/services';
 import { parseEan } from '../../parse_ean';
@@ -23,10 +24,30 @@ export async function getServices(
 ): Promise<{ services: Asset[] }> {
   validateStringDateRange(options.from, options.to);
 
-  const filters = [];
+  const filters: QueryDslQueryContainer[] = [];
 
-  if (options.parent) {
-    const { kind, id } = parseEan(options.parent);
+  if (options.filters?.ean) {
+    const eans = Array.isArray(options.filters.ean) ? options.filters.ean : [options.filters.ean];
+    const services = eans
+      .map(parseEan)
+      .filter(({ kind }) => kind === 'service')
+      .map(({ id }) => id);
+
+    if (services.length === 0) {
+      return {
+        services: [],
+      };
+    }
+
+    filters.push({
+      terms: {
+        'service.name': services,
+      },
+    });
+  }
+
+  if (options.filters?.parentEan) {
+    const { kind, id } = parseEan(options.filters?.parentEan);
 
     if (kind === 'host') {
       filters.push({
@@ -47,6 +68,31 @@ export async function getServices(
     }
   }
 
+  if (options.filters?.id) {
+    const fn = options.filters.id.includes('*') ? 'wildcard' : 'term';
+    filters.push({
+      [fn]: {
+        'service.name': options.filters.id,
+      },
+    });
+  }
+
+  if (options.filters?.['cloud.provider']) {
+    filters.push({
+      term: {
+        'cloud.provider': options.filters['cloud.provider'],
+      },
+    });
+  }
+
+  if (options.filters?.['cloud.region']) {
+    filters.push({
+      term: {
+        'cloud.region': options.filters['cloud.region'],
+      },
+    });
+  }
+
   const apmIndices = await options.getApmIndices(options.savedObjectsClient);
   const { assets } = await collectServices({
     client: options.elasticsearchClient,

x-pack/plugins/asset_manager/server/lib/asset_client.ts

@@ -5,12 +5,17 @@
  * 2.0.
  */
 
+import { orderBy } from 'lodash';
 import { Asset } from '../../common/types_api';
+import { GetAssetsOptionsPublic } from '../../common/types_client';
 import { getContainers, GetContainersOptions } from './accessors/containers/get_containers';
 import { getHosts, GetHostsOptions } from './accessors/hosts/get_hosts';
 import { getServices, GetServicesOptions } from './accessors/services/get_services';
 import { getPods, GetPodsOptions } from './accessors/pods/get_pods';
 import { AssetClientBaseOptions, AssetClientOptionsWithInjectedValues } from './asset_client_types';
+import { AssetClientDependencies } from './asset_client_types';
+
+type GetAssetsOptions = GetAssetsOptionsPublic & AssetClientDependencies;
 
 export class AssetClient {
   constructor(private baseOptions: AssetClientBaseOptions) {}
@@ -41,4 +46,11 @@ export class AssetClient {
     const withInjected = this.injectOptions(options);
     return await getPods(withInjected);
   }
+
+  async getAssets(options: GetAssetsOptions): Promise<{ assets: Asset[] }> {
+    const withInjected = this.injectOptions(options);
+    const { hosts } = await getHosts(withInjected);
+    const { services } = await getServices(withInjected);
+    return { assets: orderBy(hosts.concat(services), ['@timestamp'], ['desc']) };
+  }
 }

x-pack/plugins/asset_manager/server/routes/assets/index.ts

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { createRouteValidationFunction } from '@kbn/io-ts-utils';
+import { RequestHandlerContext } from '@kbn/core-http-request-handler-context-server';
+import { GetAssetsQueryOptions, getAssetsQueryOptionsRT } from '../../../common/types_api';
+import { debug } from '../../../common/debug_log';
+import { SetupRouteOptions } from '../types';
+import * as routePaths from '../../../common/constants_routes';
+import { getClientsFromContext, validateStringAssetFilters } from '../utils';
+import { AssetsValidationError } from '../../lib/validators/validation_error';
+
+export function assetsRoutes<T extends RequestHandlerContext>({
+  router,
+  assetClient,
+}: SetupRouteOptions<T>) {
+  const validate = createRouteValidationFunction(getAssetsQueryOptionsRT);
+  router.get<unknown, GetAssetsQueryOptions, unknown>(
+    {
+      path: routePaths.GET_ASSETS,
+      validate: {
+        query: (q, res) => {
+          const [invalidResponse, validatedFilters] = validateStringAssetFilters(q, res);
+          if (invalidResponse) {
+            return invalidResponse;
+          }
+          if (validatedFilters) {
+            q.filters = validatedFilters;
+          }
+          return validate(q, res);
+        },
+      },
+    },
+    async (context, req, res) => {
+      const { from = 'now-24h', to = 'now', filters } = req.query || {};
+      const { elasticsearchClient, savedObjectsClient } = await getClientsFromContext(context);
+
+      try {
+        const response = await assetClient.getAssets({
+          from,
+          to,
+          filters,
+          elasticsearchClient,
+          savedObjectsClient,
+        });
+
+        return res.ok({ body: response });
+      } catch (error: unknown) {
+        debug('Error while looking up asset records', error);
+
+        if (error instanceof AssetsValidationError) {
+          return res.customError({
+            statusCode: error.statusCode,
+            body: {
+              message: `Error while looking up asset records - ${error.message}`,
+            },
+          });
+        }
+        return res.customError({
+          statusCode: 500,
+          body: { message: 'Error while looking up asset records - ' + `${error}` },
+        });
+      }
+    }
+  );
+}

x-pack/plugins/asset_manager/server/routes/assets/services.ts

@@ -14,29 +14,39 @@ import {
 import { debug } from '../../../common/debug_log';
 import { SetupRouteOptions } from '../types';
 import * as routePaths from '../../../common/constants_routes';
-import { getClientsFromContext } from '../utils';
+import { getClientsFromContext, validateStringAssetFilters } from '../utils';
 import { AssetsValidationError } from '../../lib/validators/validation_error';
 
 export function servicesRoutes<T extends RequestHandlerContext>({
   router,
   assetClient,
 }: SetupRouteOptions<T>) {
+  const validate = createRouteValidationFunction(getServiceAssetsQueryOptionsRT);
   // GET /assets/services
   router.get<unknown, GetServiceAssetsQueryOptions, unknown>(
     {
       path: routePaths.GET_SERVICES,
       validate: {
-        query: createRouteValidationFunction(getServiceAssetsQueryOptionsRT),
+        query: (q, res) => {
+          const [invalidResponse, validatedFilters] = validateStringAssetFilters(q, res);
+          if (invalidResponse) {
+            return invalidResponse;
+          }
+          if (validatedFilters) {
+            q.filters = validatedFilters;
+          }
+          return validate(q, res);
+        },
       },
     },
     async (context, req, res) => {
-      const { from = 'now-24h', to = 'now', parent } = req.query || {};
+      const { from = 'now-24h', to = 'now', filters } = req.query || {};
       const { elasticsearchClient, savedObjectsClient } = await getClientsFromContext(context);
       try {
         const response = await assetClient.getServices({
           from,
           to,
-          parent,
+          filters,
           elasticsearchClient,
           savedObjectsClient,
         });

x-pack/plugins/asset_manager/server/routes/index.ts

@@ -9,6 +9,7 @@ import { RequestHandlerContext } from '@kbn/core/server';
 import { SetupRouteOptions } from './types';
 import { pingRoute } from './ping';
 import { sampleAssetsRoutes } from './sample_assets';
+import { assetsRoutes } from './assets';
 import { hostsRoutes } from './assets/hosts';
 import { servicesRoutes } from './assets/services';
 import { containersRoutes } from './assets/containers';
@@ -20,6 +21,7 @@ export function setupRoutes<T extends RequestHandlerContext>({
 }: SetupRouteOptions<T>) {
   pingRoute<T>({ router, assetClient });
   sampleAssetsRoutes<T>({ router, assetClient });
+  assetsRoutes<T>({ router, assetClient });
   hostsRoutes<T>({ router, assetClient });
   servicesRoutes<T>({ router, assetClient });
   containersRoutes<T>({ router, assetClient });

x-pack/test/api_integration/apis/asset_manager/tests/assets.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { ASSETS_ENDPOINT } from './constants';
+import { FtrProviderContext } from '../types';
+import { generateHostsData, generateServicesData } from './helpers';
+
+export default function ({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+  const synthtraceApm = getService('apmSynthtraceEsClient');
+  const synthtraceInfra = getService('infraSynthtraceEsClient');
+
+  describe('GET /assets', () => {
+    const from = new Date(Date.now() - 1000 * 60 * 2).toISOString();
+    const to = new Date().toISOString();
+
+    beforeEach(async () => {
+      await synthtraceApm.clean();
+      await synthtraceInfra.clean();
+    });
+
+    it('should return all assets', async () => {
+      await Promise.all([
+        synthtraceInfra.index(generateHostsData({ from, to, count: 5 })),
+        synthtraceApm.index(generateServicesData({ from, to, count: 5 })),
+      ]);
+
+      const response = await supertest
+        .get(ASSETS_ENDPOINT)
+        .query({
+          from,
+          to,
+        })
+        .expect(200);
+
+      expect(response.body).to.have.property('assets');
+      expect(response.body.assets.length).to.equal(10);
+    });
+
+    it('supports only hosts and services', async () => {
+      await supertest
+        .get(ASSETS_ENDPOINT)
+        .query({
+          from,
+          to,
+          stringFilters: JSON.stringify({ kind: ['host', 'service'] }),
+        })
+        .expect(200);
+
+      await supertest
+        .get(ASSETS_ENDPOINT)
+        .query({
+          from,
+          to,
+          stringFilters: JSON.stringify({ kind: ['container', 'host'] }),
+        })
+        .expect(400);
+    });
+  });
+}

x-pack/test/api_integration/apis/asset_manager/tests/helpers.ts

@@ -7,6 +7,7 @@
 
 import type { AssetWithoutTimestamp } from '@kbn/assetManager-plugin/common/types_api';
 import type { WriteSamplesPostBody } from '@kbn/assetManager-plugin/server';
+import { apm, infra, timerange } from '@kbn/apm-synthtrace-client';
 import expect from '@kbn/expect';
 import { SuperTest, Test } from 'supertest';
 
@@ -43,3 +44,61 @@ export async function viewSampleAssetDocs(supertest: KibanaSupertest) {
   expect(response.body).to.have.property('results');
   return response.body.results as AssetWithoutTimestamp[];
 }
+
+export function generateServicesData({
+  from,
+  to,
+  count = 1,
+}: {
+  from: string;
+  to: string;
+  count: number;
+}) {
+  const range = timerange(from, to);
+
+  const services = Array(count)
+    .fill(0)
+    .map((_, idx) =>
+      apm
+        .service({
+          name: `service-${idx}`,
+          environment: 'production',
+          agentName: 'nodejs',
+        })
+        .instance(`my-host-${idx}`)
+    );
+
+  return range
+    .interval('1m')
+    .rate(1)
+    .generator((timestamp, index) =>
+      services.map((service) =>
+        service
+          .transaction({ transactionName: 'GET /foo' })
+          .timestamp(timestamp)
+          .duration(500)
+          .success()
+      )
+    );
+}
+
+export function generateHostsData({
+  from,
+  to,
+  count = 1,
+}: {
+  from: string;
+  to: string;
+  count: number;
+}) {
+  const range = timerange(from, to);
+
+  const hosts = Array(count)
+    .fill(0)
+    .map((_, idx) => infra.host(`my-host-${idx}`));
+
+  return range
+    .interval('1m')
+    .rate(1)
+    .generator((timestamp, index) => hosts.map((host) => host.metrics().timestamp(timestamp)));
+}

x-pack/test/api_integration/apis/asset_manager/tests/hosts.ts

@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import { timerange, infra } from '@kbn/apm-synthtrace-client';
 import expect from '@kbn/expect';
 import { Asset } from '@kbn/assetManager-plugin/common/types_api';
 import { ASSETS_ENDPOINT } from './constants';
 import { FtrProviderContext } from '../types';
+import { generateHostsData } from './helpers';
 
 const HOSTS_ASSETS_ENDPOINT = `${ASSETS_ENDPOINT}/hosts`;
 
@@ -87,16 +87,3 @@ export default function ({ getService }: FtrProviderContext) {
     });
   });
 }
-
-function generateHostsData({ from, to, count = 1 }: { from: string; to: string; count: number }) {
-  const range = timerange(from, to);
-
-  const hosts = Array(count)
-    .fill(0)
-    .map((_, idx) => infra.host(`my-host-${idx}`));
-
-  return range
-    .interval('1m')
-    .rate(1)
-    .generator((timestamp, index) => hosts.map((host) => host.metrics().timestamp(timestamp)));
-}

x-pack/test/api_integration/apis/asset_manager/tests/index.ts

@@ -14,5 +14,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./services'));
     loadTestFile(require.resolve('./pods'));
     loadTestFile(require.resolve('./sample_assets'));
+    loadTestFile(require.resolve('./assets'));
   });
 }

x-pack/test/api_integration/apis/asset_manager/tests/services.ts

@@ -6,10 +6,10 @@
  */
 
 import { omit } from 'lodash';
-import { apm, timerange } from '@kbn/apm-synthtrace-client';
 import expect from '@kbn/expect';
 import { ASSETS_ENDPOINT } from './constants';
 import { FtrProviderContext } from '../types';
+import { generateServicesData } from './helpers';
 
 const SERVICES_ASSETS_ENDPOINT = `${ASSETS_ENDPOINT}/services`;
 
@@ -39,6 +39,32 @@ export default function ({ getService }: FtrProviderContext) {
       expect(response.body.services.length).to.equal(2);
     });
 
+    it('should return specific service', async () => {
+      const from = new Date(Date.now() - 1000 * 60 * 2).toISOString();
+      const to = new Date().toISOString();
+      await synthtrace.index(generateServicesData({ from, to, count: 5 }));
+
+      const response = await supertest
+        .get(SERVICES_ASSETS_ENDPOINT)
+        .query({
+          from,
+          to,
+          stringFilters: JSON.stringify({ ean: 'service:service-4' }),
+        })
+        .expect(200);
+
+      expect(response.body).to.have.property('services');
+      expect(response.body.services.length).to.equal(1);
+      expect(omit(response.body.services[0], ['@timestamp'])).to.eql({
+        'asset.kind': 'service',
+        'asset.id': 'service-4',
+        'asset.ean': 'service:service-4',
+        'asset.references': [],
+        'asset.parents': [],
+        'service.environment': 'production',
+      });
+    });
+
     it('should return services running on specified host', async () => {
       const from = new Date(Date.now() - 1000 * 60 * 2).toISOString();
       const to = new Date().toISOString();
@@ -49,7 +75,7 @@ export default function ({ getService }: FtrProviderContext) {
         .query({
           from,
           to,
-          parent: 'host:my-host-1',
+          stringFilters: JSON.stringify({ parentEan: 'host:my-host-1' }),
         })
         .expect(200);
 
@@ -66,40 +92,3 @@ export default function ({ getService }: FtrProviderContext) {
     });
   });
 }
-
-function generateServicesData({
-  from,
-  to,
-  count = 1,
-}: {
-  from: string;
-  to: string;
-  count: number;
-}) {
-  const range = timerange(from, to);
-
-  const services = Array(count)
-    .fill(0)
-    .map((_, idx) =>
-      apm
-        .service({
-          name: `service-${idx}`,
-          environment: 'production',
-          agentName: 'nodejs',
-        })
-        .instance(`my-host-${idx}`)
-    );
-
-  return range
-    .interval('1m')
-    .rate(1)
-    .generator((timestamp, index) =>
-      services.map((service) =>
-        service
-          .transaction({ transactionName: 'GET /foo' })
-          .timestamp(timestamp)
-          .duration(500)
-          .success()
-      )
-    );
-}