[APM] Add a logs tab for services (#107664)

* Add a logs tab for APM services

Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com>

x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap

@@ -75,6 +75,8 @@ exports[`Error HOST_NAME 1`] = `"my hostname"`;
 
 exports[`Error HOST_OS_PLATFORM 1`] = `undefined`;
 
+exports[`Error HOSTNAME 1`] = `undefined`;
+
 exports[`Error HTTP_REQUEST_METHOD 1`] = `undefined`;
 
 exports[`Error HTTP_RESPONSE_STATUS_CODE 1`] = `undefined`;
@@ -314,6 +316,8 @@ exports[`Span HOST_NAME 1`] = `undefined`;
 
 exports[`Span HOST_OS_PLATFORM 1`] = `undefined`;
 
+exports[`Span HOSTNAME 1`] = `undefined`;
+
 exports[`Span HTTP_REQUEST_METHOD 1`] = `undefined`;
 
 exports[`Span HTTP_RESPONSE_STATUS_CODE 1`] = `undefined`;
@@ -553,6 +557,8 @@ exports[`Transaction HOST_NAME 1`] = `"my hostname"`;
 
 exports[`Transaction HOST_OS_PLATFORM 1`] = `undefined`;
 
+exports[`Transaction HOSTNAME 1`] = `undefined`;
+
 exports[`Transaction HTTP_REQUEST_METHOD 1`] = `"GET"`;
 
 exports[`Transaction HTTP_RESPONSE_STATUS_CODE 1`] = `200`;

x-pack/plugins/apm/common/elasticsearch_fieldnames.ts

@@ -114,6 +114,7 @@ export const LABEL_NAME = 'labels.name';
 
 export const HOST = 'host';
 export const HOST_NAME = 'host.hostname';
+export const HOSTNAME = 'host.name';
 export const HOST_OS_PLATFORM = 'host.os.platform';
 export const CONTAINER_ID = 'container.id';
 export const KUBERNETES = 'kubernetes';

x-pack/plugins/apm/public/components/app/service_logs/index.tsx

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { isEmpty } from 'lodash';
+import { EuiLoadingSpinner, EuiEmptyPrompt } from '@elastic/eui';
+import React, { useMemo } from 'react';
+import { i18n } from '@kbn/i18n';
+import moment from 'moment';
+import { useUrlParams } from '../../../context/url_params_context/use_url_params';
+import { FETCH_STATUS, useFetcher } from '../../../hooks/use_fetcher';
+import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context';
+import { LogStream } from '../../../../../infra/public';
+import { APIReturnType } from '../../../services/rest/createCallApmApi';
+
+import {
+  CONTAINER_ID,
+  HOSTNAME,
+  POD_NAME,
+} from '../../../../common/elasticsearch_fieldnames';
+
+export function ServiceLogs() {
+  const { serviceName } = useApmServiceContext();
+  const {
+    urlParams: { environment, kuery, start, end },
+  } = useUrlParams();
+
+  const { data, status } = useFetcher(
+    (callApmApi) => {
+      if (start && end) {
+        return callApmApi({
+          endpoint: 'GET /api/apm/services/{serviceName}/infrastructure',
+          params: {
+            path: { serviceName },
+            query: {
+              environment,
+              kuery,
+              start,
+              end,
+            },
+          },
+        });
+      }
+    },
+    [environment, kuery, serviceName, start, end]
+  );
+
+  const noInfrastructureData = useMemo(() => {
+    return (
+      isEmpty(data?.serviceInfrastructure?.containerIds) &&
+      isEmpty(data?.serviceInfrastructure?.hostNames) &&
+      isEmpty(data?.serviceInfrastructure?.podNames)
+    );
+  }, [data]);
+
+  if (status === FETCH_STATUS.LOADING) {
+    return (
+      <div style={{ textAlign: 'center' }}>
+        <EuiLoadingSpinner size="m" />
+      </div>
+    );
+  }
+
+  if (status === FETCH_STATUS.SUCCESS && noInfrastructureData) {
+    return (
+      <EuiEmptyPrompt
+        title={
+          <h2>
+            {i18n.translate('xpack.apm.serviceLogs.noInfrastructureMessage', {
+              defaultMessage: 'There are no log messages to display.',
+            })}
+          </h2>
+        }
+      />
+    );
+  }
+
+  return (
+    <LogStream
+      columns={[{ type: 'timestamp' }, { type: 'message' }]}
+      height={'60vh'}
+      startTimestamp={moment(start).valueOf()}
+      endTimestamp={moment(end).valueOf()}
+      query={getInfrastructureKQLFilter(data)}
+    />
+  );
+}
+
+const getInfrastructureKQLFilter = (
+  data?: APIReturnType<'GET /api/apm/services/{serviceName}/infrastructure'>
+) => {
+  const containerIds = data?.serviceInfrastructure?.containerIds ?? [];
+  const hostNames = data?.serviceInfrastructure?.hostNames ?? [];
+  const podNames = data?.serviceInfrastructure?.podNames ?? [];
+
+  return [
+    ...containerIds.map((id) => `${CONTAINER_ID}: "${id}"`),
+    ...hostNames.map((id) => `${HOSTNAME}: "${id}"`),
+    ...podNames.map((id) => `${POD_NAME}: "${id}"`),
+  ].join(' or ');
+};

x-pack/plugins/apm/public/components/routing/service_detail/index.tsx

@@ -22,6 +22,7 @@ import { ServiceMap } from '../../app/service_map';
 import { TransactionDetails } from '../../app/transaction_details';
 import { ServiceProfiling } from '../../app/service_profiling';
 import { ServiceDependencies } from '../../app/service_dependencies';
+import { ServiceLogs } from '../../app/service_logs';
 
 function page<TPath extends string>({
   path,
@@ -233,6 +234,14 @@ export const serviceDetail = {
         hidden: true,
       },
     }),
+    page({
+      path: '/logs',
+      tab: 'logs',
+      title: i18n.translate('xpack.apm.views.logs.title', {
+        defaultMessage: 'Logs',
+      }),
+      element: <ServiceLogs />,
+    }),
     page({
       path: '/profiling',
       tab: 'profiling',

x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx

@@ -41,6 +41,7 @@ type Tab = NonNullable<EuiPageHeaderProps['tabs']>[0] & {
     | 'metrics'
     | 'nodes'
     | 'service-map'
+    | 'logs'
     | 'profiling';
   hidden?: boolean;
 };
@@ -218,6 +219,18 @@ function useTabs({ selectedTab }: { selectedTab: Tab['key'] }) {
         defaultMessage: 'Service Map',
       }),
     },
+    {
+      key: 'logs',
+      href: router.link('/services/:serviceName/logs', {
+        path: { serviceName },
+        query,
+      }),
+      label: i18n.translate('xpack.apm.home.serviceLogsTabLabel', {
+        defaultMessage: 'Logs',
+      }),
+      hidden:
+        !agentName || isRumAgentName(agentName) || isIosAgentName(agentName),
+    },
     {
       key: 'profiling',
       href: router.link('/services/:serviceName/profiling', {

x-pack/plugins/apm/server/lib/services/get_service_infrastructure.ts

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Setup, SetupTimeRange } from '../helpers/setup_request';
+import { ESFilter } from '../../../../../../src/core/types/elasticsearch';
+import { rangeQuery, kqlQuery } from '../../../../observability/server';
+import { environmentQuery } from '../../../common/utils/environment_query';
+import { ProcessorEvent } from '../../../common/processor_event';
+import {
+  SERVICE_NAME,
+  CONTAINER_ID,
+  HOSTNAME,
+  POD_NAME,
+} from '../../../common/elasticsearch_fieldnames';
+
+export const getServiceInfrastructure = async ({
+  kuery,
+  serviceName,
+  environment,
+  setup,
+}: {
+  kuery?: string;
+  serviceName: string;
+  environment?: string;
+  setup: Setup & SetupTimeRange;
+}) => {
+  const { apmEventClient, start, end } = setup;
+
+  const filter: ESFilter[] = [
+    { term: { [SERVICE_NAME]: serviceName } },
+    ...rangeQuery(start, end),
+    ...environmentQuery(environment),
+    ...kqlQuery(kuery),
+  ];
+
+  const response = await apmEventClient.search('get_service_infrastructure', {
+    apm: {
+      events: [ProcessorEvent.metric],
+    },
+    body: {
+      size: 0,
+      query: {
+        bool: {
+          filter,
+        },
+      },
+      aggs: {
+        containerIds: {
+          terms: {
+            field: CONTAINER_ID,
+            size: 500,
+          },
+        },
+        hostNames: {
+          terms: {
+            field: HOSTNAME,
+            size: 500,
+          },
+        },
+        podNames: {
+          terms: {
+            field: POD_NAME,
+            size: 500,
+          },
+        },
+      },
+    },
+  });
+
+  return {
+    containerIds:
+      response.aggregations?.containerIds?.buckets.map(
+        (bucket) => bucket.key
+      ) ?? [],
+    hostNames:
+      response.aggregations?.hostNames?.buckets.map((bucket) => bucket.key) ??
+      [],
+    podNames:
+      response.aggregations?.podNames?.buckets.map((bucket) => bucket.key) ??
+      [],
+  };
+};

x-pack/plugins/apm/server/routes/services.ts

@@ -31,6 +31,7 @@ import { getServiceTransactionTypes } from '../lib/services/get_service_transact
 import { getThroughput } from '../lib/services/get_throughput';
 import { getServiceProfilingStatistics } from '../lib/services/profiling/get_service_profiling_statistics';
 import { getServiceProfilingTimeline } from '../lib/services/profiling/get_service_profiling_timeline';
+import { getServiceInfrastructure } from '../lib/services/get_service_infrastructure';
 import { withApmSpan } from '../utils/with_apm_span';
 import { createApmServerRoute } from './create_apm_server_route';
 import { createApmServerRouteRepository } from './create_apm_server_route_repository';
@@ -853,6 +854,35 @@ const serviceAlertsRoute = createApmServerRoute({
   },
 });
 
+const serviceInfrastructureRoute = createApmServerRoute({
+  endpoint: 'GET /api/apm/services/{serviceName}/infrastructure',
+  params: t.type({
+    path: t.type({
+      serviceName: t.string,
+    }),
+    query: t.intersection([kueryRt, rangeRt, environmentRt]),
+  }),
+  options: { tags: ['access:apm'] },
+  handler: async (resources) => {
+    const setup = await setupRequest(resources);
+
+    const { params } = resources;
+
+    const {
+      path: { serviceName },
+      query: { environment, kuery },
+    } = params;
+
+    const serviceInfrastructure = await getServiceInfrastructure({
+      setup,
+      serviceName,
+      environment,
+      kuery,
+    });
+    return { serviceInfrastructure };
+  },
+});
+
 export const serviceRouteRepository = createApmServerRouteRepository()
   .add(servicesRoute)
   .add(servicesDetailedStatisticsRoute)
@@ -873,4 +903,5 @@ export const serviceRouteRepository = createApmServerRouteRepository()
   .add(serviceDependenciesBreakdownRoute)
   .add(serviceProfilingTimelineRoute)
   .add(serviceProfilingStatisticsRoute)
-  .add(serviceAlertsRoute);
+  .add(serviceAlertsRoute)
+  .add(serviceInfrastructureRoute);

x-pack/plugins/infra/public/components/log_stream/log_stream.stories.mdx

@@ -195,7 +195,11 @@ This will show a list of log entries between the specified timestamps.
 
 ## Query log entries
 
-You might want to show specific log entries in your plugin. Maybe you want to show log lines from a specific host, or for an AMP trace. The component has a `query` prop that accepts valid KQL expressions.
+You might want to show specific log entries in your plugin. Maybe you want to show log lines from a specific host, or for an AMP trace. The LogStream component supports both `query` and `filters`, and these are the standard `es-query` types. 
+
+### Query
+
+The component has a `query` prop that accepts a valid es-query `query`. You can either supply this with a `language` and `query` property, or you can just supply a string which is a shortcut for KQL expressions. 
 
 ```tsx
 <LogStream
@@ -205,6 +209,31 @@ You might want to show specific log entries in your plugin. Maybe you want to sh
 />
 ```
 
+### Filters
+
+The component also has a `filters` prop that accepts valid es-query `filters`. This example would specifiy that we want the `message` field to exist:
+
+```tsx
+<LogStream
+  startTimestamp={startTimestamp}
+  endTimestamp={endTimestamp}
+  filters={[
+    {
+      query: {
+        exists: {
+          field: 'message',
+        },
+      },
+      meta: {
+        alias: null,
+        disabled: false,
+        negate: false,
+      },
+    }]
+  }
+/>
+```
+
 ## Center the view on a specific entry
 
 By default the component will load at the bottom of the list, showing the newest entries. You can change the rendering point with the `center` prop. The prop takes a [`LogEntriesCursor`](https://github.com/elastic/kibana/blob/0a6c748cc837c016901f69ff05d81395aa2d41c8/x-pack/plugins/infra/common/http_api/log_entries/common.ts#L9-L13).
@@ -431,3 +460,7 @@ class MyPlugin {
   endTimestamp={...}
 />
 ```
+
+### Setting component height
+
+It's possible to pass a `height` prop, e.g. `60vh` or `300px`, to specify how much vertical space the component should consume.

x-pack/plugins/infra/public/components/log_stream/log_stream.tsx

@@ -5,10 +5,11 @@
  * 2.0.
  */
 
+import { buildEsQuery, Query, Filter } from '@kbn/es-query';
 import React, { useMemo, useCallback, useEffect } from 'react';
 import { noop } from 'lodash';
 import { JsonValue } from '@kbn/utility-types';
-import { DataPublicPluginStart, esQuery, Filter } from '../../../../../../src/plugins/data/public';
+import { DataPublicPluginStart } from '../../../../../../src/plugins/data/public';
 import { euiStyled } from '../../../../../../src/plugins/kibana_react/common';
 import { LogEntryCursor } from '../../../common/log_entry';
 
@@ -18,7 +19,6 @@ import { BuiltEsQuery, useLogStream } from '../../containers/logs/log_stream';
 
 import { ScrollableLogTextStreamView } from '../logging/log_text_stream';
 import { LogColumnRenderConfiguration } from '../../utils/log_column_render_configuration';
-import { Query } from '../../../../../../src/plugins/data/common';
 import { LogStreamErrorBoundary } from './log_stream_error_boundary';
 
 interface LogStreamPluginDeps {
@@ -123,9 +123,9 @@ Read more at https://github.com/elastic/kibana/blob/master/src/plugins/kibana_re
 
   const parsedQuery = useMemo<BuiltEsQuery | undefined>(() => {
     if (typeof query === 'object' && 'bool' in query) {
-      return mergeBoolQueries(query, esQuery.buildEsQuery(derivedIndexPattern, [], filters ?? []));
+      return mergeBoolQueries(query, buildEsQuery(derivedIndexPattern, [], filters ?? []));
     } else {
-      return esQuery.buildEsQuery(derivedIndexPattern, coerceToQueries(query), filters ?? []);
+      return buildEsQuery(derivedIndexPattern, coerceToQueries(query), filters ?? []);
     }
   }, [derivedIndexPattern, filters, query]);