[Stack Monitoring] support entsearch package (#148668)

### Summary

Update stack monitoring to read from data streams created by the
[enterprisesearch
package](https://github.com/elastic/integrations/pull/4926)

### Testing
- build [enterprisesearch
package](https://github.com/elastic/integrations/pull/4926)
- start stack: `elastic-package stack up -v -d --version
8.7.0-SNAPSHOT`; make sure your image contains [this
change](https://github.com/elastic/elastic-agent/pull/2121)
- start enterprisesearch service: `elastic-package service up -v`
- install elasticsearch integration with variables
- `hosts: http://host.docker.internal:9201; username: elastic; password:
changeme`
- install enterprisesearch integration with variables
- `hosts: http://host.docker.internal:3002; username: elastic; password:
changeme`
- connect [local
kibana](2d893bf40b/x-pack/plugins/monitoring/dev_docs/how_to/work_with_packages.md (connecting-a-local-kibana))
- navigate to Stack Monitoring on the local kibana, enterprise search
section shows up with populated graphs

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/monitoring/common/constants.ts

@@ -190,7 +190,7 @@ export const BEATS_SYSTEM_ID = 'beats';
  * The name of the Enterprise Search System ID used to publish and look up Enterprise Search stats through the Monitoring system.
  * @type {string}
  */
-export const ENTERPRISE_SEARCH_SYSTEM_ID = 'enterprise_search';
+export const ENTERPRISE_SEARCH_SYSTEM_ID = 'enterprisesearch';
 
 /**
  * The name of the Apm System ID used to publish and look up Apm stats through the Monitoring system.

x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.ts

@@ -69,7 +69,7 @@ export function getLegacyIndexPattern({
     case 'beats':
       indexPattern = INDEX_PATTERN_BEATS;
       break;
-    case 'enterprise_search':
+    case 'enterprisesearch':
       indexPattern = INDEX_PATTERN_ENTERPRISE_SEARCH;
       break;
     case 'filebeat':
@@ -127,6 +127,7 @@ export const getElasticsearchDataset = getDataset('elasticsearch');
 export const getKibanaDataset = getDataset('kibana');
 export const getLogstashDataset = getDataset('logstash');
 export const getBeatDataset = getDataset('beats');
+export const getEntsearchDataset = getDataset('enterprisesearch');
 
 function buildDatasetPattern(
   moduleType?: INDEX_PATTERN_TYPES,

x-pack/plugins/monitoring/server/lib/enterprise_search/create_enterprise_search_query.ts

@@ -8,6 +8,7 @@
 import { EnterpriseSearchMetric, EnterpriseSearchMetricFields } from '../metrics';
 import { createQuery } from '../create_query';
 import { STANDALONE_CLUSTER_CLUSTER_UUID } from '../../../common/constants';
+import { getEntsearchDataset } from '../cluster/get_index_patterns';
 
 /**
  * {@code createQuery} for all Enterprise Search instances.
@@ -53,6 +54,8 @@ export function createEnterpriseSearchQuery(options: {
         },
         { term: { 'event.dataset': 'enterprisesearch.health' } },
         { term: { 'event.dataset': 'enterprisesearch.stats' } },
+        { term: { 'data_stream.dataset': getEntsearchDataset('health') } },
+        { term: { 'data_stream.dataset': getEntsearchDataset('stats') } },
       ],
     },
   });

x-pack/plugins/monitoring/server/lib/enterprise_search/get_enterprise_search_for_clusters.ts

@@ -9,7 +9,7 @@ import { TimeRange } from '../../../common/http_api/shared';
 import { ElasticsearchResponse } from '../../../common/types/es';
 import { Globals } from '../../static_globals';
 import { Cluster, LegacyRequest } from '../../types';
-import { getLegacyIndexPattern } from '../cluster/get_index_patterns';
+import { getIndexPatterns } from '../cluster/get_index_patterns';
 import { EnterpriseSearchMetric } from '../metrics';
 import { createEnterpriseSearchQuery } from './create_enterprise_search_query';
 import {
@@ -37,8 +37,8 @@ export function getEnterpriseSearchForClusters(
   const config = req.server.config;
   const maxBucketSize = config.ui.max_bucket_size;
 
-  const indexPatterns = getLegacyIndexPattern({
-    moduleType: 'enterprise_search',
+  const indexPatterns = getIndexPatterns({
+    moduleType: 'enterprisesearch',
     ccs,
     config: Globals.app.config,
   });

x-pack/plugins/monitoring/server/lib/enterprise_search/get_stats.ts

@@ -10,7 +10,7 @@ import { TimeRange } from '../../../common/http_api/shared';
 import { ElasticsearchResponse } from '../../../common/types/es';
 import { Globals } from '../../static_globals';
 import { LegacyRequest } from '../../types';
-import { getLegacyIndexPattern } from '../cluster/get_index_patterns';
+import { getIndexPatterns } from '../cluster/get_index_patterns';
 import { createEnterpriseSearchQuery } from './create_enterprise_search_query';
 import {
   entSearchAggFilterPath,
@@ -27,9 +27,8 @@ export async function getStats(
   const end = moment.utc(req.payload.timeRange.max).valueOf();
   const maxBucketSize = config.ui.max_bucket_size;
 
-  // just get the legacy pattern since no integration exists yet
-  const indexPattern = getLegacyIndexPattern({
-    moduleType: 'enterprise_search',
+  const indexPattern = getIndexPatterns({
+    moduleType: 'enterprisesearch',
     config: Globals.app.config,
     ccs: req.payload.ccs,
   });

x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.ts

@@ -26,7 +26,7 @@ interface Bucket {
   };
 }
 
-type COLLECTION_PRODUCT_NAMES = Exclude<INDEX_PATTERN_TYPES, 'enterprise_search'>;
+type COLLECTION_PRODUCT_NAMES = Exclude<INDEX_PATTERN_TYPES, 'enterprisesearch'>;
 
 const NUMBER_OF_SECONDS_AGO_TO_LOOK = 30;
 

x-pack/plugins/monitoring/server/routes/api/v1/_health/errors_helpers/errors_query.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import type { MetricbeatMonitoredProduct, PackagesMonitoredProduct, QueryOptions } from '../types';
+import type { MonitoredProduct, QueryOptions } from '../types';
 
 const MAX_BUCKET_SIZE = 50;
 
@@ -14,7 +14,7 @@ const MAX_BUCKET_SIZE = 50;
  * Each module (beats, kibana...) can contain one or multiple metricsets with error messages
  */
 interface ErrorsQueryOptions extends QueryOptions {
-  products: MetricbeatMonitoredProduct[] | PackagesMonitoredProduct[];
+  products: Array<Exclude<MonitoredProduct, MonitoredProduct.Cluster>>;
   errorQueryType: 'metricbeatErrorsQuery' | 'packageErrorsQuery';
   errorQueryIsDataStream?: boolean;
 }
@@ -47,7 +47,7 @@ export const errorsQuery = ({
               },
               {
                 range: {
-                  timestamp: {
+                  '@timestamp': {
                     gte: timeRange.min,
                     lte: timeRange.max,
                   },

x-pack/plugins/monitoring/server/routes/api/v1/_health/index.ts

@@ -60,9 +60,10 @@ export function registerV1HealthRoute(server: MonitoringCore) {
         getDsIndexPattern({ config, moduleType: 'kibana' }),
         getDsIndexPattern({ config, moduleType: 'logstash' }),
         getDsIndexPattern({ config, moduleType: 'beats' }),
+        getDsIndexPattern({ config, moduleType: 'enterprisesearch' }),
       ].join(',');
 
-      const entSearchIndex = getIndexPatterns({ config, moduleType: 'enterprise_search' });
+      const entSearchIndex = getIndexPatterns({ config, moduleType: 'enterprisesearch' });
 
       const monitoredClustersFn = () =>
         fetchMonitoredClusters({

x-pack/plugins/monitoring/server/routes/api/v1/_health/monitored_clusters/build_monitored_clusters.ts

@@ -43,7 +43,8 @@ const metricbeatMonitoring7Pattern =
   /(.*:)?\.monitoring-(es|kibana|beats|logstash|ent-search)-7.*-mb.*/;
 const metricbeatMonitoring8Pattern =
   /(.*:)?\.ds-\.monitoring-(es|kibana|beats|logstash|ent-search)-8-mb.*/;
-const packagePattern = /(.*:)?\.ds-metrics-(elasticsearch|kibana|beats|logstash)\..*/;
+const packagePattern =
+  /(.*:)?\.ds-metrics-(elasticsearch|kibana|beats|logstash|enterprisesearch)\..*/;
 
 const getCollectionMode = (index: string): CollectionMode => {
   if (internalMonitoringPattern.test(index)) return CollectionMode.Internal;

x-pack/plugins/monitoring/server/routes/api/v1/_health/monitored_clusters/fetch_monitored_clusters.test.ts

@@ -224,7 +224,7 @@ describe(__filename, () => {
               {
                 key: 'cluster-id.1',
                 doc_count: 11874,
-                enterpriseSearch: {
+                enterprisesearch: {
                   meta: {},
                   doc_count_error_upper_bound: 0,
                   sum_other_doc_count: 0,
@@ -310,7 +310,7 @@ describe(__filename, () => {
               },
             },
 
-            enterpriseSearch: {
+            enterprisesearch: {
               'ent-search-node-id.1': {
                 health: {
                   'metricbeat-8': {

x-pack/plugins/monitoring/server/routes/api/v1/_health/monitored_clusters/monitored_clusters_query.ts

@@ -11,6 +11,7 @@ import {
   getElasticsearchDataset,
   getKibanaDataset,
   getLogstashDataset,
+  getEntsearchDataset,
 } from '../../../../../lib/cluster/get_index_patterns';
 
 const MAX_BUCKET_SIZE = 100;
@@ -187,7 +188,7 @@ export const enterpriseSearchQuery = ({ timeRange, timeout }: QueryOptions) => {
           size: MAX_BUCKET_SIZE,
         },
         aggs: {
-          enterpriseSearch: {
+          enterprisesearch: {
             terms: {
               field: 'agent.id',
             },
@@ -202,6 +203,11 @@ export const enterpriseSearchQuery = ({ timeRange, timeout }: QueryOptions) => {
                             'metricset.name': 'health',
                           },
                         },
+                        {
+                          term: {
+                            'data_stream.dataset': getEntsearchDataset('health'),
+                          },
+                        },
                       ],
                     },
                   },
@@ -219,6 +225,11 @@ export const enterpriseSearchQuery = ({ timeRange, timeout }: QueryOptions) => {
                             'metricset.name': 'stats',
                           },
                         },
+                        {
+                          term: {
+                            'data_stream.dataset': getEntsearchDataset('stats'),
+                          },
+                        },
                       ],
                     },
                   },

x-pack/plugins/monitoring/server/routes/api/v1/_health/package/fetch_package_errors.ts

@@ -37,6 +37,7 @@ export const fetchPackageErrors = async ({
           MonitoredProduct.Elasticsearch,
           MonitoredProduct.Kibana,
           MonitoredProduct.Logstash,
+          MonitoredProduct.EnterpriseSearch,
         ],
         errorQueryType: 'packageErrorsQuery',
         errorQueryIsDataStream: true,

x-pack/plugins/monitoring/server/routes/api/v1/_health/types.ts

@@ -15,14 +15,8 @@ export enum MonitoredProduct {
   Kibana = 'kibana',
   Beats = 'beat',
   Logstash = 'logstash',
-  EnterpriseSearch = 'enterpriseSearch',
+  EnterpriseSearch = 'enterprisesearch',
 }
-export type MetricbeatMonitoredProduct = Exclude<MonitoredProduct, MonitoredProduct.Cluster>;
-export type PackagesMonitoredProduct = Exclude<
-  MetricbeatMonitoredProduct,
-  MonitoredProduct.EnterpriseSearch
->;
-
 export type SearchFn = (params: any) => Promise<ElasticsearchResponse>;
 
 export interface QueryOptions {

x-pack/plugins/monitoring/server/routes/api/v1/enterprise_search/overview.ts

@@ -33,7 +33,7 @@ export function entSearchOverviewRoute(server: MonitoringCore) {
       try {
         const [stats, metrics] = await Promise.all([
           getStats(req, clusterUuid),
-          getMetrics(req, 'enterprise_search', metricSet, [], {
+          getMetrics(req, 'enterprisesearch', metricSet, [], {
             skipClusterUuidFilter: true,
           }),
         ]);

x-pack/test/monitoring_api_integration/apis/enterprisesearch/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Enterprisesearch', () => {
+    loadTestFile(require.resolve('./overview'));
+  });
+}

x-pack/test/monitoring_api_integration/apis/enterprisesearch/overview.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
+import { getTestRunner } from '../../utils/test_runner';
+
+import response from '../../fixtures/enterprisesearch/overview.json';
+
+export default function ({ getService }: FtrProviderContext) {
+  const supertest = getService('supertest');
+
+  const testRunner = getTestRunner({
+    testName: 'Overview',
+    archiveRoot: 'x-pack/test/monitoring_api_integration/archives/enterprisesearch',
+    getService,
+  });
+
+  const timeRange = {
+    min: '2023-01-11T20:54:00.000Z',
+    max: '2023-01-11T21:01:00.000Z',
+  };
+
+  testRunner(() => {
+    it('should summarize enterprisesearch cluster with metrics', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/clusters/FDTNEesSQ7GbzOXIO9qImw/enterprise_search')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(response);
+    });
+  });
+}

x-pack/test/monitoring_api_integration/apis/index.ts

@@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
   describe('Monitoring Endpoints', function () {
     loadTestFile(require.resolve('./apm'));
     loadTestFile(require.resolve('./beats'));
+    loadTestFile(require.resolve('./enterprisesearch'));
   });
 }

x-pack/test/monitoring_api_integration/packages.ts

@@ -7,7 +7,10 @@
 
 import path from 'path';
 
-const PACKAGES = [{ name: 'beat', version: '0.0.1' }];
+const PACKAGES = [
+  { name: 'beat', version: '0.0.1' },
+  { name: 'enterprisesearch', version: '1.0.0' },
+];
 
 export const getPackagesArgs = (): string[] => {
   return PACKAGES.flatMap((pkg, i) => {

x-pack/test/monitoring_api_integration/scripts/transform_archive.js

@@ -56,6 +56,8 @@ const {
           ? 'kibana'
           : source.beat
           ? 'beats'
+          : source.enterprisesearch
+          ? 'ent-search'
           : source.elasticsearch
           ? 'es'
           : null;

x-pack/test/monitoring_api_integration/utils/lifecycle_methods.ts

@@ -33,7 +33,7 @@ export const getLifecycleMethods = (getService: FtrProviderContext['getService']
       // the tests suites. since the archiver doesn't have any reference to the
       // mappings it can't automatically delete it and we have to do the cleanup manually
       await deleteDataStream('.monitoring-*');
-      await deleteDataStream('metrics-beats.stack_monitoring.*');
+      await deleteDataStream('metrics-*.stack_monitoring.*');
     },
   };
 };