[Ingest Manager] Add usage collector for telemetry. (#69294)

* Add usage collector for telemetry.

* Make minimal usage collector work.

* Add all fields to Usage and schema

* Type packages as array.

* Temporarily remove schema.

* Temporarily exclude our collector from schema checks.

* Add fleet telemetry.

* Remove events from agent stats.

* Add package telemetry.

* Use correct import.

* Add telemetry about enabled packages.

* Clean up comments.

* Update x-pack/plugins/ingest_manager/server/collectors/package_collectors.ts

Co-authored-by: Alejandro Fernández Haro <afharo@gmail.com>

* Update x-pack/plugins/ingest_manager/server/collectors/package_collectors.ts

Co-authored-by: Nicolas Chaulet <n.chaulet@gmail.com>

* Correctly check for element in array.

* Use a real SavedObjectsClient.

* Remove useless use of undefined.

* Use less deep path to import SavedObjectsClient.

Co-authored-by: Alejandro Fernández Haro <afharo@gmail.com>
Co-authored-by: Nicolas Chaulet <n.chaulet@gmail.com>

x-pack/.telemetryrc.json

@@ -7,6 +7,7 @@
     "plugins/apm/server/lib/apm_telemetry/index.ts",
     "plugins/canvas/server/collectors/collector.ts",
     "plugins/infra/server/usage/usage_collector.ts",
+    "plugins/ingest_manager/server/collectors/register.ts",
     "plugins/lens/server/usage/collectors.ts",
     "plugins/reporting/server/usage/reporting_usage_collector.ts",
     "plugins/maps/server/maps_telemetry/collectors/register.ts"

x-pack/plugins/ingest_manager/kibana.json

@@ -5,6 +5,6 @@
   "ui": true,
   "configPath": ["xpack", "ingestManager"],
   "requiredPlugins": ["licensing", "data", "encryptedSavedObjects"],
-  "optionalPlugins": ["security", "features", "cloud"],
+  "optionalPlugins": ["security", "features", "cloud", "usageCollection"],
   "extraPublicDirs": ["common"]
 }

x-pack/plugins/ingest_manager/server/collectors/agent_collectors.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SavedObjectsClient } from 'kibana/server';
+import * as AgentService from '../services/agents';
+export interface AgentUsage {
+  total: number;
+  online: number;
+  error: number;
+  offline: number;
+}
+
+export const getAgentUsage = async (soClient?: SavedObjectsClient): Promise<AgentUsage> => {
+  // TODO: unsure if this case is possible at all.
+  if (!soClient) {
+    return {
+      total: 0,
+      online: 0,
+      error: 0,
+      offline: 0,
+    };
+  }
+  const { total, online, error, offline } = await AgentService.getAgentStatusForConfig(soClient);
+  return {
+    total,
+    online,
+    error,
+    offline,
+  };
+};

x-pack/plugins/ingest_manager/server/collectors/config_collectors.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IngestManagerConfigType } from '..';
+
+export const getIsFleetEnabled = (config: IngestManagerConfigType) => {
+  return config.fleet.enabled;
+};

x-pack/plugins/ingest_manager/server/collectors/helpers.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CoreSetup } from 'kibana/server';
+import { SavedObjectsClient } from '../../../../../src/core/server';
+
+export async function getInternalSavedObjectsClient(core: CoreSetup) {
+  return core.getStartServices().then(async ([coreStart]) => {
+    const savedObjectsRepo = coreStart.savedObjects.createInternalRepository();
+    return new SavedObjectsClient(savedObjectsRepo);
+  });
+}

x-pack/plugins/ingest_manager/server/collectors/package_collectors.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SavedObjectsClient } from 'kibana/server';
+import _ from 'lodash';
+import { getPackageSavedObjects } from '../services/epm/packages/get';
+import { agentConfigService } from '../services';
+import { NewPackageConfig } from '../types';
+
+export interface PackageUsage {
+  name: string;
+  version: string;
+  enabled: boolean;
+}
+
+export const getPackageUsage = async (soClient?: SavedObjectsClient): Promise<PackageUsage[]> => {
+  if (!soClient) {
+    return [];
+  }
+  const packagesSavedObjects = await getPackageSavedObjects(soClient);
+  const agentConfigs = await agentConfigService.list(soClient, {
+    perPage: 1000, // avoiding pagination
+    withPackageConfigs: true,
+  });
+
+  // Once we provide detailed telemetry on agent configs, this logic should probably be moved
+  // to the (then to be created) agent config collector, so we only query and loop over these
+  // objects once.
+
+  const packagesInConfigs = agentConfigs.items.map((agentConfig) => {
+    const packageConfigs: NewPackageConfig[] = agentConfig.package_configs as NewPackageConfig[];
+    return packageConfigs
+      .map((packageConfig) => packageConfig.package?.name)
+      .filter((packageName): packageName is string => packageName !== undefined);
+  });
+
+  const enabledPackages = _.uniq(_.flatten(packagesInConfigs));
+
+  return packagesSavedObjects.saved_objects.map((p) => {
+    return {
+      name: p.attributes.name,
+      version: p.attributes.version,
+      enabled: enabledPackages.includes(p.attributes.name),
+    };
+  });
+};

x-pack/plugins/ingest_manager/server/collectors/register.ts

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
+import { CoreSetup } from 'kibana/server';
+import { getIsFleetEnabled } from './config_collectors';
+import { AgentUsage, getAgentUsage } from './agent_collectors';
+import { getInternalSavedObjectsClient } from './helpers';
+import { PackageUsage, getPackageUsage } from './package_collectors';
+import { IngestManagerConfigType } from '..';
+
+interface Usage {
+  fleet_enabled: boolean;
+  agents: AgentUsage;
+  packages: PackageUsage[];
+}
+
+export function registerIngestManagerUsageCollector(
+  core: CoreSetup,
+  config: IngestManagerConfigType,
+  usageCollection: UsageCollectionSetup | undefined
+): void {
+  // usageCollection is an optional dependency, so make sure to return if it is not registered.
+  // if for any reason the saved objects client is not available, also return
+  if (!usageCollection) {
+    return;
+  }
+
+  // create usage collector
+  const ingestManagerCollector = usageCollection.makeUsageCollector<Usage>({
+    type: 'ingest_manager',
+    isReady: () => true,
+    fetch: async () => {
+      const soClient = await getInternalSavedObjectsClient(core);
+      return {
+        fleet_enabled: getIsFleetEnabled(config),
+        agents: await getAgentUsage(soClient),
+        packages: await getPackageUsage(soClient),
+      };
+    },
+    // schema: { // temporarily disabled because of type errors
+    //   fleet_enabled: { type: 'boolean' },
+    //   agents: {
+    //    total: { type: 'number' },
+    //    online: { type: 'number' },
+    //    error: { type: 'number' },
+    //    offline: { type: 'number' },
+    //   },
+    //   packages: {
+    //     name: { type: 'keyword' },
+    //     version: { type: 'keyword' },
+    //     enabled: { type: boolean },
+    //   },
+    // },
+  });
+
+  // register usage collector
+  usageCollection.registerCollector(ingestManagerCollector);
+}

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -14,6 +14,7 @@ import {
   SavedObjectsServiceStart,
   HttpServiceSetup,
 } from 'kibana/server';
+import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { LicensingPluginSetup, ILicense } from '../../licensing/server';
 import {
   EncryptedSavedObjectsPluginStart,
@@ -62,6 +63,7 @@ import {
 } from './services/agents';
 import { CloudSetup } from '../../cloud/server';
 import { agentCheckinState } from './services/agents/checkin/state';
+import { registerIngestManagerUsageCollector } from './collectors/register';
 
 export interface IngestManagerSetupDeps {
   licensing: LicensingPluginSetup;
@@ -69,6 +71,7 @@ export interface IngestManagerSetupDeps {
   features?: FeaturesPluginSetup;
   encryptedSavedObjects: EncryptedSavedObjectsPluginSetup;
   cloud?: CloudSetup;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export type IngestManagerStartDeps = object;
@@ -198,6 +201,9 @@ export class IngestManagerPlugin
     const router = core.http.createRouter();
     const config = await this.config$.pipe(first()).toPromise();
 
+    // Register usage collection
+    registerIngestManagerUsageCollector(core, config, deps.usageCollection);
+
     // Always register app routes for permissions checking
     registerAppRoutes(router);