[TaskManager] log health on interval with background_tasks only role (#158890)

resolves https://github.com/elastic/kibana/issues/158870

## Summary

For Kibana servers that only have node role `background_tasks`, log the
task manager health report to the Kibana logs on an interval, currently
every hour.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/task_manager/server/lib/log_health_metrics.ts

@@ -6,6 +6,7 @@
  */
 
 import { isEmpty } from 'lodash';
+import { Observable } from 'rxjs';
 import { Logger, DocLinksServiceSetup } from '@kbn/core/server';
 import { HealthStatus } from '../monitoring';
 import { TaskManagerConfig } from '../config';
@@ -23,6 +24,29 @@ let lastLogLevel: LogLevel | null = null;
 export function resetLastLogLevel() {
   lastLogLevel = null;
 }
+
+export function setupIntervalLogging(
+  monitoredHealth$: Observable<MonitoredHealth>,
+  logger: Logger,
+  minutes: number
+) {
+  let monitoredHealth: MonitoredHealth | undefined;
+  monitoredHealth$.subscribe((m) => {
+    monitoredHealth = m;
+  });
+
+  setInterval(onInterval, 1000 * 60 * minutes);
+
+  function onInterval() {
+    const meta = { tags: ['task-manager-background-node-health'] };
+    if (!monitoredHealth) {
+      return logger.warn('unable to log health metrics, not initialized yet', meta);
+    }
+
+    logger.info(`background node health: ${JSON.stringify(monitoredHealth)}`, meta);
+  }
+}
+
 export function logHealthMetrics(
   monitoredHealth: MonitoredHealth,
   logger: Logger,

x-pack/plugins/task_manager/server/plugin.ts

@@ -34,6 +34,7 @@ import { EphemeralTask, ConcreteTaskInstance } from './task';
 import { registerTaskManagerUsageCollector } from './usage';
 import { TASK_MANAGER_INDEX } from './constants';
 import { AdHocTaskCounter } from './lib/adhoc_task_counter';
+import { setupIntervalLogging } from './lib/log_health_metrics';
 
 export interface TaskManagerSetupContract {
   /**
@@ -66,6 +67,8 @@ export type TaskManagerStartContract = Pick<
     getRegisteredTypes: () => string[];
   };
 
+const LogHealthForBackgroundTasksOnlyMinutes = 60;
+
 export class TaskManagerPlugin
   implements Plugin<TaskManagerSetupContract, TaskManagerStartContract>
 {
@@ -82,6 +85,7 @@ export class TaskManagerPlugin
   private shouldRunBackgroundTasks: boolean;
   private readonly kibanaVersion: PluginInitializerContext['env']['packageInfo']['version'];
   private adHocTaskCounter: AdHocTaskCounter;
+  private nodeRoles: PluginInitializerContext['node']['roles'];
 
   constructor(private readonly initContext: PluginInitializerContext) {
     this.initContext = initContext;
@@ -89,10 +93,16 @@ export class TaskManagerPlugin
     this.config = initContext.config.get<TaskManagerConfig>();
     this.definitions = new TaskTypeDictionary(this.logger);
     this.kibanaVersion = initContext.env.packageInfo.version;
-    this.shouldRunBackgroundTasks = initContext.node.roles.backgroundTasks;
+    this.nodeRoles = initContext.node.roles;
+    this.shouldRunBackgroundTasks = this.nodeRoles.backgroundTasks;
     this.adHocTaskCounter = new AdHocTaskCounter();
   }
 
+  isNodeBackgroundTasksOnly() {
+    const { backgroundTasks, migrator, ui } = this.nodeRoles;
+    return backgroundTasks && !migrator && !ui;
+  }
+
   public setup(
     core: CoreSetup,
     plugins: { usageCollection?: UsageCollectionSetup }
@@ -184,6 +194,11 @@ export class TaskManagerPlugin
       this.logger.warn(`Disabling authentication for background task utilization API`);
     }
 
+    // for nodes with background_tasks mode only, log health metrics every hour
+    if (this.isNodeBackgroundTasksOnly()) {
+      setupIntervalLogging(monitoredHealth$, this.logger, LogHealthForBackgroundTasksOnlyMinutes);
+    }
+
     return {
       index: TASK_MANAGER_INDEX,
       addMiddleware: (middleware: Middleware) => {