github-mirrors/prowlarr
1
0
Fork 0
mirror of https://github.com/Prowlarr/Prowlarr.git synced 2025-04-24 05:47:22 -04:00
Code Issues Projects Releases Packages Wiki Activity

Fixed: Duplicate notifications for failed health checks

Browse source 
(cherry picked from commit c0e54773e213f526a5046fa46aa7b57532471128)

Mock debouncer for testing

(cherry picked from commit bb7b2808e2f70389157408809ec47cc8860b4938)
This commit is contained in:
Mark McDowall 2023-09-10 13:07:57 -07:00 • committed by Bogdan
parent 42c533386b
commit c41a7e0ccc
6 changed files with 157 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/NzbDrone.Common/TPL/DebounceManager.cs Normal file
View file

@ -0,0 +1,17 @@
using System;
namespace NzbDrone.Common.TPL
{
public interface IDebounceManager
{
Debouncer CreateDebouncer(Action action, TimeSpan debounceDuration);
}
public class DebounceManager : IDebounceManager
{
public Debouncer CreateDebouncer(Action action, TimeSpan debounceDuration)
{
return new Debouncer(action, debounceDuration);
}
}
}

14
src/NzbDrone.Common/TPL/Debouncer.cs
View file

@ -4,11 +4,11 @@ namespace NzbDrone.Common.TPL
{
public class Debouncer
{
private readonly Action _action;
private readonly System.Timers.Timer _timer;
protected readonly Action _action;
protected readonly System.Timers.Timer _timer;
private volatile int _paused;
private volatile bool _triggered;
protected volatile int _paused;
protected volatile bool _triggered;
public Debouncer(Action action, TimeSpan debounceDuration)
{
@ -27,7 +27,7 @@ namespace NzbDrone.Common.TPL
}
}
public void Execute()
public virtual void Execute()
{
lock (_timer)
{
@ -39,7 +39,7 @@ namespace NzbDrone.Common.TPL
}
}
public void Pause()
public virtual void Pause()
{
lock (_timer)
{
@ -48,7 +48,7 @@ namespace NzbDrone.Common.TPL
}
}
public void Resume()
public virtual void Resume()
{
lock (_timer)
{

10
src/NzbDrone.Core.Test/HealthCheck/HealthCheckServiceFixture.cs
View file

@ -1,10 +1,14 @@
using System;
using System.Collections.Generic;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Messaging;
using NzbDrone.Common.TPL;
using NzbDrone.Core.HealthCheck;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Test.Common;
namespace NzbDrone.Core.Test.HealthCheck
{
@ -19,10 +23,10 @@ namespace NzbDrone.Core.Test.HealthCheck
Mocker.SetConstant<IEnumerable<IProvideHealthCheck>>(new[] { _healthCheck });
Mocker.SetConstant<ICacheManager>(Mocker.Resolve<CacheManager>());
Mocker.SetConstant<IDebounceManager>(Mocker.Resolve<DebounceManager>());
Mocker.GetMock<IServerSideNotificationService>()
.Setup(v => v.GetServerChecks())
.Returns(new List<Core.HealthCheck.HealthCheck>());
Mocker.GetMock<IDebounceManager>().Setup(s => s.CreateDebouncer(It.IsAny<Action>(), It.IsAny<TimeSpan>()))
.Returns<Action, TimeSpan>((a, t) => new MockDebouncer(a, t));
}
[Test]

128
src/NzbDrone.Core/HealthCheck/HealthCheckService.cs
View file

@ -6,6 +6,7 @@ using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Messaging;
using NzbDrone.Common.Reflection;
using NzbDrone.Common.TPL;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
@ -27,35 +28,35 @@ namespace NzbDrone.Core.HealthCheck
private readonly IProvideHealthCheck[] _startupHealthChecks;
private readonly IProvideHealthCheck[] _scheduledHealthChecks;
private readonly Dictionary<Type, IEventDrivenHealthCheck[]> _eventDrivenHealthChecks;
private readonly IServerSideNotificationService _serverSideNotificationService;
private readonly IEventAggregator _eventAggregator;
private readonly ICacheManager _cacheManager;
private readonly Logger _logger;
private readonly ICached<HealthCheck> _healthCheckResults;
private readonly HashSet<IProvideHealthCheck> _pendingHealthChecks;
private readonly Debouncer _debounce;
private bool _hasRunHealthChecksAfterGracePeriod;
private bool _isRunningHealthChecksAfterGracePeriod;
public HealthCheckService(IEnumerable<IProvideHealthCheck> healthChecks,
IServerSideNotificationService serverSideNotificationService,
IEventAggregator eventAggregator,
ICacheManager cacheManager,
IDebounceManager debounceManager,
IRuntimeInfo runtimeInfo,
Logger logger)
{
_healthChecks = healthChecks.ToArray();
_serverSideNotificationService = serverSideNotificationService;
_eventAggregator = eventAggregator;
_cacheManager = cacheManager;
_logger = logger;
_healthCheckResults = _cacheManager.GetCache<HealthCheck>(GetType());
_healthCheckResults = cacheManager.GetCache<HealthCheck>(GetType());
_pendingHealthChecks = new HashSet<IProvideHealthCheck>();
_debounce = debounceManager.CreateDebouncer(ProcessHealthChecks, TimeSpan.FromSeconds(5));
_startupHealthChecks = _healthChecks.Where(v => v.CheckOnStartup).ToArray();
_scheduledHealthChecks = _healthChecks.Where(v => v.CheckOnSchedule).ToArray();
_eventDrivenHealthChecks = GetEventDrivenHealthChecks();
_startupGracePeriodEndTime = runtimeInfo.StartTime.AddMinutes(15);
_startupGracePeriodEndTime = runtimeInfo.StartTime + TimeSpan.FromMinutes(15);
}
public List<HealthCheck> Results()
@ -77,70 +78,93 @@ namespace NzbDrone.Core.HealthCheck
.ToDictionary(g => g.Key, g => g.ToArray());
}
private void PerformHealthCheck(IProvideHealthCheck[] healthChecks, bool performServerChecks = false)
private void ProcessHealthChecks()
{
var results = healthChecks.Select(c =>
{
_logger.Trace("Check health -> {0}", c.GetType().Name);
var result = c.Check();
_logger.Trace("Check health <- {0}", c.GetType().Name);
List<IProvideHealthCheck> healthChecks;
return result;
})
.ToList();
if (performServerChecks)
lock (_pendingHealthChecks)
{
results.AddRange(_serverSideNotificationService.GetServerChecks());
healthChecks = _pendingHealthChecks.ToList();
_pendingHealthChecks.Clear();
}
foreach (var result in results)
_debounce.Pause();
try
{
if (result.Type == HealthCheckResult.Ok)
{
var previous = _healthCheckResults.Find(result.Source.Name);
if (previous != null)
var results = healthChecks.Select(c =>
{
_eventAggregator.PublishEvent(new HealthCheckRestoredEvent(previous, !_hasRunHealthChecksAfterGracePeriod));
}
_logger.Trace("Check health -> {0}", c.GetType().Name);
var result = c.Check();
_logger.Trace("Check health <- {0}", c.GetType().Name);
_healthCheckResults.Remove(result.Source.Name);
}
else
return result;
})
.ToList();
foreach (var result in results)
{
if (_healthCheckResults.Find(result.Source.Name) == null)
if (result.Type == HealthCheckResult.Ok)
{
_eventAggregator.PublishEvent(new HealthCheckFailedEvent(result, !_hasRunHealthChecksAfterGracePeriod));
}
var previous = _healthCheckResults.Find(result.Source.Name);
_healthCheckResults.Set(result.Source.Name, result);
if (previous != null)
{
_eventAggregator.PublishEvent(new HealthCheckRestoredEvent(previous, !_hasRunHealthChecksAfterGracePeriod));
}
_healthCheckResults.Remove(result.Source.Name);
}
else
{
if (_healthCheckResults.Find(result.Source.Name) == null)
{
_eventAggregator.PublishEvent(new HealthCheckFailedEvent(result, !_hasRunHealthChecksAfterGracePeriod));
}
_healthCheckResults.Set(result.Source.Name, result);
}
}
}
finally
{
_debounce.Resume();
}
_eventAggregator.PublishEvent(new HealthCheckCompleteEvent());
}
public void Execute(CheckHealthCommand message)
{
if (message.Trigger == CommandTrigger.Manual)
var healthChecks = message.Trigger == CommandTrigger.Manual ? _healthChecks : _scheduledHealthChecks;
lock (_pendingHealthChecks)
{
PerformHealthCheck(_healthChecks, true);
}
else
{
PerformHealthCheck(_scheduledHealthChecks, true);
foreach (var healthCheck in healthChecks)
{
_pendingHealthChecks.Add(healthCheck);
}
}
ProcessHealthChecks();
}
public void HandleAsync(ApplicationStartedEvent message)
{
PerformHealthCheck(_startupHealthChecks, true);
lock (_pendingHealthChecks)
{
foreach (var healthCheck in _startupHealthChecks)
{
_pendingHealthChecks.Add(healthCheck);
}
}
ProcessHealthChecks();
}
public void HandleAsync(IEvent message)
{
if (message is HealthCheckCompleteEvent)
if (message is HealthCheckCompleteEvent || message is ApplicationStartedEvent)
{
return;
}
@ -151,7 +175,16 @@ namespace NzbDrone.Core.HealthCheck
{
_isRunningHealthChecksAfterGracePeriod = true;
PerformHealthCheck(_startupHealthChecks);
lock (_pendingHealthChecks)
{
foreach (var healthCheck in _startupHealthChecks)
{
_pendingHealthChecks.Add(healthCheck);
}
}
// Call it directly so it's not debounced and any alerts can be sent.
ProcessHealthChecks();
// Update after running health checks so new failure notifications aren't sent 2x.
_hasRunHealthChecksAfterGracePeriod = true;
@ -183,11 +216,16 @@ namespace NzbDrone.Core.HealthCheck
if (eventDrivenHealthCheck.ShouldExecute(message, previouslyFailed))
{
filteredChecks.Add(eventDrivenHealthCheck.HealthCheck);
continue;
}
}
// TODO: Add debounce
PerformHealthCheck(filteredChecks.ToArray());
lock (_pendingHealthChecks)
{
filteredChecks.ForEach(h => _pendingHealthChecks.Add(h));
}
_debounce.Execute();
}
}
}

46
src/NzbDrone.Core/HealthCheck/ServerSideNotificationService.cs
View file

@ -9,50 +9,43 @@ using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Localization;
namespace NzbDrone.Core.HealthCheck
{
public interface IServerSideNotificationService
{
public List<HealthCheck> GetServerChecks();
}
public class ServerSideNotificationService : IServerSideNotificationService
public class ServerSideNotificationService : HealthCheckBase
{
private readonly IHttpClient _client;
private readonly IProwlarrCloudRequestBuilder _cloudRequestBuilder;
private readonly IConfigFileProvider _configFileProvider;
private readonly IHttpRequestBuilderFactory _cloudRequestBuilder;
private readonly Logger _logger;
private readonly ICached<List<HealthCheck>> _cache;
private readonly ICached<HealthCheck> _cache;
public ServerSideNotificationService(IHttpClient client,
IConfigFileProvider configFileProvider,
IProwlarrCloudRequestBuilder cloudRequestBuilder,
ICacheManager cacheManager,
Logger logger)
public ServerSideNotificationService(IHttpClient client, IProwlarrCloudRequestBuilder cloudRequestBuilder, IConfigFileProvider configFileProvider, ICacheManager cacheManager, ILocalizationService localizationService, Logger logger)
: base(localizationService)
{
_client = client;
_configFileProvider = configFileProvider;
_cloudRequestBuilder = cloudRequestBuilder.Services;
_cloudRequestBuilder = cloudRequestBuilder;
_logger = logger;
_cache = cacheManager.GetCache<List<HealthCheck>>(GetType());
_cache = cacheManager.GetCache<HealthCheck>(GetType());
}
public List<HealthCheck> GetServerChecks()
public override HealthCheck Check()
{
return _cache.Get("ServerChecks", RetrieveServerChecks, TimeSpan.FromHours(2));
}
private List<HealthCheck> RetrieveServerChecks()
private HealthCheck RetrieveServerChecks()
{
if (BuildInfo.IsDebug)
{
return new List<HealthCheck>();
return new HealthCheck(GetType());
}
var request = _cloudRequestBuilder.Create()
var request = _cloudRequestBuilder.Services.Create()
.Resource("/notification")
.AddQueryParam("version", BuildInfo.Version)
.AddQueryParam("os", OsInfo.Os.ToString().ToLowerInvariant())
@ -63,17 +56,22 @@ namespace NzbDrone.Core.HealthCheck
try
{
_logger.Trace("Getting server side health notifications");
_logger.Trace("Getting notifications");
var response = _client.Execute(request);
var result = Json.Deserialize<List<ServerNotificationResponse>>(response.Content);
return result.Select(x => new HealthCheck(GetType(), x.Type, x.Message, x.WikiUrl)).ToList();
var checks = result.Select(x => new HealthCheck(GetType(), x.Type, x.Message, x.WikiUrl)).ToList();
// Only one health check is supported, services returns an ordered list, so use the first one
return checks.FirstOrDefault() ?? new HealthCheck(GetType());
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to retrieve server notifications");
}
_logger.Error(ex, "Failed to retrieve notifications");
return new List<HealthCheck>();
return new HealthCheck(GetType());
}
}
}

21
src/NzbDrone.Test.Common/MockDebouncer.cs Normal file
View file

@ -0,0 +1,21 @@
using System;
using NzbDrone.Common.TPL;
namespace NzbDrone.Test.Common
{
public class MockDebouncer : Debouncer
{
public MockDebouncer(Action action, TimeSpan debounceDuration)
: base(action, debounceDuration)
{
}
public override void Execute()
{
lock (_timer)
{
_action();
}
}
}
}