From 77cdb542b68b2d112e7b706e9da82ec81a69e380 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Sun, 27 Aug 2017 00:09:15 -0700 Subject: [PATCH] Provider Status housekeeping Fixed: Clean up indexer status if stored times are in the future Fixed: Clean up download client status if stored times are in the future Closes #1396 --- ...xFutureDownloadClientStatusTimesFixture.cs | 119 ++++++++++++++++++ .../FixFutureIndexerStatusTimesFixture.cs | 119 ++++++++++++++++++ .../FixFutureRunScheduledTasksFixture.cs | 47 ------- .../NzbDrone.Core.Test.csproj | 3 +- .../FixFutureDownloadClientStatusTimes.cs | 12 ++ .../FixFutureIndexerStatusTimes.cs | 12 ++ .../FixFutureProviderStatusTimes.cs | 56 +++++++++ .../Indexers/IndexerStatusService.cs | 6 +- src/NzbDrone.Core/NzbDrone.Core.csproj | 4 + .../Status/EscalationBackOff.cs | 18 +++ .../Status/ProviderStatusServiceBase.cs | 17 +-- 11 files changed, 345 insertions(+), 68 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimesFixture.cs create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureIndexerStatusTimesFixture.cs delete mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureRunScheduledTasksFixture.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimes.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureIndexerStatusTimes.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureProviderStatusTimes.cs create mode 100644 src/NzbDrone.Core/ThingiProvider/Status/EscalationBackOff.cs diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimesFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimesFixture.cs new file mode 100644 index 000000000..96c3f0676 --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimesFixture.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Download; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class FixFutureDownloadClientStatusTimesFixture : CoreTest + { + [Test] + public void should_set_disabled_till_when_its_too_far_in_the_future() + { + var disabledTillTime = EscalationBackOff.Periods[1]; + var downloadClientStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(downloadClientStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.DisabledTill.Value < DateTime.UtcNow.AddMinutes(disabledTillTime))) + ) + ); + } + + [Test] + public void should_set_initial_failure_when_its_in_the_future() + { + var downloadClientStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(downloadClientStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.InitialFailure.Value < DateTime.UtcNow)) + ) + ); + } + + [Test] + public void should_set_most_recent_failure_when_its_in_the_future() + { + var downloadClientStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(downloadClientStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.MostRecentFailure.Value < DateTime.UtcNow)) + ) + ); + } + + [Test] + public void should_not_change_statuses_when_times_are_in_the_past() + { + var downloadClientStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 0) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(downloadClientStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.Count == 0) + ) + ); + } + + + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureIndexerStatusTimesFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureIndexerStatusTimesFixture.cs new file mode 100644 index 000000000..c25494cc3 --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureIndexerStatusTimesFixture.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FizzWare.NBuilder; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class FixFutureIndexerStatusTimesFixture : CoreTest + { + [Test] + public void should_set_disabled_till_when_its_too_far_in_the_future() + { + var disabledTillTime = EscalationBackOff.Periods[1]; + var indexerStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(indexerStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.DisabledTill.Value < DateTime.UtcNow.AddMinutes(disabledTillTime))) + ) + ); + } + + [Test] + public void should_set_initial_failure_when_its_in_the_future() + { + var indexerStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(indexerStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.InitialFailure.Value < DateTime.UtcNow)) + ) + ); + } + + [Test] + public void should_set_most_recent_failure_when_its_in_the_future() + { + var indexerStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(5)) + .With(t => t.EscalationLevel = 1) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(indexerStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.All( + s => s.MostRecentFailure.Value < DateTime.UtcNow)) + ) + ); + } + + [Test] + public void should_not_change_statuses_when_times_are_in_the_past() + { + var indexerStatuses = Builder.CreateListOfSize(5) + .All() + .With(t => t.DisabledTill = DateTime.UtcNow.AddDays(-5)) + .With(t => t.InitialFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.MostRecentFailure = DateTime.UtcNow.AddDays(-5)) + .With(t => t.EscalationLevel = 0) + .BuildListOfNew(); + + Mocker.GetMock() + .Setup(s => s.All()) + .Returns(indexerStatuses); + + Subject.Clean(); + + Mocker.GetMock() + .Verify(v => v.UpdateMany( + It.Is>(i => i.Count == 0) + ) + ); + } + + + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureRunScheduledTasksFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureRunScheduledTasksFixture.cs deleted file mode 100644 index 4235b217e..000000000 --- a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/FixFutureRunScheduledTasksFixture.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using FizzWare.NBuilder; -using FluentAssertions; -using Microsoft.Practices.ObjectBuilder2; -using NUnit.Framework; -using NzbDrone.Core.Housekeeping.Housekeepers; -using NzbDrone.Core.Jobs; -using NzbDrone.Core.Test.Framework; - -namespace NzbDrone.Core.Test.Housekeeping.Housekeepers -{ - [TestFixture] - public class FixFutureRunScheduledTasksFixture : DbTest - { - [Test] - public void should_set_last_execution_time_to_now_when_its_in_the_future() - { - var tasks = Builder.CreateListOfSize(5) - .All() - .With(t => t.LastExecution = DateTime.UtcNow.AddDays(5)) - .BuildListOfNew(); - - Db.InsertMany(tasks); - - Subject.Clean(); - - AllStoredModels.ForEach(t => t.LastExecution.Should().BeBefore(DateTime.UtcNow)); - } - - [Test] - public void should_not_change_last_execution_time_when_its_in_the_past() - { - var expectedTime = DateTime.UtcNow.AddHours(-1); - - var tasks = Builder.CreateListOfSize(5) - .All() - .With(t => t.LastExecution = expectedTime) - .BuildListOfNew(); - - Db.InsertMany(tasks); - - Subject.Clean(); - - AllStoredModels.ForEach(t => t.LastExecution.Should().Be(expectedTime)); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index d5dfb6972..9c51c5f18 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -241,7 +241,8 @@ - + + diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimes.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimes.cs new file mode 100644 index 000000000..58361e0b7 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureDownloadClientStatusTimes.cs @@ -0,0 +1,12 @@ +using NzbDrone.Core.Download; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class FixFutureDownloadClientStatusTimes : FixFutureProviderStatusTimes, IHousekeepingTask + { + public FixFutureDownloadClientStatusTimes(IDownloadClientStatusRepository downloadClientStatusRepository) + : base(downloadClientStatusRepository) + { + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureIndexerStatusTimes.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureIndexerStatusTimes.cs new file mode 100644 index 000000000..f635698d5 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureIndexerStatusTimes.cs @@ -0,0 +1,12 @@ +using NzbDrone.Core.Indexers; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class FixFutureIndexerStatusTimes : FixFutureProviderStatusTimes, IHousekeepingTask + { + public FixFutureIndexerStatusTimes(IIndexerStatusRepository indexerStatusRepository) + : base(indexerStatusRepository) + { + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureProviderStatusTimes.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureProviderStatusTimes.cs new file mode 100644 index 000000000..80bf5c8b9 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureProviderStatusTimes.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public abstract class FixFutureProviderStatusTimes where TModel : ProviderStatusBase, new() + { + private readonly IProviderStatusRepository _repo; + + protected FixFutureProviderStatusTimes(IProviderStatusRepository repo) + { + _repo = repo; + } + + public void Clean() + { + var now = DateTime.UtcNow; + var statuses = _repo.All().ToList(); + var toUpdate = new List(); + + foreach (var status in statuses) + { + var updated = false; + var escalationDelay = EscalationBackOff.Periods[status.EscalationLevel]; + var disabledTill = now.AddMinutes(escalationDelay); + + if (status.DisabledTill > disabledTill) + { + status.DisabledTill = disabledTill; + updated = true; + } + + if (status.InitialFailure > now) + { + status.InitialFailure = now; + updated = true; + } + + if (status.MostRecentFailure > now) + { + status.MostRecentFailure = now; + updated = true; + } + + if (updated) + { + toUpdate.Add(status); + } + } + + _repo.UpdateMany(toUpdate); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 6d3b89545..08baa51a5 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -1,10 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using NLog; +using NLog; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.ThingiProvider.Events; using NzbDrone.Core.ThingiProvider.Status; namespace NzbDrone.Core.Indexers diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index e7af7236f..4a2beab42 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -607,6 +607,9 @@ + + + @@ -1108,6 +1111,7 @@ + diff --git a/src/NzbDrone.Core/ThingiProvider/Status/EscalationBackOff.cs b/src/NzbDrone.Core/ThingiProvider/Status/EscalationBackOff.cs new file mode 100644 index 000000000..304613d58 --- /dev/null +++ b/src/NzbDrone.Core/ThingiProvider/Status/EscalationBackOff.cs @@ -0,0 +1,18 @@ +namespace NzbDrone.Core.ThingiProvider.Status +{ + public static class EscalationBackOff + { + public static readonly int[] Periods = + { + 0, + 5 * 60, + 15 * 60, + 30 * 60, + 60 * 60, + 3 * 60 * 60, + 6 * 60 * 60, + 12 * 60 * 60, + 24 * 60 * 60 + }; + } +} diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs index 7c5710e7b..b74153947 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs @@ -3,7 +3,6 @@ using System.Linq; using NLog; using NzbDrone.Core.Messaging.Events; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.ThingiProvider.Status @@ -21,25 +20,13 @@ public abstract class ProviderStatusServiceBase : IProviderSt where TProvider : IProvider where TModel : ProviderStatusBase, new() { - private static readonly int[] EscalationBackOffPeriods = { - 0, - 5 * 60, - 15 * 60, - 30 * 60, - 60 * 60, - 3 * 60 * 60, - 6 * 60 * 60, - 12 * 60 * 60, - 24 * 60 * 60 - }; - protected readonly object _syncRoot = new object(); protected readonly IProviderStatusRepository _providerStatusRepository; protected readonly IEventAggregator _eventAggregator; protected readonly Logger _logger; - protected int MaximumEscalationLevel { get; set; } = EscalationBackOffPeriods.Length - 1; + protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1; protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero; public ProviderStatusServiceBase(IProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) @@ -63,7 +50,7 @@ protected virtual TimeSpan CalculateBackOffPeriod(TModel status) { var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel); - return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]); + return TimeSpan.FromSeconds(EscalationBackOff.Periods[level]); } public virtual void RecordSuccess(int providerId)