From ce59db528b95a77c015c37df7d0f786b34592290 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Tue, 25 Dec 2018 00:22:35 +0100 Subject: [PATCH 1/6] Fixed: Mono bug causing memory leakage when http connections use gzip compression. The bug is registered upstream, but this commit works around the problem by doing the gzip decompression separately from the http stack. Ref #2296 --- .../Extensions/Pipelines/GZipPipeline.cs | 34 ++++++++++++++----- .../Http/Dispatchers/ManagedHttpDispatcher.cs | 33 +++++++++++++++--- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/src/NzbDrone.Api/Extensions/Pipelines/GZipPipeline.cs b/src/NzbDrone.Api/Extensions/Pipelines/GZipPipeline.cs index 8aa9f4ad2..2366b80ac 100644 --- a/src/NzbDrone.Api/Extensions/Pipelines/GZipPipeline.cs +++ b/src/NzbDrone.Api/Extensions/Pipelines/GZipPipeline.cs @@ -5,6 +5,7 @@ using Nancy; using Nancy.Bootstrapper; using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; namespace NzbDrone.Api.Extensions.Pipelines @@ -15,9 +16,14 @@ public class GzipCompressionPipeline : IRegisterNancyPipeline public int Order => 0; + private readonly Action, Stream> _writeGZipStream; + public GzipCompressionPipeline(Logger logger) { _logger = logger; + + // On Mono GZipStream/DeflateStream leaks memory if an exception is thrown, use an intermediate buffer in that case. + _writeGZipStream = PlatformInfo.IsMono ? WriteGZipStreamMono : (Action, Stream>)WriteGZipStream; } public void Register(IPipelines pipelines) @@ -43,14 +49,7 @@ private void CompressResponse(NancyContext context) var contents = response.Contents; response.Headers["Content-Encoding"] = "gzip"; - response.Contents = responseStream => - { - using (var gzip = new GZipStream(responseStream, CompressionMode.Compress, true)) - using (var buffered = new BufferedStream(gzip, 8192)) - { - contents.Invoke(buffered); - } - }; + response.Contents = responseStream => _writeGZipStream(contents, responseStream); } } @@ -61,6 +60,25 @@ private void CompressResponse(NancyContext context) } } + private static void WriteGZipStreamMono(Action innerContent, Stream targetStream) + { + using (var membuffer = new MemoryStream()) + { + WriteGZipStream(innerContent, membuffer); + membuffer.Position = 0; + membuffer.CopyTo(targetStream); + } + } + + private static void WriteGZipStream(Action innerContent, Stream targetStream) + { + using (var gzip = new GZipStream(targetStream, CompressionMode.Compress, true)) + using (var buffered = new BufferedStream(gzip, 8192)) + { + innerContent.Invoke(buffered); + } + } + private static bool ContentLengthIsTooSmall(Response response) { var contentLength = response.Headers.GetValueOrDefault("Content-Length"); diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index 70ee66390..0970a332c 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.IO.Compression; using System.Net; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; @@ -25,11 +26,20 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); - // Deflate is not a standard and could break depending on implementation. - // we should just stick with the more compatible Gzip - //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net - webRequest.AutomaticDecompression = DecompressionMethods.GZip; - + if (PlatformInfo.IsMono) + { + // On Mono GZipStream/DeflateStream leaks memory if an exception is thrown, use an intermediate buffer in that case. + webRequest.AutomaticDecompression = DecompressionMethods.None; + webRequest.Headers.Add("Accept-Encoding", "gzip"); + } + else + { + // Deflate is not a standard and could break depending on implementation. + // we should just stick with the more compatible Gzip + //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net + webRequest.AutomaticDecompression = DecompressionMethods.GZip; + } + webRequest.Method = request.Method.ToString(); webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); webRequest.KeepAlive = request.ConnectionKeepAlive; @@ -107,6 +117,19 @@ public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) try { data = responseStream.ToBytes(); + + if (PlatformInfo.IsMono && httpWebResponse.ContentEncoding == "gzip") + { + using (var compressedStream = new MemoryStream(data)) + using (var gzip = new GZipStream(compressedStream, CompressionMode.Decompress)) + using (var decompressedStream = new MemoryStream()) + { + gzip.CopyTo(decompressedStream); + data = decompressedStream.ToArray(); + } + + httpWebResponse.Headers.Remove("Content-Encoding"); + } } catch (Exception ex) { From 04900e5f906bf6e1aa13b6dbb0877db39d5603da Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 12 Jan 2019 11:12:57 +0100 Subject: [PATCH 2/6] Tweaked reverse title detection to handle triple digit episode numbers. fixes #2871 --- src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs | 7 +++++++ src/NzbDrone.Core/Parser/Parser.cs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs b/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs index 7c028016a..6dcf2dbde 100644 --- a/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs @@ -87,6 +87,13 @@ public class HashedReleaseFixture : CoreTest "Fargo", Quality.WEBDL1080p, "RARBG" + }, + new object[] + { + @"C:\Test\XxQVHK4GJMP3n2dLpmhW\XxQVHK4GJMP3n2dLpmhW\MKV\010E70S.yhcranA.fo.snoS.mkv".AsOsAgnostic(), + "Sons of Anarchy", + Quality.HDTV720p, + "" } }; diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 884d2bec2..7c70b28b7 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -290,7 +290,7 @@ public static class Parser }; //Regex to detect whether the title was reversed. - private static readonly Regex ReversedTitleRegex = new Regex(@"[-._ ](p027|p0801|\d{2}E\d{2}S)[-._ ]", RegexOptions.Compiled); + private static readonly Regex ReversedTitleRegex = new Regex(@"[-._ ](p027|p0801|\d{2,3}E\d{2}S)[-._ ]", RegexOptions.Compiled); private static readonly Regex NormalizeRegex = new Regex(@"((?:\b|_)(? Date: Sat, 12 Jan 2019 11:23:47 +0100 Subject: [PATCH 3/6] Added 10-bit to parser cleanup. fixes #2870 --- .../AbsoluteEpisodeNumberParserFixture.cs | 12 ++++++++++++ src/NzbDrone.Core/Parser/Parser.cs | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs index d35b74ee7..0e18c4d1f 100644 --- a/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs @@ -136,5 +136,17 @@ public void should_parse_multi_episode_absolute_numbers(string postTitle, string result.SeriesTitle.Should().Be(title); result.FullSeason.Should().BeFalse(); } + + [TestCase("[Vivid] Living Sky Saga S01 [Web][MKV][h264 10-bit][1080p][AAC 2.0]", "Living Sky Saga", 1)] + public void should_parse_anime_season_packs(string postTitle, string title, int seasonNumber) + { + var result = Parser.Parser.ParseTitle(postTitle); + result.Should().NotBeNull(); + result.AbsoluteEpisodeNumbers.Should().BeEmpty(); + result.SeriesTitle.Should().Be(title); + result.FullSeason.Should().BeTrue(); + result.SeasonNumber.Should().Be(seasonNumber); + } + } } diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 7c70b28b7..5353ff008 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -298,7 +298,7 @@ public static class Parser private static readonly Regex FileExtensionRegex = new Regex(@"\.[a-z0-9]{2,4}$", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex SimpleTitleRegex = new Regex(@"(?:(480|720|1080|2160)[ip]|[xh][\W_]?26[45]|DD\W?5\W1|[<>?*:|]|848x480|1280x720|1920x1080|3840x2160|4096x2160|(8|10)b(it)?)\s*?", + private static readonly Regex SimpleTitleRegex = new Regex(@"(?:(480|720|1080|2160)[ip]|[xh][\W_]?26[45]|DD\W?5\W1|[<>?*:|]|848x480|1280x720|1920x1080|3840x2160|4096x2160|(8|10)b(it)?|10-bit)\s*?", RegexOptions.IgnoreCase | RegexOptions.Compiled); private static readonly Regex WebsitePrefixRegex = new Regex(@"^\[\s*[a-z]+(\.[a-z]+)+\s*\][- ]*|^www\.[a-z]+\.(?:com|net)[ -]*", From 2b4429f8b7931376f11caae07fa9cb8d56d2b522 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sat, 12 Jan 2019 12:18:57 +0100 Subject: [PATCH 4/6] Fixed: Erroneously matching Anime 10.5 special as 10. fixes #2868 --- .../AbsoluteEpisodeNumberParserFixture.cs | 12 +++++ .../Parser/Model/ParsedEpisodeInfo.cs | 2 + src/NzbDrone.Core/Parser/Parser.cs | 54 +++++++++++-------- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs index 0e18c4d1f..9670fdd67 100644 --- a/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/AbsoluteEpisodeNumberParserFixture.cs @@ -148,5 +148,17 @@ public void should_parse_anime_season_packs(string postTitle, string title, int result.SeasonNumber.Should().Be(seasonNumber); } + [TestCase("[HorribleSubs] Goblin Slayer - 10.5 [1080p].mkv", "Goblin Slayer", 10.5)] + public void should_handle_anime_recap_numbering(string postTitle, string title, double specialEpisodeNumber) + { + var result = Parser.Parser.ParseTitle(postTitle); + result.Should().NotBeNull(); + result.SeriesTitle.Should().Be(title); + result.AbsoluteEpisodeNumbers.Should().BeEmpty(); + result.SpecialAbsoluteEpisodeNumbers.Should().NotBeEmpty(); + result.SpecialAbsoluteEpisodeNumbers.Should().BeEquivalentTo(new[] { (decimal)specialEpisodeNumber }); + result.FullSeason.Should().BeFalse(); + } + } } diff --git a/src/NzbDrone.Core/Parser/Model/ParsedEpisodeInfo.cs b/src/NzbDrone.Core/Parser/Model/ParsedEpisodeInfo.cs index 6dc338dcf..449f68017 100644 --- a/src/NzbDrone.Core/Parser/Model/ParsedEpisodeInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ParsedEpisodeInfo.cs @@ -13,6 +13,7 @@ public class ParsedEpisodeInfo public int SeasonNumber { get; set; } public int[] EpisodeNumbers { get; set; } public int[] AbsoluteEpisodeNumbers { get; set; } + public decimal[] SpecialAbsoluteEpisodeNumbers { get; set; } public string AirDate { get; set; } public Language Language { get; set; } public bool FullSeason { get; set; } @@ -27,6 +28,7 @@ public ParsedEpisodeInfo() { EpisodeNumbers = new int[0]; AbsoluteEpisodeNumbers = new int[0]; + SpecialAbsoluteEpisodeNumbers = new decimal[0]; } public bool IsDaily diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 5353ff008..2adc2f4ef 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text.RegularExpressions; @@ -39,15 +40,15 @@ public static class Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title Episode Absolute Episode Number ([SubGroup] Series Title Episode 01) - new Regex(@"^(?:\[(?.+?)\][-_. ]?)(?.+?)[-_. ](?:Episode)(?:[-_. ]+(?<absoluteepisode>(?<!\d+)\d{2,3}(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", + new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)(?<title>.+?)[-_. ](?:Episode)(?:[-_. ]+(?<absoluteepisode>(?<!\d+)\d{2,3}(\.\d{1,2})?(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title Absolute Episode Number + Season+Episode - new Regex(@"^(?:\[(?<subgroup>.+?)\](?:_|-|\s|\.)?)(?<title>.+?)(?:(?:[-_\W](?<![()\[!]))+(?<absoluteepisode>\d{2,3}))+(?:_|-|\s|\.)+(?:S?(?<season>(?<!\d+)\d{1,2}(?!\d+))(?:(?:\-|[ex]|\W[ex]){1,2}(?<episode>\d{2}(?!\d+)))+).*?(?<hash>[(\[]\w{8}[)\]])?(?:$|\.)", + new Regex(@"^(?:\[(?<subgroup>.+?)\](?:_|-|\s|\.)?)(?<title>.+?)(?:(?:[-_\W](?<![()\[!]))+(?<absoluteepisode>\d{2,3}(\.\d{1,2})?))+(?:_|-|\s|\.)+(?:S?(?<season>(?<!\d+)\d{1,2}(?!\d+))(?:(?:\-|[ex]|\W[ex]){1,2}(?<episode>\d{2}(?!\d+)))+).*?(?<hash>[(\[]\w{8}[)\]])?(?:$|\.)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title Season+Episode + Absolute Episode Number - new Regex(@"^(?:\[(?<subgroup>.+?)\](?:_|-|\s|\.)?)(?<title>.+?)(?:[-_\W](?<![()\[!]))+(?:S?(?<season>(?<!\d+)\d{1,2}(?!\d+))(?:(?:\-|[ex]|\W[ex]){1,2}(?<episode>\d{2}(?!\d+)))+)(?:(?:_|-|\s|\.)+(?<absoluteepisode>(?<!\d+)\d{2,3}(?!\d+)))+.*?(?<hash>\[\w{8}\])?(?:$|\.)", + new Regex(@"^(?:\[(?<subgroup>.+?)\](?:_|-|\s|\.)?)(?<title>.+?)(?:[-_\W](?<![()\[!]))+(?:S?(?<season>(?<!\d+)\d{1,2}(?!\d+))(?:(?:\-|[ex]|\W[ex]){1,2}(?<episode>\d{2}(?!\d+)))+)(?:(?:_|-|\s|\.)+(?<absoluteepisode>(?<!\d+)\d{2,3}(\.\d{1,2})?(?!\d+)))+.*?(?<hash>\[\w{8}\])?(?:$|\.)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title Season+Episode @@ -55,15 +56,15 @@ public static class Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title with trailing number Absolute Episode Number - new Regex(@"^\[(?<subgroup>.+?)\][-_. ]?(?<title>[^-]+?\d+?)[-_. ]+(?:[-_. ]?(?<absoluteepisode>\d{3}(?!\d+)))+(?:[-_. ]+(?<special>special|ova|ovd))?.*?(?<hash>\[\w{8}\])?(?:$|\.mkv)", + new Regex(@"^\[(?<subgroup>.+?)\][-_. ]?(?<title>[^-]+?\d+?)[-_. ]+(?:[-_. ]?(?<absoluteepisode>\d{3}(\.\d{1,2})?(?!\d+)))+(?:[-_. ]+(?<special>special|ova|ovd))?.*?(?<hash>\[\w{8}\])?(?:$|\.mkv)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title - Absolute Episode Number - new Regex(@"^\[(?<subgroup>.+?)\][-_. ]?(?<title>.+?)(?:[. ]-[. ](?<absoluteepisode>\d{2,3}(?!\d+|[-])))+(?:[-_. ]+(?<special>special|ova|ovd))?.*?(?<hash>\[\w{8}\])?(?:$|\.mkv)", + new Regex(@"^\[(?<subgroup>.+?)\][-_. ]?(?<title>.+?)(?:[. ]-[. ](?<absoluteepisode>\d{2,3}(\.\d{1,2})?(?!\d+|[-])))+(?:[-_. ]+(?<special>special|ova|ovd))?.*?(?<hash>\[\w{8}\])?(?:$|\.mkv)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - [SubGroup] Title Absolute Episode Number - new Regex(@"^\[(?<subgroup>.+?)\][-_. ]?(?<title>.+?)[-_. ]+\(?(?:[-_. ]?#?(?<absoluteepisode>\d{2,3}(?!\d+)))+\)?(?:[-_. ]+(?<special>special|ova|ovd))?.*?(?<hash>\[\w{8}\])?(?:$|\.mkv)", + new Regex(@"^\[(?<subgroup>.+?)\][-_. ]?(?<title>.+?)[-_. ]+\(?(?:[-_. ]?#?(?<absoluteepisode>\d{2,3}(\.\d{1,2})?(?!\d+)))+\)?(?:[-_. ]+(?<special>special|ova|ovd))?.*?(?<hash>\[\w{8}\])?(?:$|\.mkv)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Multi-episode Repeated (S01E05 - S01E06, 1x05 - 1x06, etc) @@ -75,7 +76,7 @@ public static class Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - Title Season EpisodeNumber + Absolute Episode Number [SubGroup] - new Regex(@"^(?<title>.+?)(?:[-_\W](?<![()\[!]))+(?:S?(?<season>(?<!\d+)\d{1,2}(?!\d+))(?:(?:[ex]|\W[ex]){1,2}(?<episode>(?<!\d+)\d{2}(?!\d+)))).+?(?:[-_. ]?(?<absoluteepisode>(?<!\d+)\d{3}(?!\d+)))+.+?\[(?<subgroup>.+?)\](?:$|\.mkv)", + new Regex(@"^(?<title>.+?)(?:[-_\W](?<![()\[!]))+(?:S?(?<season>(?<!\d+)\d{1,2}(?!\d+))(?:(?:[ex]|\W[ex]){1,2}(?<episode>(?<!\d+)\d{2}(?!\d+)))).+?(?:[-_. ]?(?<absoluteepisode>(?<!\d+)\d{3}(\.\d{1,2})?(?!\d+)))+.+?\[(?<subgroup>.+?)\](?:$|\.mkv)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Multi-Episode with a title (S01E05E06, S01E05-06, S01E05 E06, etc) and trailing info in slashes @@ -83,11 +84,11 @@ public static class Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - Title Absolute Episode Number [SubGroup] - new Regex(@"^(?<title>.+?)(?:(?:_|-|\s|\.)+(?<absoluteepisode>\d{3}(?!\d+)))+(?:.+?)\[(?<subgroup>.+?)\].*?(?<hash>\[\w{8}\])?(?:$|\.)", + new Regex(@"^(?<title>.+?)(?:(?:_|-|\s|\.)+(?<absoluteepisode>\d{3}(\.\d{1,2})?(?!\d+)))+(?:.+?)\[(?<subgroup>.+?)\].*?(?<hash>\[\w{8}\])?(?:$|\.)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - Title Absolute Episode Number [Hash] - new Regex(@"^(?<title>.+?)(?:(?:_|-|\s|\.)+(?<absoluteepisode>\d{2,3}(?!\d+)))+(?:[-_. ]+(?<special>special|ova|ovd))?[-_. ]+.*?(?<hash>\[\w{8}\])(?:$|\.)", + new Regex(@"^(?<title>.+?)(?:(?:_|-|\s|\.)+(?<absoluteepisode>\d{2,3}(\.\d{1,2})?(?!\d+)))+(?:[-_. ]+(?<special>special|ova|ovd))?[-_. ]+.*?(?<hash>\[\w{8}\])(?:$|\.)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Episodes with airdate AND season/episode number, capture season/epsiode only @@ -171,11 +172,11 @@ public static class Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), // Anime - Title with season number - Absolute Episode Number (Title S01 - EP14) - new Regex(@"^(?<title>.+?S\d{1,2})[-_. ]{3,}(?:EP)?(?<absoluteepisode>\d{2,3}(?!\d+|[-]))", + new Regex(@"^(?<title>.+?S\d{1,2})[-_. ]{3,}(?:EP)?(?<absoluteepisode>\d{2,3}(\.\d{1,2})?(?!\d+|[-]))", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Anime - French titles with single episode numbers, with or without leading sub group ([RlsGroup] Title - Episode 1) - new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)[-_. ]+?(?:Episode[-_. ]+?)(?<absoluteepisode>\d{1}(?!\d+))", + new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)[-_. ]+?(?:Episode[-_. ]+?)(?<absoluteepisode>\d{1}(\.\d{1,2})?(?!\d+))", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Season only releases @@ -230,19 +231,19 @@ public static class Parser // TODO: THIS ONE //Anime - Title Absolute Episode Number (e66) - new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)(?:(?:_|-|\s|\.)+(?:e|ep)(?<absoluteepisode>\d{2,3}))+.*?(?<hash>\[\w{8}\])?(?:$|\.)", + new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)(?:(?:_|-|\s|\.)+(?:e|ep)(?<absoluteepisode>\d{2,3}(\.\d{1,2})?))+.*?(?<hash>\[\w{8}\])?(?:$|\.)", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - Title Episode Absolute Episode Number (Series Title Episode 01) - new Regex(@"^(?<title>.+?)[-_. ](?:Episode)(?:[-_. ]+(?<absoluteepisode>(?<!\d+)\d{2,3}(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", + new Regex(@"^(?<title>.+?)[-_. ](?:Episode)(?:[-_. ]+(?<absoluteepisode>(?<!\d+)\d{2,3}(\.\d{1,2})?(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - Title Absolute Episode Number - new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)(?:[-_. ]+(?<absoluteepisode>(?<!\d+)\d{2,3}(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", + new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)(?:[-_. ]+(?<absoluteepisode>(?<!\d+)\d{2,3}(\.\d{1,2})?(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Anime - Title {Absolute Episode Number} - new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)(?:(?:[-_\W](?<![()\[!]))+(?<absoluteepisode>(?<!\d+)\d{2,3}(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", + new Regex(@"^(?:\[(?<subgroup>.+?)\][-_. ]?)?(?<title>.+?)(?:(?:[-_\W](?<![()\[!]))+(?<absoluteepisode>(?<!\d+)\d{2,3}(\.\d{1,2})?(?!\d+)))+(?:_|-|\s|\.)*?(?<hash>\[.{8}\])?(?:$|\.)?", RegexOptions.IgnoreCase | RegexOptions.Compiled), //Extant, terrible multi-episode naming (extant.10708.hdtv-lol.mp4) @@ -653,21 +654,32 @@ private static ParsedEpisodeInfo ParseMatchCollection(MatchCollection matchColle if (absoluteEpisodeCaptures.Any()) { - var first = Convert.ToInt32(absoluteEpisodeCaptures.First().Value); - var last = Convert.ToInt32(absoluteEpisodeCaptures.Last().Value); + var first = Convert.ToDecimal(absoluteEpisodeCaptures.First().Value, CultureInfo.InvariantCulture); + var last = Convert.ToDecimal(absoluteEpisodeCaptures.Last().Value, CultureInfo.InvariantCulture); if (first > last) { return null; } - var count = last - first + 1; - result.AbsoluteEpisodeNumbers = Enumerable.Range(first, count).ToArray(); - - if (matchGroup.Groups["special"].Success) + if ((first % 1) != 0 || (last % 1) != 0) { + if (absoluteEpisodeCaptures.Count != 1) + return null; // Multiple matches not allowed for specials + + result.SpecialAbsoluteEpisodeNumbers = new decimal[] { first }; result.Special = true; } + else + { + var count = last - first + 1; + result.AbsoluteEpisodeNumbers = Enumerable.Range((int)first, (int)count).ToArray(); + + if (matchGroup.Groups["special"].Success) + { + result.Special = true; + } + } } if (!episodeCaptures.Any() && !absoluteEpisodeCaptures.Any()) From 00283e3d6e75bb97b98e24af6b3afcf20266a5ff Mon Sep 17 00:00:00 2001 From: Taloth Saldono <Taloth@users.noreply.github.com> Date: Sat, 12 Jan 2019 12:56:11 +0100 Subject: [PATCH 5/6] New: Limit indexer/download client backoff to 5 min during the first 15 min of application start. closes #2366 --- .../EnvironmentInfo/IRuntimeInfo.cs | 3 ++ .../EnvironmentInfo/RuntimeInfo.cs | 9 ++++ .../DownloadClientStatusServiceFixture.cs | 7 ++- .../IndexerStatusServiceFixture.cs | 7 ++- .../ProviderStatusServiceFixture.cs | 49 +++++++++++++++++-- .../Download/DownloadClientStatusService.cs | 7 +-- .../Indexers/IndexerStatusService.cs | 7 +-- .../Status/ProviderStatusServiceBase.cs | 18 ++++++- 8 files changed, 93 insertions(+), 14 deletions(-) diff --git a/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs b/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs index d387001ef..729b169e1 100644 --- a/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs +++ b/src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs @@ -1,7 +1,10 @@ +using System; + namespace NzbDrone.Common.EnvironmentInfo { public interface IRuntimeInfo { + DateTime StartTime { get; } bool IsUserInteractive { get; } bool IsAdmin { get; } bool IsWindowsService { get; } diff --git a/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs b/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs index 3337b99b8..ad82d0cad 100644 --- a/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs +++ b/src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs @@ -12,6 +12,7 @@ namespace NzbDrone.Common.EnvironmentInfo public class RuntimeInfo : IRuntimeInfo { private readonly Logger _logger; + private readonly DateTime _startTime = DateTime.UtcNow; public RuntimeInfo(IServiceProvider serviceProvider, Logger logger) { @@ -37,6 +38,14 @@ static RuntimeInfo() IsProduction = InternalIsProduction(); } + public DateTime StartTime + { + get + { + return _startTime; + } + } + public static bool IsUserInteractive => Environment.UserInteractive; bool IRuntimeInfo.IsUserInteractive => IsUserInteractive; diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs index 08aca1cdb..a608321d9 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Download; using NzbDrone.Core.Test.Framework; @@ -16,6 +17,10 @@ public class DownloadClientStatusServiceFixture : CoreTest<DownloadClientStatusS public void SetUp() { _epoch = DateTime.UtcNow; + + Mocker.GetMock<IRuntimeInfo>() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); } private DownloadClientStatus WithStatus(DownloadClientStatus status) diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs index 646212116..55ed8abfb 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Indexers; using NzbDrone.Core.Test.Framework; @@ -16,6 +17,10 @@ public class IndexerStatusServiceFixture : CoreTest<IndexerStatusService> public void SetUp() { _epoch = DateTime.UtcNow; + + Mocker.GetMock<IRuntimeInfo>() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); } private void WithStatus(IndexerStatus status) diff --git a/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs index 32a9c4b7a..80d533ed2 100644 --- a/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs @@ -1,9 +1,10 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; using NLog; using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Test.Framework; using NzbDrone.Core.ThingiProvider; @@ -25,8 +26,8 @@ public interface IMockProviderStatusRepository : IProviderStatusRepository<MockP public class MockProviderStatusService : ProviderStatusServiceBase<IMockProvider, MockProviderStatus> { - public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) - : base(providerStatusRepository, eventAggregator, logger) + public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) { } @@ -40,9 +41,20 @@ public class ProviderStatusServiceFixture : CoreTest<MockProviderStatusService> public void SetUp() { _epoch = DateTime.UtcNow; + + Mocker.GetMock<IRuntimeInfo>() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); } - private void WithStatus(MockProviderStatus status) + private void GivenRecentStartup() + { + Mocker.GetMock<IRuntimeInfo>() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromMinutes(12)); + } + + private MockProviderStatus WithStatus(MockProviderStatus status) { Mocker.GetMock<IMockProviderStatusRepository>() .Setup(v => v.FindByProviderId(1)) @@ -51,6 +63,8 @@ private void WithStatus(MockProviderStatus status) Mocker.GetMock<IMockProviderStatusRepository>() .Setup(v => v.All()) .Returns(new[] { status }); + + return status; } private void VerifyUpdate() @@ -122,5 +136,32 @@ public void should_preserve_escalation_on_intermittent_success() status.DisabledTill.Should().HaveValue(); status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500); } + + [Test] + public void should_not_escalate_further_till_after_5_minutes_since_startup() + { + GivenRecentStartup(); + + var origStatus = WithStatus(new MockProviderStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + + origStatus.EscalationLevel.Should().Be(3); + status.DisabledTill.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500); + } } } diff --git a/src/NzbDrone.Core/Download/DownloadClientStatusService.cs b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs index b4fd0f83c..11eecfe89 100644 --- a/src/NzbDrone.Core/Download/DownloadClientStatusService.cs +++ b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs @@ -1,5 +1,6 @@ -using System; +using System; using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider.Status; @@ -12,8 +13,8 @@ public interface IDownloadClientStatusService : IProviderStatusServiceBase<Downl public class DownloadClientStatusService : ProviderStatusServiceBase<IDownloadClient, DownloadClientStatus>, IDownloadClientStatusService { - public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) - : base(providerStatusRepository, eventAggregator, logger) + public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) { MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5); MaximumEscalationLevel = 5; diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 08baa51a5..1bdf81533 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -1,4 +1,5 @@ -using NLog; +using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider.Status; @@ -14,8 +15,8 @@ public interface IIndexerStatusService : IProviderStatusServiceBase<IndexerStatu public class IndexerStatusService : ProviderStatusServiceBase<IIndexer, IndexerStatus>, IIndexerStatusService { - public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) - : base(providerStatusRepository, eventAggregator, logger) + public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) { } diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs index 500559072..129617d3b 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using NLog; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider.Events; @@ -24,15 +25,18 @@ public abstract class ProviderStatusServiceBase<TProvider, TModel> : IProviderSt protected readonly IProviderStatusRepository<TModel> _providerStatusRepository; protected readonly IEventAggregator _eventAggregator; + protected readonly IRuntimeInfo _runtimeInfo; protected readonly Logger _logger; protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1; protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero; + protected TimeSpan MinimumTimeSinceStartup { get; set; } = TimeSpan.FromMinutes(15); - public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) { _providerStatusRepository = providerStatusRepository; _eventAggregator = eventAggregator; + _runtimeInfo = runtimeInfo; _logger = logger; } @@ -89,9 +93,10 @@ protected virtual void RecordFailure(int providerId, TimeSpan minimumBackOff, bo escalate = false; } + var inStartupGracePeriod = (_runtimeInfo.StartTime + MinimumTimeSinceStartup) > now; var inGracePeriod = (status.InitialFailure.Value + MinimumTimeSinceInitialFailure) > now; - if (escalate && !inGracePeriod) + if (escalate && !inGracePeriod && !inStartupGracePeriod) { status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1); } @@ -109,6 +114,15 @@ protected virtual void RecordFailure(int providerId, TimeSpan minimumBackOff, bo status.DisabledTill = now + CalculateBackOffPeriod(status); } + if (inStartupGracePeriod && minimumBackOff == TimeSpan.Zero && status.DisabledTill.HasValue) + { + var maximumDisabledTill = now + TimeSpan.FromSeconds(EscalationBackOff.Periods[1]); + if (maximumDisabledTill < status.DisabledTill) + { + status.DisabledTill = maximumDisabledTill; + } + } + _providerStatusRepository.Upsert(status); _eventAggregator.PublishEvent(new ProviderStatusChangedEvent<TProvider>(providerId, status)); From 779ab39f50b73adf2001b4fd3e5fe802702fc0eb Mon Sep 17 00:00:00 2001 From: Taloth Saldono <Taloth@users.noreply.github.com> Date: Sat, 12 Jan 2019 13:30:08 +0100 Subject: [PATCH 6/6] Fixed failing test --- src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs b/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs index 6dcf2dbde..6eec4058f 100644 --- a/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/HashedReleaseFixture.cs @@ -93,7 +93,7 @@ public class HashedReleaseFixture : CoreTest @"C:\Test\XxQVHK4GJMP3n2dLpmhW\XxQVHK4GJMP3n2dLpmhW\MKV\010E70S.yhcranA.fo.snoS.mkv".AsOsAgnostic(), "Sons of Anarchy", Quality.HDTV720p, - "" + null } };