From 40ecdbc12de8b320a4d650aea65a36e8edea77d8 Mon Sep 17 00:00:00 2001 From: bakerboy448 <55419169+bakerboy448@users.noreply.github.com> Date: Tue, 5 Apr 2022 19:10:26 -0500 Subject: [PATCH] New: Support for new Nyaa RSS Feed format Closes #4614 --- .../Files/Indexers/Nyaa/Nyaa2021.xml | 66 +++++++++++++++++ .../IndexerTests/NyaaTests/NyaaFixture.cs | 43 ++++++++++-- .../Indexers/EzrssTorrentRssParser.cs | 49 ++----------- src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs | 2 +- .../Indexers/TorrentRssParser.cs | 70 ++++++++++++++++--- 5 files changed, 170 insertions(+), 60 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml diff --git a/src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml b/src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml new file mode 100644 index 000000000..0d63ce046 --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml @@ -0,0 +1,66 @@ + + + Nyaa - Home - Torrent File RSS + RSS Feed for Home + https://nyaa.si/ + + + [Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv + https://nyaa.si/download/1424896.torrent + https://nyaa.si/view/1424896 + Tue, 24 Aug 2021 22:18:46 -0000 + 4 + 3 + 2 + e8ca5e20eca876339f41c3d9e95ea66c1d7caaee + 1_3 + Anime - Non-English-translated + 609.6 MiB + 0 + No + No + + #1424896 | [Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv | 609.6 MiB | Anime - Non-English-translated | E8CA5E20ECA876339F41C3D9E95EA66C1D7CAAEE ]]> + + + + Macross Zero (BDRip 1920x1080p x265 HEVC TrueHD, FLAC 5.1+2.0)[sxales] + https://nyaa.si/download/1424895.torrent + https://nyaa.si/view/1424895 + Tue, 24 Aug 2021 22:03:11 -0000 + 23 + 32 + 17 + 26f37f26d5b3475b41a98dc575fabfa6f8d32a76 + 1_2 + Anime - English-translated + 5.7 GiB + 2 + No + No + + #1424895 | Macross Zero (BDRip 1920x1080p x265 HEVC TrueHD, FLAC 5.1+2.0)[sxales] | 5.7 GiB | Anime - English-translated | 26F37F26D5B3475B41A98DC575FABFA6F8D32A76 ]]> + + + + Fumetsu no Anata e - 19 [WEBDL 1080p] Ukr DVO + https://nyaa.si/download/1424887.torrent + https://nyaa.si/view/1424887 + Tue, 24 Aug 2021 21:23:06 -0000 + 5 + 4 + 4 + 3e4300e24b39983802162877755aab4380bd137a + 1_3 + Anime - Non-English-translated + 1.4 GiB + 0 + No + No + + #1424887 | Fumetsu no Anata e - 19 [WEBDL 1080p] Ukr DVO | 1.4 GiB | Anime - Non-English-translated | 3E4300E24B39983802162877755AAB4380BD137A ]]> + + + + \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs index 3ceb43a55..5b317f8e9 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs @@ -18,13 +18,15 @@ public class NyaaFixture : CoreTest public void Setup() { Subject.Definition = new IndexerDefinition() - { - Name = "Nyaa", - Settings = new NyaaSettings() - }; + { + Name = "Nyaa", + Settings = new NyaaSettings() + }; } - [Test] +/* [Test] + // Legacy Nyaa feed test + public void should_parse_recent_feed_from_Nyaa() { var recentFeed = ReadAllText(@"Files/Indexers/Nyaa/Nyaa.xml"); @@ -50,8 +52,37 @@ public void should_parse_recent_feed_from_Nyaa() torrentInfo.Size.Should().Be(2523293286); //2.35 GiB torrentInfo.InfoHash.Should().Be(null); torrentInfo.MagnetUrl.Should().Be(null); - torrentInfo.Peers.Should().Be(2+1); + torrentInfo.Peers.Should().Be(2 + 1); torrentInfo.Seeders.Should().Be(1); + }*/ + + [Test] + public void should_parse_2021_recent_feed_from_Nyaa() + { + var recentFeed = ReadAllText(@"Files/Indexers/Nyaa/Nyaa2021.xml"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(3); + releases.First().Should().BeOfType(); + + var torrentInfo = releases.First() as TorrentInfo; + + torrentInfo.Title.Should().Be("[Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("https://nyaa.si/download/1424896.torrent"); + torrentInfo.InfoUrl.Should().Be("https://nyaa.si/view/1424896"); + torrentInfo.CommentUrl.Should().BeNullOrEmpty(); + torrentInfo.Indexer.Should().Be(Subject.Definition.Name); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("Tue, 24 Aug 2021 22:18:46")); + torrentInfo.Size.Should().Be(639211930); //609.6 MiB + torrentInfo.MagnetUrl.Should().Be(null); + torrentInfo.Seeders.Should().Be(4); + torrentInfo.Peers.Should().Be(3+4); } } } diff --git a/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs b/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs index 7c995077a..b80782878 100644 --- a/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs @@ -12,6 +12,10 @@ public EzrssTorrentRssParser() UseGuidInfoUrl = true; UseEnclosureLength = false; UseEnclosureUrl = true; + SeedsElementName = "seeds"; + InfoHashElementName = "infoHash"; + SizeElementName = "contentLength"; + MagnetElementName = "magnetURI"; } protected override bool PreProcess(IndexerResponse indexerResponse) @@ -27,50 +31,5 @@ protected override bool PreProcess(IndexerResponse indexerResponse) return base.PreProcess(indexerResponse); } - protected override long GetSize(XElement item) - { - var contentLength = item.FindDecendants("contentLength").SingleOrDefault(); - - if (contentLength != null) - { - return (long)contentLength; - } - - return base.GetSize(item); - } - - protected override string GetInfoHash(XElement item) - { - var infoHash = item.FindDecendants("infoHash").SingleOrDefault(); - return (string)infoHash; - } - - protected override string GetMagnetUrl(XElement item) - { - var magnetURI = item.FindDecendants("magnetURI").SingleOrDefault(); - return (string)magnetURI; - } - - protected override int? GetSeeders(XElement item) - { - var seeds = item.FindDecendants("seeds").SingleOrDefault(); - if (seeds != null) - { - return (int)seeds; - } - - return base.GetSeeders(item); - } - - protected override int? GetPeers(XElement item) - { - var peers = item.FindDecendants("peers").SingleOrDefault(); - if (peers != null) - { - return (int)peers; - } - - return base.GetPeers(item); - } } } diff --git a/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs b/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs index 82d1a1c3d..b652fdbc8 100644 --- a/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs +++ b/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs @@ -25,7 +25,7 @@ public override IIndexerRequestGenerator GetRequestGenerator() public override IParseIndexerResponse GetParser() { - return new TorrentRssParser() { UseGuidInfoUrl = true, ParseSizeInDescription = true, ParseSeedersInDescription = true }; + return new TorrentRssParser() { UseGuidInfoUrl = true, SizeElementName = "size", InfoHashElementName = "infoHash", PeersElementName = "leechers", CalculatePeersAsSum = true, SeedsElementName = "seeders" }; } } } \ No newline at end of file diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index d1ace5afb..f73598af0 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; using System.Xml.Linq; using MonoTorrent; @@ -9,12 +10,27 @@ namespace NzbDrone.Core.Indexers { public class TorrentRssParser : RssParser { + // Use to sum/calculate Peers as Leechers+Seeders + public bool CalculatePeersAsSum { get; set; } = false; + + // Use the specified element name to determine the Infohash + public string InfoHashElementName { get; set; } + // Parse various seeder/leecher/peers formats in the description element to determine number of seeders. public bool ParseSeedersInDescription { get; set; } - // Use the specified element name to determine the size + // Use the specified element name to determine the Peers + public string PeersElementName { get; set; } + + // Use the specified element name to determine the Seeds + public string SeedsElementName { get; set; } + + // Use the specified element name to determine the Size public string SizeElementName { get; set; } + // Use the specified element name to determine the Magnet link + public string MagnetElementName { get; set; } + public TorrentRssParser() { PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes; @@ -40,14 +56,27 @@ protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInf result.InfoHash = GetInfoHash(item); result.MagnetUrl = GetMagnetUrl(item); result.Seeders = GetSeeders(item); - result.Peers = GetPeers(item); + if (CalculatePeersAsSum) + { + result.Peers = GetPeers(item) + result.Seeders; + } + else + { + result.Peers = GetPeers(item); + } return result; } protected virtual string GetInfoHash(XElement item) { + if (InfoHashElementName.IsNotNullOrWhiteSpace()) + { + return item.FindDecendants(InfoHashElementName).FirstOrDefault().Value; + } + var magnetUrl = GetMagnetUrl(item); + if (magnetUrl.IsNotNullOrWhiteSpace()) { try @@ -64,10 +93,21 @@ protected virtual string GetInfoHash(XElement item) protected virtual string GetMagnetUrl(XElement item) { - var downloadUrl = GetDownloadUrl(item); - if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:")) + if (MagnetElementName.IsNotNullOrWhiteSpace()) { - return downloadUrl; + var magnetURL = item.FindDecendants(MagnetElementName).FirstOrDefault().Value; + if (magnetURL.IsNotNullOrWhiteSpace() && magnetURL.StartsWith("magnet:")) + { + return magnetURL; + } + } + else + { + var downloadUrl = GetDownloadUrl(item); + if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:")) + { + return downloadUrl; + } } return null; @@ -75,6 +115,9 @@ protected virtual string GetMagnetUrl(XElement item) protected virtual int? GetSeeders(XElement item) { + // safe to always use the element if it's present (and valid) + // fall back to description if ParseSeedersInDescription is enabled + if (ParseSeedersInDescription && item.Element("description") != null) { var matchSeeders = ParseSeedersRegex.Match(item.Element("description").Value); @@ -92,6 +135,11 @@ protected virtual string GetMagnetUrl(XElement item) return int.Parse(matchPeers.Groups["value"].Value) - int.Parse(matchLeechers.Groups["value"].Value); } } + var seeds = item.FindDecendants(SeedsElementName).SingleOrDefault(); + if (seeds != null) + { + return (int)seeds; + } return null; } @@ -116,6 +164,12 @@ protected virtual string GetMagnetUrl(XElement item) } } + if (PeersElementName.IsNotNullOrWhiteSpace()) + { + var itempeers = item.FindDecendants(PeersElementName).SingleOrDefault(); + return int.Parse(itempeers.Value); + } + return null; } @@ -124,12 +178,12 @@ protected override long GetSize(XElement item) var size = base.GetSize(item); if (size == 0 && SizeElementName.IsNotNullOrWhiteSpace()) { - if (item.Element(SizeElementName) != null) + var itemsize = item.FindDecendants(SizeElementName).SingleOrDefault(); + if (itemsize != null) { - size = ParseSize(item.Element(SizeElementName).Value, true); + size = ParseSize(itemsize.Value, true); } } - return size; }