self-hosted/Sonarr
1
0
Fork 0
mirror of https://github.com/Sonarr/Sonarr.git synced 2024-12-16 11:37:58 +02:00
Code Issues Releases Activity

Import process improvements

Browse Source 
New: Post processing scripts can pass "Path" via API to scan a specific folder directly
Fixed: Do not delete folder from drone factory when non-sample video files exist
This commit is contained in:
Mark McDowall 2014-04-03 17:08:51 -07:00
parent 037127163f
commit e5263f143d
9 changed files with 385 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/NzbDrone.Core.Test/MediaFiles/DownloadedEpisodesImportServiceFixture.cs
View File

@ -12,6 +12,7 @@
using NzbDrone.Core.MediaFiles.EpisodeImport;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Tv;
using NzbDrone.Test.Common;
@ -22,7 +23,7 @@ namespace NzbDrone.Core.Test.MediaFiles
public class DownloadedEpisodesImportServiceFixture : CoreTest<DownloadedEpisodesImportService>
{
private string[] _subFolders = new[] { "c:\\root\\foldername".AsOsAgnostic() };
private string[] _videoFiles = new[] { "c:\\root\\foldername\\video.ext".AsOsAgnostic() };
private string[] _videoFiles = new[] { "c:\\root\\foldername\\30.rock.s01e01.ext".AsOsAgnostic() };
[SetUp]
public void Setup()
@ -113,6 +114,8 @@ public void should_not_import_if_folder_is_a_series_path()
Mocker.GetMock<IParsingService>()
.Verify(v => v.GetSeries(It.IsAny<String>()), Times.Never());
ExceptionVerification.ExpectedWarns(1);
}
[Test]
@ -129,7 +132,7 @@ public void should_not_delete_folder_if_no_files_were_imported()
}
[Test]
public void should_delete_folder_if_files_were_imported()
public void should_delete_folder_if_files_were_imported_and_video_files_remain()
{
GivenValidSeries();
@ -148,6 +151,40 @@ public void should_delete_folder_if_files_were_imported()
Subject.Execute(new DownloadedEpisodesScanCommand());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFolder(It.IsAny<String>(), true), Times.Never());
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void should_delete_folder_if_files_were_imported_and_only_sample_files_remain()
{
GivenValidSeries();
var localEpisode = new LocalEpisode();
var imported = new List<ImportDecision>();
imported.Add(new ImportDecision(localEpisode));
Mocker.GetMock<IMakeImportDecision>()
.Setup(s => s.GetImportDecisions(It.IsAny<List<String>>(), It.IsAny<Series>(), true, null))
.Returns(imported);
Mocker.GetMock<IImportApprovedEpisodes>()
.Setup(s => s.Import(It.IsAny<List<ImportDecision>>(), true))
.Returns(imported);
Mocker.GetMock<ISampleService>()
.Setup(s => s.IsSample(It.IsAny<Series>(),
It.IsAny<QualityModel>(),
It.IsAny<String>(),
It.IsAny<Int64>(),
It.IsAny<Int32>()))
.Returns(true);
Subject.Execute(new DownloadedEpisodesScanCommand());
Mocker.GetMock<IDiskProvider>()
.Verify(v => v.DeleteFolder(It.IsAny<String>(), true), Times.Once());
}

151
src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/SampleServiceFixture.cs Normal file
View File

@ -0,0 +1,151 @@
using System;
using System.Linq;
using FizzWare.NBuilder;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.MediaFiles.EpisodeImport;
using NzbDrone.Core.MediaFiles.MediaInfo;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport
{
[TestFixture]
public class SampleServiceFixture : CoreTest<SampleService>
{
private Series _series;
private LocalEpisode _localEpisode;
[SetUp]
public void Setup()
{
_series = Builder<Series>.CreateNew()
.With(s => s.SeriesType = SeriesTypes.Standard)
.Build();
var episodes = Builder<Episode>.CreateListOfSize(1)
.All()
.With(e => e.SeasonNumber = 1)
.Build()
.ToList();
_localEpisode = new LocalEpisode
{
Path = @"C:\Test\30 Rock\30.rock.s01e01.avi",
Episodes = episodes,
Series = _series,
Quality = new QualityModel(Quality.HDTV720p)
};
}
private void GivenFileSize(long size)
{
_localEpisode.Size = size;
}
private void GivenRuntime(int seconds)
{
Mocker.GetMock<IVideoFileInfoReader>()
.Setup(s => s.GetRunTime(It.IsAny<String>()))
.Returns(new TimeSpan(0, 0, seconds));
}
[Test]
public void should_return_true_if_series_is_daily()
{
_series.SeriesType = SeriesTypes.Daily;
ShouldBeFalse();
}
[Test]
public void should_return_true_if_season_zero()
{
_localEpisode.Episodes[0].SeasonNumber = 0;
ShouldBeFalse();
}
[Test]
public void should_return_true_for_flv()
{
_localEpisode.Path = @"C:\Test\some.show.s01e01.flv";
ShouldBeFalse();
Mocker.GetMock<IVideoFileInfoReader>().Verify(c => c.GetRunTime(It.IsAny<string>()), Times.Never());
}
[Test]
public void should_use_runtime()
{
GivenRuntime(120);
GivenFileSize(1000.Megabytes());
Subject.IsSample(_localEpisode.Series,
_localEpisode.Quality,
_localEpisode.Path,
_localEpisode.Size,
_localEpisode.SeasonNumber);
Mocker.GetMock<IVideoFileInfoReader>().Verify(v => v.GetRunTime(It.IsAny<String>()), Times.Once());
}
[Test]
public void should_return_false_if_runtime_is_less_than_minimum()
{
GivenRuntime(60);
ShouldBeTrue();
}
[Test]
public void should_return_true_if_runtime_greater_than_than_minimum()
{
GivenRuntime(120);
ShouldBeFalse();
}
[Test]
public void should_fall_back_to_file_size_if_mediainfo_dll_not_found_acceptable_size()
{
Mocker.GetMock<IVideoFileInfoReader>()
.Setup(s => s.GetRunTime(It.IsAny<String>()))
.Throws<DllNotFoundException>();
GivenFileSize(1000.Megabytes());
ShouldBeFalse();
}
[Test]
public void should_fall_back_to_file_size_if_mediainfo_dll_not_found_undersize()
{
Mocker.GetMock<IVideoFileInfoReader>()
.Setup(s => s.GetRunTime(It.IsAny<String>()))
.Throws<DllNotFoundException>();
GivenFileSize(1.Megabytes());
ShouldBeTrue();
}
private void ShouldBeTrue()
{
Subject.IsSample(_localEpisode.Series,
_localEpisode.Quality,
_localEpisode.Path,
_localEpisode.Size,
_localEpisode.SeasonNumber).Should().BeTrue();
}
private void ShouldBeFalse()
{
Subject.IsSample(_localEpisode.Series,
_localEpisode.Quality,
_localEpisode.Path,
_localEpisode.Size,
_localEpisode.SeasonNumber).Should().BeFalse();
}
}
}

85
src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Specifications/NotSampleSpecificationFixture.cs
View File

@ -41,96 +41,11 @@ public void Setup()
};
}
private void GivenFileSize(long size)
{
_localEpisode.Size = size;
}
private void GivenRuntime(int seconds)
{
Mocker.GetMock<IVideoFileInfoReader>()
.Setup(s => s.GetRunTime(It.IsAny<String>()))
.Returns(new TimeSpan(0, 0, seconds));
}
[Test]
public void should_return_true_if_series_is_daily()
{
_series.SeriesType = SeriesTypes.Daily;
Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue();
}
[Test]
public void should_return_true_if_season_zero()
{
_localEpisode.Episodes[0].SeasonNumber = 0;
Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue();
}
[Test]
public void should_return_true_for_existing_file()
{
_localEpisode.ExistingFile = true;
Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue();
}
[Test]
public void should_return_true_for_flv()
{
_localEpisode.Path = @"C:\Test\some.show.s01e01.flv";
Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue();
Mocker.GetMock<IVideoFileInfoReader>().Verify(c => c.GetRunTime(It.IsAny<string>()), Times.Never());
}
[Test]
public void should_use_runtime()
{
GivenRuntime(120);
GivenFileSize(1000.Megabytes());
Subject.IsSatisfiedBy(_localEpisode);
Mocker.GetMock<IVideoFileInfoReader>().Verify(v => v.GetRunTime(It.IsAny<String>()), Times.Once());
}
[Test]
public void should_return_false_if_runtime_is_less_than_minimum()
{
GivenRuntime(60);
Subject.IsSatisfiedBy(_localEpisode).Should().BeFalse();
}
[Test]
public void should_return_true_if_runtime_greater_than_than_minimum()
{
GivenRuntime(120);
Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue();
}
[Test]
public void should_fall_back_to_file_size_if_mediainfo_dll_not_found_acceptable_size()
{
Mocker.GetMock<IVideoFileInfoReader>()
.Setup(s => s.GetRunTime(It.IsAny<String>()))
.Throws<DllNotFoundException>();
GivenFileSize(1000.Megabytes());
Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue();
}
[Test]
public void should_fall_back_to_file_size_if_mediainfo_dll_not_found_undersize()
{
Mocker.GetMock<IVideoFileInfoReader>()
.Setup(s => s.GetRunTime(It.IsAny<String>()))
.Throws<DllNotFoundException>();
GivenFileSize(1.Megabytes());
Subject.IsSatisfiedBy(_localEpisode).Should().BeFalse();
}
}
}

1
src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj
View File

@ -156,6 +156,7 @@
<Compile Include="MediaFiles\EpisodeImport\Specifications\FreeSpaceSpecificationFixture.cs" />
<Compile Include="MediaFiles\EpisodeImport\ImportDecisionMakerFixture.cs" />
<Compile Include="MediaFiles\EpisodeImport\Specifications\NotInUseSpecificationFixture.cs" />
<Compile Include="MediaFiles\EpisodeImport\SampleServiceFixture.cs" />
<Compile Include="MediaFiles\EpisodeImport\Specifications\NotSampleSpecificationFixture.cs" />
<Compile Include="MediaFiles\EpisodeImport\Specifications\NotUnpackingSpecificationFixture.cs" />
<Compile Include="MediaFiles\EpisodeImport\Specifications\UpgradeSpecificationFixture.cs" />

4
src/NzbDrone.Core/MediaFiles/Commands/DownloadedEpisodesScanCommand.cs
View File

@ -1,3 +1,4 @@
using System;
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.MediaFiles.Commands
@ -12,6 +13,7 @@ public override bool SendUpdatesToClient
}
}
public bool SendUpdates { get; set; }
public Boolean SendUpdates { get; set; }
public String Path { get; set; }
}
}

97
src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs
View File

@ -1,10 +1,12 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using NLog;
using NzbDrone.Common;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnsureThat;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.MediaFiles.Commands;
using NzbDrone.Core.MediaFiles.EpisodeImport;
@ -24,6 +26,7 @@ public class DownloadedEpisodesImportService : IExecute<DownloadedEpisodesScanCo
private readonly IConfigService _configService;
private readonly IMakeImportDecision _importDecisionMaker;
private readonly IImportApprovedEpisodes _importApprovedEpisodes;
private readonly ISampleService _sampleService;
private readonly Logger _logger;
public DownloadedEpisodesImportService(IDiskProvider diskProvider,
@ -33,6 +36,7 @@ public DownloadedEpisodesImportService(IDiskProvider diskProvider,
IConfigService configService,
IMakeImportDecision importDecisionMaker,
IImportApprovedEpisodes importApprovedEpisodes,
ISampleService sampleService,
Logger logger)
{
_diskProvider = diskProvider;
@ -42,6 +46,7 @@ public DownloadedEpisodesImportService(IDiskProvider diskProvider,
_configService = configService;
_importDecisionMaker = importDecisionMaker;
_importApprovedEpisodes = importApprovedEpisodes;
_sampleService = sampleService;
_logger = logger;
}
@ -64,24 +69,7 @@ private void ProcessDownloadedEpisodesFolder()
foreach (var subFolder in _diskProvider.GetDirectories(downloadedEpisodesFolder))
{
try
{
if (_seriesService.SeriesPathExists(subFolder))
{
continue;
}
var importedFiles = ProcessSubFolder(new DirectoryInfo(subFolder));
if (importedFiles.Any())
{
_diskProvider.DeleteFolder(subFolder, true);
}
}
catch (Exception e)
{
_logger.ErrorException("An error has occurred while importing folder: " + subFolder, e);
}
ProcessFolder(subFolder);
}
foreach (var videoFile in _diskScanService.GetVideoFiles(downloadedEpisodesFolder, false))
@ -97,9 +85,9 @@ private void ProcessDownloadedEpisodesFolder()
}
}
private List<ImportDecision> ProcessSubFolder(DirectoryInfo subfolderInfo)
private List<ImportDecision> ProcessFolder(DirectoryInfo directoryInfo)
{
var cleanedUpName = GetCleanedUpFolderName(subfolderInfo.Name);
var cleanedUpName = GetCleanedUpFolderName(directoryInfo.Name);
var series = _parsingService.GetSeries(cleanedUpName);
var quality = QualityParser.ParseQuality(cleanedUpName);
_logger.Debug("{0} folder quality: {1}", cleanedUpName, quality);
@ -110,7 +98,7 @@ private List<ImportDecision> ProcessSubFolder(DirectoryInfo subfolderInfo)
return new List<ImportDecision>();
}
var videoFiles = _diskScanService.GetVideoFiles(subfolderInfo.FullName);
var videoFiles = _diskScanService.GetVideoFiles(directoryInfo.FullName);
return ProcessFiles(series, quality, videoFiles);
}
@ -140,6 +128,33 @@ private List<ImportDecision> ProcessFiles(Series series, QualityModel quality, p
return _importApprovedEpisodes.Import(decisions, true);
}
private void ProcessFolder(string path)
{
Ensure.That(path, () => path).IsValidPath();
try
{
if (_seriesService.SeriesPathExists(path))
{
_logger.Warn("Unable to process folder that contains sorted TV Shows");
return;
}
var directoryFolderInfo = new DirectoryInfo(path);
var importedFiles = ProcessFolder(directoryFolderInfo);
if (importedFiles.Any() && ShouldDeleteFolder(directoryFolderInfo))
{
_logger.Debug("Deleting folder after importing valid files");
_diskProvider.DeleteFolder(path, true);
}
}
catch (Exception e)
{
_logger.ErrorException("An error has occurred while importing folder: " + path, e);
}
}
private string GetCleanedUpFolderName(string folder)
{
folder = folder.Replace("_UNPACK_", "")
@ -148,9 +163,47 @@ private string GetCleanedUpFolderName(string folder)
return folder;
}
private bool ShouldDeleteFolder(DirectoryInfo directoryInfo)
{
var videoFiles = _diskScanService.GetVideoFiles(directoryInfo.FullName);
var cleanedUpName = GetCleanedUpFolderName(directoryInfo.Name);
var series = _parsingService.GetSeries(cleanedUpName);
foreach (var videoFile in videoFiles)
{
var episodeParseResult = Parser.Parser.ParseTitle(Path.GetFileName(videoFile));
if (episodeParseResult == null)
{
_logger.Warn("Unable to parse file on import: [{0}]", videoFile);
return false;
}
var size = _diskProvider.GetFileSize(videoFile);
var quality = QualityParser.ParseQuality(videoFile);
if (!_sampleService.IsSample(series, quality, videoFile, size,
episodeParseResult.SeasonNumber))
{
_logger.Warn("Non-sample file has not been imported: [{0}]", videoFile);
return false;
}
}
return true;
}
public void Execute(DownloadedEpisodesScanCommand message)
{
ProcessDownloadedEpisodesFolder();
if (message.Path.IsNullOrWhiteSpace())
{
ProcessDownloadedEpisodesFolder();
}
else
{
ProcessFolder(message.Path);
}
}
}
}

107
src/NzbDrone.Core/MediaFiles/EpisodeImport/SampleService.cs Normal file
View File

@ -0,0 +1,107 @@
using System;
using System.Collections.Generic;
using System.IO;
using NLog;
using NzbDrone.Core.MediaFiles.MediaInfo;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.MediaFiles.EpisodeImport
{
public interface ISampleService
{
bool IsSample(Series series, QualityModel quality, string path, long size, int seasonNumber);
}
public class SampleService : ISampleService
{
private readonly IVideoFileInfoReader _videoFileInfoReader;
private readonly Logger _logger;
private static List<Quality> _largeSampleSizeQualities = new List<Quality> { Quality.HDTV1080p, Quality.WEBDL1080p, Quality.Bluray1080p };
public SampleService(IVideoFileInfoReader videoFileInfoReader, Logger logger)
{
_videoFileInfoReader = videoFileInfoReader;
_logger = logger;
}
public static long SampleSizeLimit
{
get
{
return 70.Megabytes();
}
}
public bool IsSample(Series series, QualityModel quality, string path, long size, int seasonNumber)
{
if (series.SeriesType == SeriesTypes.Daily)
{
_logger.Debug("Daily Series, skipping sample check");
return false;
}
if (seasonNumber == 0)
{
_logger.Debug("Special, skipping sample check");
return false;
}
var extension = Path.GetExtension(path);
if (extension != null && extension.Equals(".flv", StringComparison.InvariantCultureIgnoreCase))
{
_logger.Debug("Skipping sample check for .flv file");
return false;
}
try
{
var runTime = _videoFileInfoReader.GetRunTime(path);
if (runTime.TotalMinutes.Equals(0))
{
_logger.Error("[{0}] has a runtime of 0, is it a valid video file?", path);
return true;
}
if (runTime.TotalSeconds < 90)
{
_logger.Debug("[{0}] appears to be a sample. Size: {1} Runtime: {2}", path, size, runTime);
return true;
}
}
catch (DllNotFoundException)
{
_logger.Debug("Falling back to file size detection");
return CheckSize(size, quality);
}
_logger.Debug("Runtime is over 90 seconds");
return false;
}
private bool CheckSize(long size, QualityModel quality)
{
if (_largeSampleSizeQualities.Contains(quality.Quality))
{
if (size < SampleSizeLimit * 2)
{
_logger.Debug("1080p file is less than sample limit");
return true;
}
}
if (size < SampleSizeLimit)
{
_logger.Debug("File is less than sample limit");
return true;
}
return false;
}
}
}

87
src/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotSampleSpecification.cs
View File

@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.IO;
using NLog;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Core.MediaFiles.MediaInfo;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Qualities;
@ -12,25 +11,16 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Specifications
{
public class NotSampleSpecification : IImportDecisionEngineSpecification
{
private readonly IVideoFileInfoReader _videoFileInfoReader;
private readonly ISampleService _sampleService;
private readonly Logger _logger;
private static List<Quality> _largeSampleSizeQualities = new List<Quality> { Quality.HDTV1080p, Quality.WEBDL1080p, Quality.Bluray1080p };
public NotSampleSpecification(IVideoFileInfoReader videoFileInfoReader,
public NotSampleSpecification(ISampleService sampleService,
Logger logger)
{
_videoFileInfoReader = videoFileInfoReader;
_sampleService = sampleService;
_logger = logger;
}
public static long SampleSizeLimit
{
get
{
return 70.Megabytes();
}
}
public string RejectionReason { get { return "Sample"; } }
public bool IsSatisfiedBy(LocalEpisode localEpisode)
@ -41,72 +31,11 @@ public bool IsSatisfiedBy(LocalEpisode localEpisode)
return true;
}
if (localEpisode.Series.SeriesType == SeriesTypes.Daily)
{
_logger.Debug("Daily Series, skipping sample check");
return true;
}
if (localEpisode.SeasonNumber == 0)
{
_logger.Debug("Special, skipping sample check");
return true;
}
var extension = Path.GetExtension(localEpisode.Path);
if (extension != null && extension.Equals(".flv", StringComparison.InvariantCultureIgnoreCase))
{
_logger.Debug("Skipping sample check for .flv file");
return true;
}
try
{
var runTime = _videoFileInfoReader.GetRunTime(localEpisode.Path);
if (runTime.TotalMinutes.Equals(0))
{
_logger.Error("[{0}] has a runtime of 0, is it a valid video file?", localEpisode);
return false;
}
if (runTime.TotalSeconds < 90)
{
_logger.Debug("[{0}] appears to be a sample. Size: {1} Runtime: {2}", localEpisode.Path, localEpisode.Size, runTime);
return false;
}
}
catch (DllNotFoundException)
{
_logger.Debug("Falling back to file size detection");
return CheckSize(localEpisode);
}
_logger.Debug("Runtime is over 90 seconds");
return true;
}
private bool CheckSize(LocalEpisode localEpisode)
{
if (_largeSampleSizeQualities.Contains(localEpisode.Quality.Quality))
{
if (localEpisode.Size < SampleSizeLimit * 2)
{
_logger.Debug("1080p file is less than sample limit");
return false;
}
}
if (localEpisode.Size < SampleSizeLimit)
{
_logger.Debug("File is less than sample limit");
return false;
}
return true;
return !_sampleService.IsSample(localEpisode.Series,
localEpisode.Quality,
localEpisode.Path,
localEpisode.Size,
localEpisode.SeasonNumber);
}
}
}

1
src/NzbDrone.Core/NzbDrone.Core.csproj
View File

@ -327,6 +327,7 @@
<Compile Include="MediaCover\MediaCoversUpdatedEvent.cs" />
<Compile Include="MediaFiles\Commands\RenameFilesCommand.cs" />
<Compile Include="MediaFiles\EpisodeFileMoveResult.cs" />
<Compile Include="MediaFiles\EpisodeImport\SampleService.cs" />
<Compile Include="MediaFiles\EpisodeImport\Specifications\FullSeasonSpecification.cs" />
<Compile Include="MediaFiles\Events\SeriesScannedEvent.cs" />
<Compile Include="MediaFiles\FileDateType.cs" />