Fixed: Prevent duplicate parsing of extra files

src/NzbDrone.Core/Extras/ExistingExtraFileService.cs

@@ -14,18 +14,18 @@ namespace NzbDrone.Core.Extras
     public class ExistingExtraFileService : IHandle<SeriesScannedEvent>
     {
         private readonly IDiskProvider _diskProvider;
+        private readonly IDiskScanService _diskScanService;
         private readonly List<IImportExistingExtraFiles> _existingExtraFileImporters;
-        private readonly List<IManageExtraFiles> _extraFileManagers;
         private readonly Logger _logger;
 
         public ExistingExtraFileService(IDiskProvider diskProvider,
+                                        IDiskScanService diskScanService,
                                         List<IImportExistingExtraFiles> existingExtraFileImporters,
-                                        List<IManageExtraFiles> extraFileManagers,
                                         Logger logger)
         {
             _diskProvider = diskProvider;
+            _diskScanService = diskScanService;
             _existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList();
-            _extraFileManagers = extraFileManagers.OrderBy(e => e.Order).ToList();
             _logger = logger;
         }
 
@@ -41,9 +41,8 @@ public void Handle(SeriesScannedEvent message)
 
             _logger.Debug("Looking for existing extra files in {0}", series.Path);
 
-            var filesOnDisk = _diskProvider.GetFiles(series.Path, SearchOption.AllDirectories);
+            var filesOnDisk = _diskScanService.GetNonVideoFiles(series.Path);
-            var possibleExtraFiles = filesOnDisk.Where(c => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(c).ToLower()) &&
+            var possibleExtraFiles = _diskScanService.FilterFiles(series, filesOnDisk);
-                                                            !c.StartsWith(Path.Combine(series.Path, "EXTRAS"))).ToList();
 
             var filteredFiles = possibleExtraFiles;
             var importedFiles = new List<string>();
@@ -55,7 +54,7 @@ public void Handle(SeriesScannedEvent message)
                 importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath)));
             }
 
-            _logger.Info("Found {0} extra files", extraFiles);
+            _logger.Info("Found {0} extra files", extraFiles.Count);
         }
     }
 }

src/NzbDrone.Core/Extras/ImportExistingExtraFileFilterResult.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using NzbDrone.Core.Extras.Files;
+
+namespace NzbDrone.Core.Extras
+{
+    public class ImportExistingExtraFileFilterResult<TExtraFile>
+        where TExtraFile : ExtraFile, new()
+    {
+        public ImportExistingExtraFileFilterResult(List<TExtraFile> previouslyImported, List<string> filesOnDisk)
+        {
+            PreviouslyImported = previouslyImported;
+            FilesOnDisk = filesOnDisk;
+        }
+
+        public List<TExtraFile> PreviouslyImported { get; set; }
+        public List<string> FilesOnDisk { get; set; }
+    }
+}

src/NzbDrone.Core/Extras/ImportExistingExtraFilesBase.cs

@@ -21,7 +21,7 @@ public ImportExistingExtraFilesBase(IExtraFileService<TExtraFile> extraFileServi
         public abstract int Order { get; }
         public abstract IEnumerable<ExtraFile> ProcessFiles(Series series, List<string> filesOnDisk, List<string> importedFiles);
 
-        public virtual List<string> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
+        public virtual ImportExistingExtraFileFilterResult<TExtraFile> FilterAndClean(Series series, List<string> filesOnDisk, List<string> importedFiles)
         {
             var seriesFiles = _extraFileService.GetFilesBySeries(series.Id);
 
@@ -30,12 +30,16 @@ public virtual List<string> FilterAndClean(Series series, List<string> filesOnDi
             return Filter(series, filesOnDisk, importedFiles, seriesFiles);
         }
 
-        private List<string> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
+        private ImportExistingExtraFileFilterResult<TExtraFile> Filter(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)
         {
-            var filteredFiles = filesOnDisk;
+            var previouslyImported = seriesFiles.IntersectBy(s => Path.Combine(series.Path, s.RelativePath), filesOnDisk, f => f, PathEqualityComparer.Instance).ToList();
+            var filteredFiles = filesOnDisk.Except(previouslyImported.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance)
+                                           .Except(importedFiles, PathEqualityComparer.Instance)
+                                           .ToList();
 
-            filteredFiles = filteredFiles.Except(seriesFiles.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance).ToList();
+            // Return files that are already imported so they aren't imported again by other importers.
-            return filteredFiles.Except(importedFiles, PathEqualityComparer.Instance).ToList();
+            // Filter out files that were previously imported and as well as ones imported by other importers.
+            return new ImportExistingExtraFileFilterResult<TExtraFile>(previouslyImported, filteredFiles);
         }
 
         private void Clean(Series series, List<string> filesOnDisk, List<string> importedFiles, List<TExtraFile> seriesFiles)

src/NzbDrone.Core/Extras/MetaData/ExistingMetadataImporter.cs

@@ -5,6 +5,7 @@
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Extras.Files;
 using NzbDrone.Core.Extras.Metadata.Files;
+using NzbDrone.Core.Extras.Subtitles;
 using NzbDrone.Core.Parser;
 using NzbDrone.Core.Tv;
 
@@ -42,10 +43,17 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
             _logger.Debug("Looking for existing metadata in {0}", series.Path);
 
             var metadataFiles = new List<MetadataFile>();
-            var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
+            var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
 
-            foreach (var possibleMetadataFile in filteredFiles)
+            foreach (var possibleMetadataFile in filterResult.FilesOnDisk)
             {
+                // Don't process files that have known Subtitle file extensions (saves a bit of unecessary processing)
+
+                if (SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(possibleMetadataFile)))
+                {
+                    continue;
+                }
+
                 foreach (var consumer in _consumers)
                 {
                     var metadata = consumer.FindMetadataFile(series, possibleMetadataFile);
@@ -90,7 +98,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
             _logger.Info("Found {0} existing metadata files", metadataFiles.Count);
             _metadataFileService.Upsert(metadataFiles);
 
-            return metadataFiles;
+            // Return files that were just imported along with files that were
+            // previously imported so previously imported files aren't imported twice
+
+            return metadataFiles.Concat(filterResult.PreviouslyImported);
         }
     }
 }

src/NzbDrone.Core/Extras/Others/ExistingOtherExtraImporter.cs

@@ -38,9 +38,9 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
             _logger.Debug("Looking for existing extra files in {0}", series.Path);
 
             var extraFiles = new List<OtherExtraFile>();
-            var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
+            var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
 
-            foreach (var possibleExtraFile in filteredFiles)
+            foreach (var possibleExtraFile in filterResult.FilesOnDisk)
             {
                 var localEpisode = _parsingService.GetLocalEpisode(possibleExtraFile, series);
 
@@ -77,7 +77,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
             _logger.Info("Found {0} existing other extra files", extraFiles.Count);
             _otherExtraFileService.Upsert(extraFiles);
 
-            return extraFiles;
+            // Return files that were just imported along with files that were
+            // previously imported so previously imported files aren't imported twice
+
+            return extraFiles.Concat(filterResult.PreviouslyImported);
         }
     }
 }

src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs

@@ -38,9 +38,9 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
             _logger.Debug("Looking for existing subtitle files in {0}", series.Path);
 
             var subtitleFiles = new List<SubtitleFile>();
-            var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles);
+            var filterResult = FilterAndClean(series, filesOnDisk, importedFiles);
 
-            foreach (var possibleSubtitleFile in filteredFiles)
+            foreach (var possibleSubtitleFile in filterResult.FilesOnDisk)
             {
                 var extension = Path.GetExtension(possibleSubtitleFile);
 
@@ -83,7 +83,10 @@ public override IEnumerable<ExtraFile> ProcessFiles(Series series, List<string>
             _logger.Info("Found {0} existing subtitle files", subtitleFiles.Count);
             _subtitleFileService.Upsert(subtitleFiles);
 
-            return subtitleFiles;
+            // Return files that were just imported along with files that were
+            // previously imported so previously imported files aren't imported twice
+
+            return subtitleFiles.Concat(filterResult.PreviouslyImported);
         }
     }
 }

src/NzbDrone.Core/MediaFiles/DiskScanService.cs

@@ -23,6 +23,8 @@ public interface IDiskScanService
     {
         void Scan(Series series);
         string[] GetVideoFiles(string path, bool allDirectories = true);
+        string[] GetNonVideoFiles(string path, bool allDirectories = true);
+        List<string> FilterFiles(Series series, IEnumerable<string> files);
     }
 
     public class DiskScanService :
@@ -59,7 +61,7 @@ public DiskScanService(IDiskProvider diskProvider,
         }
 
         private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
-        private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+        private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase);
 
         public void Scan(Series series)
         {
@@ -133,10 +135,25 @@ public string[] GetVideoFiles(string path, bool allDirectories = true)
             return mediaFileList.ToArray();
         }
 
-        private IEnumerable<string> FilterFiles(Series series, IEnumerable<string> videoFiles)
+        public string[] GetNonVideoFiles(string path, bool allDirectories = true)
         {
-            return videoFiles.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
+            _logger.Debug("Scanning '{0}' for non-video files", path);
-                             .Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)));
+
+            var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
+            var filesOnDisk = _diskProvider.GetFiles(path, searchOption);
+
+            var mediaFileList = filesOnDisk.Where(file => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(file).ToLower()))
+                                           .ToList();
+
+            _logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path);
+            return mediaFileList.ToArray();
+        }
+
+        public List<string> FilterFiles(Series series, IEnumerable<string> files)
+        {
+            return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file)))
+                        .Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file)))
+                        .ToList();
         }
 
         private void SetPermissions(string path)

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -495,6 +495,7 @@
     <Compile Include="Extras\Files\ExtraFileRepository.cs" />
     <Compile Include="Extras\ExtraService.cs" />
     <Compile Include="Extras\IImportExistingExtraFiles.cs" />
+    <Compile Include="Extras\ImportExistingExtraFileFilterResult.cs" />
     <Compile Include="Extras\ImportExistingExtraFilesBase.cs" />
     <Compile Include="Extras\Metadata\Files\MetadataFile.cs" />
     <Compile Include="Extras\Metadata\Files\MetadataFileRepository.cs" />