New: Preserve language tags when importing subtitle files

Closes #2570
Closes #3278

src/NzbDrone.Core.Test/Extras/Subtitles/SubtitleServiceFixture.cs

@@ -82,8 +82,8 @@ public void should_not_import_non_subtitle_file(string filePath)
         [TestCase("Series Title - S01E01.srt", "Series Title - S01E01.srt")]
         [TestCase("Series.Title.S01E01.en.srt", "Series Title - S01E01.en.srt")]
         [TestCase("Series.Title.S01E01.english.srt", "Series Title - S01E01.en.srt")]
-        [TestCase("Series-Title-S01E01-fr-cc.srt", "Series Title - S01E01.fr.srt")]
+        [TestCase("Series-Title-S01E01-fr-cc.srt", "Series Title - S01E01.fr.cc.srt")]
-        [TestCase("Series Title S01E01_en_sdh_forced.srt", "Series Title - S01E01.en.srt")]
+        [TestCase("Series Title S01E01_en_sdh_forced.srt", "Series Title - S01E01.en.sdh.forced.srt")]
         [TestCase("Series_Title_S01E01 en.srt", "Series Title - S01E01.en.srt")]
         [TestCase(@"Subs\S01E01.en.srt", "Series Title - S01E01.en.srt")]
         [TestCase(@"Subs\Series.Title.S01E01\2_en.srt", "Series Title - S01E01.en.srt")]
@@ -104,7 +104,7 @@ public void should_import_multiple_subtitle_files_per_language()
             var files = new List<string>
             {
                 Path.Combine(_episodeFolder, "Series.Title.S01E01.en.srt").AsOsAgnostic(),
-                Path.Combine(_episodeFolder, "Series.Title.S01E01.english.srt").AsOsAgnostic(),
+                Path.Combine(_episodeFolder, "Series.Title.S01E01.eng.srt").AsOsAgnostic(),
                 Path.Combine(_episodeFolder, "Subs", "Series_Title_S01E01_en_forced.srt").AsOsAgnostic(),
                 Path.Combine(_episodeFolder, "Subs", "Series.Title.S01E01", "2_fr.srt").AsOsAgnostic()
             };
@@ -113,7 +113,7 @@ public void should_import_multiple_subtitle_files_per_language()
             {
                 "Series Title - S01E01.1.en.srt",
                 "Series Title - S01E01.2.en.srt",
-                "Series Title - S01E01.3.en.srt",
+                "Series Title - S01E01.en.forced.srt",
                 "Series Title - S01E01.fr.srt",
             };
 
@@ -127,6 +127,35 @@ public void should_import_multiple_subtitle_files_per_language()
             }
         }
         
+        [Test]
+        public void should_import_multiple_subtitle_files_per_language_with_tags()
+        {
+            var files = new List<string>
+            {
+                Path.Combine(_episodeFolder, "Series.Title.S01E01.en.forced.cc.srt").AsOsAgnostic(),
+                Path.Combine(_episodeFolder, "Series.Title.S01E01.other.en.forced.cc.srt").AsOsAgnostic(),
+                Path.Combine(_episodeFolder, "Series.Title.S01E01.en.forced.sdh.srt").AsOsAgnostic(),
+                Path.Combine(_episodeFolder, "Series.Title.S01E01.en.forced.default.srt").AsOsAgnostic(),
+            };
+
+            var expectedOutputs = new[]
+            {
+                "Series Title - S01E01.1.en.forced.cc.srt",
+                "Series Title - S01E01.2.en.forced.cc.srt",
+                "Series Title - S01E01.en.forced.sdh.srt",
+                "Series Title - S01E01.en.forced.default.srt"
+            };
+
+            var results = Subject.ImportFiles(_localEpisode, _episodeFile, files, true).ToList();
+
+            results.Count().Should().Be(expectedOutputs.Length);
+
+            for (int i = 0; i < expectedOutputs.Length; i++)
+            {
+                results[i].RelativePath.AsOsAgnostic().PathEquals(Path.Combine("Season 1", expectedOutputs[i]).AsOsAgnostic()).Should().Be(true);
+            }
+        }
+
         [Test]
         [TestCase("sub.srt", "Series Title - S01E01.srt")]
         [TestCase(@"Subs\2_en.srt", "Series Title - S01E01.en.srt")]

src/NzbDrone.Core/Datastore/Migration/170_add_language_tags_to_subtitle_files.cs

@@ -0,0 +1,14 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(170)]
+    public class add_language_tags_to_subtitle_files : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Alter.Table("SubtitleFiles").AddColumn("LanguageTags").AsString().Nullable();
+        }
+    }
+}

src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs

@@ -1,4 +1,5 @@
-using NzbDrone.Core.Extras.Files;
+using System.Collections.Generic;
+using NzbDrone.Core.Extras.Files;
 using NzbDrone.Core.Languages;
 
 namespace NzbDrone.Core.Extras.Subtitles
@@ -6,5 +7,11 @@ namespace NzbDrone.Core.Extras.Subtitles
     public class SubtitleFile : ExtraFile
     {
         public Language Language { get; set; }
+
+        public string AggregateString => Language + LanguageTagsAsString + Extension;
+
+        public List<string> LanguageTags { get; set; }
+
+        private string LanguageTagsAsString => string.Join(".", LanguageTags);
     }
 }

src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs

@@ -72,7 +72,7 @@ public override IEnumerable<ExtraFile> MoveFilesAfterRename(Series series, List<
             foreach (var episodeFile in episodeFiles)
             {
                 var groupedExtraFilesForEpisodeFile = subtitleFiles.Where(m => m.EpisodeFileId == episodeFile.Id)
-                                                            .GroupBy(s => s.Language + s.Extension).ToList();
+                                                            .GroupBy(s => s.AggregateString).ToList();
 
                 foreach (var group in groupedExtraFilesForEpisodeFile)
                 {
@@ -81,7 +81,7 @@ public override IEnumerable<ExtraFile> MoveFilesAfterRename(Series series, List<
 
                     foreach (var subtitleFile in group)
                     {
-                        var suffix = GetSuffix(subtitleFile.Language, copy, groupCount > 1);
+                        var suffix = GetSuffix(subtitleFile.Language, copy, subtitleFile.LanguageTags, groupCount > 1);
                         movedFiles.AddIfNotNull(MoveFile(series, episodeFile, subtitleFile, suffix));
 
                         copy++;
@@ -116,7 +116,7 @@ public override IEnumerable<ExtraFile> ImportFiles(LocalEpisode localEpisode, Ep
                 try
                 {
                     // Filename match
-                    if (Path.GetFileNameWithoutExtension(file).StartsWith(sourceFileName, StringComparison.InvariantCultureIgnoreCase))
+                    if (Path.GetFileNameWithoutExtension(file).StartsWithIgnoreCase(sourceFileName))
                     {
                         matchingFiles.Add(file);
                         continue;
@@ -175,16 +175,24 @@ public override IEnumerable<ExtraFile> ImportFiles(LocalEpisode localEpisode, Ep
                 }
             }
 
-            var subtitleFiles = new List<Tuple<string, Language, string>>();
+            var subtitleFiles = new List<SubtitleFile>();
 
             foreach (string file in matchingFiles)
             {
                 var language = LanguageParser.ParseSubtitleLanguage(file);
                 var extension = Path.GetExtension(file);
-                subtitleFiles.Add(new Tuple<string, Language, string>(file, language, extension));
+                var languageTags = LanguageParser.ParseLanguageTags(file);
+                var subFile = new SubtitleFile
+                {
+                    Language = language,
+                    Extension = extension
+                };
+                subFile.LanguageTags = languageTags.ToList();
+                subFile.RelativePath = PathExtensions.GetRelativePath(sourceFolder, file);
+                subtitleFiles.Add(subFile);
             }
 
-            var groupedSubtitleFiles = subtitleFiles.GroupBy(s => s.Item2 + s.Item3).ToList();
+            var groupedSubtitleFiles = subtitleFiles.GroupBy(s => s.AggregateString).ToList();
 
             foreach (var group in groupedSubtitleFiles)
             {
@@ -193,14 +201,15 @@ public override IEnumerable<ExtraFile> ImportFiles(LocalEpisode localEpisode, Ep
 
                 foreach (var file in group)
                 {
+                    var path = Path.Combine(sourceFolder, file.RelativePath);
+                    var language = file.Language;
+                    var extension = file.Extension;
+                    var suffix = GetSuffix(language, copy, file.LanguageTags, groupCount > 1);
                     try
                     {
-                        var path = file.Item1;
-                        var language = file.Item2;
-                        var extension = file.Item3;
-                        var suffix = GetSuffix(language, copy, groupCount > 1);
                         var subtitleFile = ImportFile(localEpisode.Series, episodeFile, path, isReadOnly, extension, suffix);
                         subtitleFile.Language = language;
+                        subtitleFile.LanguageTags = file.LanguageTags;
 
                         _mediaFileAttributeService.SetFilePermissions(path);
                         _subtitleFileService.Upsert(subtitleFile);
@@ -211,7 +220,7 @@ public override IEnumerable<ExtraFile> ImportFiles(LocalEpisode localEpisode, Ep
                     }
                     catch (Exception ex)
                     {
-                        _logger.Warn(ex, "Failed to import subtitle file: {0}", file.Item1);
+                        _logger.Warn(ex, "Failed to import subtitle file: {0}", path);
                     }
                 }
             }
@@ -219,7 +228,7 @@ public override IEnumerable<ExtraFile> ImportFiles(LocalEpisode localEpisode, Ep
             return importedFiles;
         }
 
-        private string GetSuffix(Language language, int copy, bool multipleCopies = false)
+        private string GetSuffix(Language language, int copy, List<string> languageTags, bool multipleCopies = false)
         {
             var suffixBuilder = new StringBuilder();
 
@@ -235,6 +244,12 @@ private string GetSuffix(Language language, int copy, bool multipleCopies = fals
                 suffixBuilder.Append(IsoLanguages.Get(language).TwoLetterCode);
             }
 
+            if (languageTags.Any())
+            {
+                suffixBuilder.Append(".");
+                suffixBuilder.Append(string.Join(".", languageTags));    
+            }
+
             return suffixBuilder.ToString();
         }
     }

src/NzbDrone.Core/Parser/LanguageParser.cs

@@ -1,8 +1,10 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
 using NLog;
+using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Instrumentation;
 using NzbDrone.Core.Languages;
 
@@ -153,6 +155,25 @@ public static Language ParseSubtitleLanguage(string fileName)
             return Language.Unknown;
         }
         
+        public static IEnumerable<string> ParseLanguageTags(string fileName)
+        {
+            try
+            {
+                var simpleFilename = Path.GetFileNameWithoutExtension(fileName);
+                var match = SubtitleLanguageRegex.Match(simpleFilename);
+                var languageTags = match.Groups["tags"].Captures.Cast<Capture>()
+                    .Where(tag => !tag.Value.Empty())
+                    .Select(tag => tag.Value.ToLower());
+                return languageTags;
+            }
+            catch (Exception ex)
+            {
+                Logger.Debug(ex, "Failed parsing language tags from subtitle file: {0}", fileName);
+            }
+
+            return Enumerable.Empty<string>();
+        }
+
         private static Language RegexLanguage(string title)
         {
             // Case sensitive