From 69f99373e56a2fca49a2be645e6640624cf12339 Mon Sep 17 00:00:00 2001 From: Jendrik Weise Date: Sun, 21 Jan 2024 00:19:33 +0100 Subject: [PATCH] New: Parse subtitle titles Closes #5955 --- ...les_from_existing_subtitle_filesFixture.cs | 100 ++++++++++++++++++ .../AggregateSubtitleInfoFixture.cs | 48 +++++++++ .../ParserTests/LanguageParserFixture.cs | 33 ++++++ ...rse_titles_from_existing_subtitle_files.cs | 88 +++++++++++++++ .../Subtitles/ExistingSubtitleImporter.cs | 13 ++- .../Extras/Subtitles/SubtitleFile.cs | 30 +++++- .../Extras/Subtitles/SubtitleService.cs | 31 ++++-- .../Aggregation/AggregationService.cs | 3 +- .../Aggregators/AggregateEpisodes.cs | 2 + .../Aggregators/AggregateLanguage.cs | 2 + .../Aggregators/AggregateQuality.cs | 2 + .../Aggregators/AggregateReleaseGroup.cs | 2 + .../Aggregators/AggregateReleaseInfo.cs | 2 + .../Aggregators/AggregateSubtitleInfo.cs | 65 ++++++++++++ .../Aggregators/IAggregateLocalEpisode.cs | 1 + src/NzbDrone.Core/Parser/LanguageParser.cs | 73 ++++++++++++- .../Parser/Model/LocalEpisode.cs | 1 + .../Parser/Model/SubtitleTitleInfo.cs | 15 +++ 18 files changed, 494 insertions(+), 17 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs create mode 100644 src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs create mode 100644 src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs create mode 100644 src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs diff --git a/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs b/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs new file mode 100644 index 000000000..754c2308c --- /dev/null +++ b/src/NzbDrone.Core.Test/Datastore/Migration/198_parse_titles_from_existing_subtitle_filesFixture.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Dapper; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Datastore.Migration +{ + [TestFixture] + public class parse_title_from_existing_subtitle_filesFixture : MigrationTest + { + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.default.eng.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].eng.default.testtitle.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.eng.testtitle.forced.ass", "Name (2020)/Season 1/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.forced.eng.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].eng.forced.testtitle.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].forced.eng.testtitle.ass", "Name (2020)/Season 1/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.default.fra.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].fra.default.testtitle.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.fra.testtitle.forced.ass", "Name (2020)/Season 1/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.forced.fra.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].fra.forced.testtitle.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].forced.fra.testtitle.ass", "Name (2020)/Season 1/Name (2020).mkv", "testtitle", 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 0)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle - 3.default.eng.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle - 3.forced.eng.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].eng.forced.testtitle - 3.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].fra.default.testtitle - 3.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "testtitle", 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].3.default.eng.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].3.forced.eng.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].eng.forced.3.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 3)] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].fra.default.3.forced.ass", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", null, 3)] + public void should_process_file_with_missing_title(string subtitlePath, string episodePath, string title, int copy) + { + var now = DateTime.UtcNow; + + var db = WithDapperMigrationTestDb(c => + { + c.Insert.IntoTable("SubtitleFiles").Row(new + { + SeriesId = 1, + SeasonNumber = 1, + EpisodeFileId = 1, + RelativePath = subtitlePath, + Added = now, + LastUpdated = now, + Extension = Path.GetExtension(subtitlePath), + Language = 10, + LanguageTags = new List { "sdh" }.ToJson() + }); + + c.Insert.IntoTable("EpisodeFiles").Row(new + { + Id = 1, + SeriesId = 1, + RelativePath = episodePath, + OriginalFilePath = string.Empty, + Quality = new { }.ToJson(), + Size = 0, + DateAdded = now, + SeasonNumber = 1, + Languages = new List { 1 }.ToJson() + }); + }); + + var files = db.Query("SELECT * FROM \"SubtitleFiles\"").ToList(); + + files.Should().HaveCount(1); + + files.First().Title.Should().Be(title); + files.First().Copy.Should().Be(copy); + files.First().LanguageTags.Should().NotContain("sdh"); + files.First().Language.Should().NotBe(10); + } + } + + public class SubtitleFile198 + { + public int Id { get; set; } + public int SeriesId { get; set; } + public int? EpisodeFileId { get; set; } + public int? SeasonNumber { get; set; } + public string RelativePath { get; set; } + public DateTime Added { get; set; } + public DateTime LastUpdated { get; set; } + public string Extension { get; set; } + public int Language { get; set; } + public int Copy { get; set; } + public string Title { get; set; } + public List LanguageTags { get; set; } + } +} diff --git a/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs b/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs new file mode 100644 index 000000000..d5e4a472a --- /dev/null +++ b/src/NzbDrone.Core.Test/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfoFixture.cs @@ -0,0 +1,48 @@ +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.MediaFiles.EpisodeImport.Aggregation.Aggregators +{ + [TestFixture] + public class AggregateSubtitleInfoFixture : CoreTest + { + [TestCase("Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "", "Name (2020) - S01E20 - [AAC 2.0].default.eng.forced.ass")] + [TestCase("Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "", "Name (2020) - S01E20 - [AAC 2.0].eng.default.ass")] + [TestCase("Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "", "Name (2020) - S01E20 - [AAC 2.0].fra.ass")] + [TestCase("", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "Name (2020) - S01E20 - [AAC 2.0].default.eng.forced.ass")] + [TestCase("", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "Name (2020) - S01E20 - [AAC 2.0].eng.default.ass")] + [TestCase("", "Name (2020)/Season 1/Name (2020) - S01E20 - [AAC 2.0].mkv", "Name (2020) - S01E20 - [AAC 2.0].fra.ass")] + public void should_do_basic_parse(string relativePath, string originalFilePath, string path) + { + var episodeFile = new EpisodeFile + { + RelativePath = relativePath, + OriginalFilePath = originalFilePath + }; + + var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(episodeFile, path); + + subtitleTitleInfo.Title.Should().BeNull(); + subtitleTitleInfo.Copy.Should().Be(0); + } + + [TestCase("Default (2020)/Season 1/Default (2020) - S01E20 - [AAC 2.0].mkv", "Default (2020) - S01E20 - [AAC 2.0].default.eng.forced.ass")] + [TestCase("Default (2020)/Season 1/Default (2020) - S01E20 - [AAC 2.0].mkv", "Default (2020) - S01E20 - [AAC 2.0].eng.default.ass")] + [TestCase("Default (2020)/Season 1/Default (2020) - S01E20 - [AAC 2.0].mkv", "Default (2020) - S01E20 - [AAC 2.0].default.eng.testtitle.forced.ass")] + [TestCase("Default (2020)/Season 1/Default (2020) - S01E20 - [AAC 2.0].mkv", "Default (2020) - S01E20 - [AAC 2.0].testtitle.eng.default.ass")] + public void should_not_parse_default(string relativePath, string path) + { + var episodeFile = new EpisodeFile + { + RelativePath = relativePath + }; + + var subtitleTitleInfo = Subject.CleanSubtitleTitleInfo(episodeFile, path); + + subtitleTitleInfo.LanguageTags.Should().NotContain("default"); + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs b/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs index 42d06dc4c..0f3cf2749 100644 --- a/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/LanguageParserFixture.cs @@ -428,5 +428,38 @@ public void should_add_original_language_and_english_to_german_release_with_ml_t result.Languages.Should().Contain(Language.Original); result.Languages.Should().Contain(Language.English); } + + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.default.eng.forced.ass", new[] { "default", "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].eng.default.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.eng.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.forced.eng.ass", new[] { "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].eng.forced.testtitle.ass", new[] { "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].forced.eng.testtitle.ass", new[] { "forced" }, "testtitle", "English")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.default.fra.forced.ass", new[] { "default", "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].fra.default.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.fra.testtitle.forced.ass", new[] { "default", "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.forced.fra.ass", new[] { "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].fra.forced.testtitle.ass", new[] { "forced" }, "testtitle", "French")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].forced.fra.testtitle.ass", new[] { "forced" }, "testtitle", "French")] + public void should_parse_title_and_tags(string postTitle, string[] expectedTags, string expectedTitle, string expectedLanguage) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(postTitle); + + subtitleTitleInfo.LanguageTags.Should().BeEquivalentTo(expectedTags); + subtitleTitleInfo.Title.Should().BeEquivalentTo(expectedTitle); + subtitleTitleInfo.Language.Should().BeEquivalentTo((Language)expectedLanguage); + } + + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.forced.ass")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].default.ass")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].ass")] + [TestCase("Name (2020) - S01E20 - [AAC 2.0].testtitle.ass")] + public void should_not_parse_false_title(string postTitle) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(postTitle); + subtitleTitleInfo.Language.Should().Be(Language.Unknown); + subtitleTitleInfo.LanguageTags.Should().BeEmpty(); + subtitleTitleInfo.RawTitle.Should().BeNull(); + } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs b/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs new file mode 100644 index 000000000..5b8d18199 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/198_parse_titles_from_existing_subtitle_files.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using Dapper; +using FluentMigrator; +using NLog; +using NzbDrone.Common.Instrumentation; +using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(198)] + public class parse_title_from_existing_subtitle_files : NzbDroneMigrationBase + { + private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(AggregateSubtitleInfo)); + + protected override void MainDbUpgrade() + { + Alter.Table("SubtitleFiles").AddColumn("Title").AsString().Nullable(); + Alter.Table("SubtitleFiles").AddColumn("Copy").AsInt32().WithDefaultValue(0); + Execute.WithConnection(UpdateTitles); + } + + private void UpdateTitles(IDbConnection conn, IDbTransaction tran) + { + var updates = new List(); + + using (var cmd = conn.CreateCommand()) + { + cmd.Transaction = tran; + cmd.CommandText = "SELECT \"SubtitleFiles\".\"Id\", \"SubtitleFiles\".\"RelativePath\", \"EpisodeFiles\".\"RelativePath\", \"EpisodeFiles\".\"OriginalFilePath\" FROM \"SubtitleFiles\" JOIN \"EpisodeFiles\" ON \"SubtitleFiles\".\"EpisodeFileId\" = \"EpisodeFiles\".\"Id\""; + + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + var id = reader.GetInt32(0); + var relativePath = reader.GetString(1); + var episodeFileRelativePath = reader.GetString(2); + var episodeFileOriginalFilePath = reader.GetString(3); + + var subtitleTitleInfo = CleanSubtitleTitleInfo(episodeFileRelativePath, episodeFileOriginalFilePath, relativePath); + + updates.Add(new + { + Id = id, + Title = subtitleTitleInfo.Title, + Language = subtitleTitleInfo.Language, + LanguageTags = subtitleTitleInfo.LanguageTags, + Copy = subtitleTitleInfo.Copy + }); + } + } + + var updateSubtitleFilesSql = "UPDATE \"SubtitleFiles\" SET \"Title\" = @Title, \"Copy\" = @Copy, \"Language\" = @Language, \"LanguageTags\" = @LanguageTags, \"LastUpdated\" = CURRENT_TIMESTAMP WHERE \"Id\" = @Id"; + conn.Execute(updateSubtitleFilesSql, updates, transaction: tran); + } + + private static SubtitleTitleInfo CleanSubtitleTitleInfo(string relativePath, string originalFilePath, string path) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path); + + var episodeFileTitle = Path.GetFileNameWithoutExtension(relativePath); + var originalEpisodeFileTitle = Path.GetFileNameWithoutExtension(originalFilePath) ?? string.Empty; + + if (subtitleTitleInfo.TitleFirst && (episodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalEpisodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase))) + { + Logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle); + + subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path); + } + + var cleanedTags = subtitleTitleInfo.LanguageTags.Where(t => !episodeFileTitle.Contains(t, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count) + { + Logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); + subtitleTitleInfo.LanguageTags = cleanedTags; + } + + return subtitleTitleInfo; + } + } +} diff --git a/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs b/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs index ee1c63179..6c5a5481e 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs @@ -5,7 +5,6 @@ using NzbDrone.Common.Extensions; using NzbDrone.Core.Extras.Files; using NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation; -using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Tv; @@ -71,15 +70,19 @@ public override IEnumerable ProcessFiles(Series series, List continue; } + var firstEpisode = localEpisode.Episodes.First(); + var subtitleFile = new SubtitleFile { SeriesId = series.Id, SeasonNumber = localEpisode.SeasonNumber, - EpisodeFileId = localEpisode.Episodes.First().EpisodeFileId, + EpisodeFileId = firstEpisode.EpisodeFileId, RelativePath = series.Path.GetRelativePath(possibleSubtitleFile), - Language = LanguageParser.ParseSubtitleLanguage(possibleSubtitleFile), - LanguageTags = LanguageParser.ParseLanguageTags(possibleSubtitleFile), - Extension = extension + Language = localEpisode.SubtitleInfo.Language, + LanguageTags = localEpisode.SubtitleInfo.LanguageTags, + Title = localEpisode.SubtitleInfo.Title, + Extension = extension, + Copy = localEpisode.SubtitleInfo.Copy }; subtitleFiles.Add(subtitleFile); diff --git a/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs b/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs index 41a28ef4a..c3a3b7db1 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/SubtitleFile.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Text; using NzbDrone.Core.Extras.Files; using NzbDrone.Core.Languages; @@ -13,15 +14,40 @@ public SubtitleFile() public Language Language { get; set; } - public string AggregateString => Language + LanguageTagsAsString + Extension; + public string AggregateString => Language + Title + LanguageTagsAsString + Extension; + + public int Copy { get; set; } public List LanguageTags { get; set; } + public string Title { get; set; } + private string LanguageTagsAsString => string.Join(".", LanguageTags); public override string ToString() { - return $"[{Id}] {RelativePath} ({Language}{(LanguageTags.Count > 0 ? "." : "")}{LanguageTagsAsString}{Extension})"; + var stringBuilder = new StringBuilder(); + stringBuilder.AppendFormat("[{0}] ", Id); + stringBuilder.Append(RelativePath); + + stringBuilder.Append(" ("); + stringBuilder.Append(Language); + if (Title is not null) + { + stringBuilder.Append('.'); + stringBuilder.Append(Title); + } + + if (LanguageTags.Count > 0) + { + stringBuilder.Append('.'); + stringBuilder.Append(LanguageTagsAsString); + } + + stringBuilder.Append(Extension); + stringBuilder.Append(')'); + + return stringBuilder.ToString(); } } } diff --git a/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs b/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs index 0b3a4e557..465783f16 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/SubtitleService.cs @@ -76,16 +76,20 @@ public override IEnumerable MoveFilesAfterRename(Series series, List< foreach (var group in groupedExtraFilesForEpisodeFile) { - var groupCount = group.Count(); - var copy = 1; + var multipleCopies = group.Count() > 1; + var orderedGroup = group.OrderBy(s => -s.Copy).ToList(); + var copy = group.First().Copy; - foreach (var subtitleFile in group) + foreach (var subtitleFile in orderedGroup) { - var suffix = GetSuffix(subtitleFile.Language, copy, subtitleFile.LanguageTags, groupCount > 1); + if (multipleCopies && subtitleFile.Copy == 0) + { + subtitleFile.Copy = ++copy; + } + + var suffix = GetSuffix(subtitleFile.Language, subtitleFile.Copy, subtitleFile.LanguageTags, multipleCopies, subtitleFile.Title); movedFiles.AddIfNotNull(MoveFile(series, episodeFile, subtitleFile, suffix)); - - copy++; } } } @@ -229,11 +233,22 @@ public override IEnumerable ImportFiles(LocalEpisode localEpisode, Ep return importedFiles; } - private string GetSuffix(Language language, int copy, List languageTags, bool multipleCopies = false) + private string GetSuffix(Language language, int copy, List languageTags, bool multipleCopies = false, string title = null) { var suffixBuilder = new StringBuilder(); - if (multipleCopies) + if (title is not null) + { + suffixBuilder.Append('.'); + suffixBuilder.Append(title); + + if (multipleCopies) + { + suffixBuilder.Append(" - "); + suffixBuilder.Append(copy); + } + } + else if (multipleCopies) { suffixBuilder.Append('.'); suffixBuilder.Append(copy); diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/AggregationService.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/AggregationService.cs index 69dd300cc..49cdda995 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/AggregationService.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/AggregationService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Core.Configuration; @@ -30,7 +31,7 @@ public AggregationService(IEnumerable augmenters, IConfigService configService, Logger logger) { - _augmenters = augmenters; + _augmenters = augmenters.OrderBy(a => a.Order).ToList(); _diskProvider = diskProvider; _videoFileInfoReader = videoFileInfoReader; _configService = configService; diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateEpisodes.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateEpisodes.cs index f243537e5..3c978576e 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateEpisodes.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateEpisodes.cs @@ -10,6 +10,8 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public class AggregateEpisodes : IAggregateLocalEpisode { + public int Order => 1; + private readonly IParsingService _parsingService; public AggregateEpisodes(IParsingService parsingService) diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateLanguage.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateLanguage.cs index 07c0dd34e..fade77302 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateLanguage.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateLanguage.cs @@ -10,6 +10,8 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public class AggregateLanguage : IAggregateLocalEpisode { + public int Order => 1; + private readonly List _augmentLanguages; private readonly Logger _logger; diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateQuality.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateQuality.cs index 434663f5e..234568a92 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateQuality.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateQuality.cs @@ -10,6 +10,8 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public class AggregateQuality : IAggregateLocalEpisode { + public int Order => 1; + private readonly List _augmentQualities; private readonly Logger _logger; diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseGroup.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseGroup.cs index a8b1d1b8e..db909a4c7 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseGroup.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseGroup.cs @@ -6,6 +6,8 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public class AggregateReleaseGroup : IAggregateLocalEpisode { + public int Order => 1; + public LocalEpisode Aggregate(LocalEpisode localEpisode, DownloadClientItem downloadClientItem) { // Prefer ReleaseGroup from DownloadClient/Folder if they're not a season pack diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseInfo.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseInfo.cs index 111b465f6..c68464049 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseInfo.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateReleaseInfo.cs @@ -8,6 +8,8 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public class AggregateReleaseInfo : IAggregateLocalEpisode { + public int Order => 1; + private readonly IHistoryService _historyService; public AggregateReleaseInfo(IHistoryService historyService) diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs new file mode 100644 index 000000000..5beabf7d5 --- /dev/null +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/AggregateSubtitleInfo.cs @@ -0,0 +1,65 @@ +using System; +using System.IO; +using System.Linq; +using NLog; +using NzbDrone.Core.Download; +using NzbDrone.Core.Extras.Subtitles; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators +{ + public class AggregateSubtitleInfo : IAggregateLocalEpisode + { + public int Order => 2; + + private readonly Logger _logger; + + public AggregateSubtitleInfo(Logger logger) + { + _logger = logger; + } + + public LocalEpisode Aggregate(LocalEpisode localEpisode, DownloadClientItem downloadClientItem) + { + var path = localEpisode.Path; + var isSubtitleFile = SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(path)); + + if (!isSubtitleFile) + { + return localEpisode; + } + + var firstEpisode = localEpisode.Episodes.First(); + var episodeFile = firstEpisode.EpisodeFile.Value; + localEpisode.SubtitleInfo = CleanSubtitleTitleInfo(episodeFile, path); + + return localEpisode; + } + + public SubtitleTitleInfo CleanSubtitleTitleInfo(EpisodeFile episodeFile, string path) + { + var subtitleTitleInfo = LanguageParser.ParseSubtitleLanguageInformation(path); + + var episodeFileTitle = Path.GetFileNameWithoutExtension(episodeFile.RelativePath); + var originalEpisodeFileTitle = Path.GetFileNameWithoutExtension(episodeFile.OriginalFilePath) ?? string.Empty; + + if (subtitleTitleInfo.TitleFirst && (episodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase) || originalEpisodeFileTitle.Contains(subtitleTitleInfo.RawTitle, StringComparison.OrdinalIgnoreCase))) + { + _logger.Debug("Subtitle title '{0}' is in episode file title '{1}'. Removing from subtitle title.", subtitleTitleInfo.RawTitle, episodeFileTitle); + + subtitleTitleInfo = LanguageParser.ParseBasicSubtitle(path); + } + + var cleanedTags = subtitleTitleInfo.LanguageTags.Where(t => !episodeFileTitle.Contains(t, StringComparison.OrdinalIgnoreCase)).ToList(); + + if (cleanedTags.Count != subtitleTitleInfo.LanguageTags.Count) + { + _logger.Debug("Removed language tags '{0}' from subtitle title '{1}'.", string.Join(", ", subtitleTitleInfo.LanguageTags.Except(cleanedTags)), subtitleTitleInfo.RawTitle); + subtitleTitleInfo.LanguageTags = cleanedTags; + } + + return subtitleTitleInfo; + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/IAggregateLocalEpisode.cs b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/IAggregateLocalEpisode.cs index e32cf7a02..7cc4f58be 100644 --- a/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/IAggregateLocalEpisode.cs +++ b/src/NzbDrone.Core/MediaFiles/EpisodeImport/Aggregation/Aggregators/IAggregateLocalEpisode.cs @@ -5,6 +5,7 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Aggregation.Aggregators { public interface IAggregateLocalEpisode { + int Order { get; } LocalEpisode Aggregate(LocalEpisode localEpisode, DownloadClientItem downloadClientItem); } } diff --git a/src/NzbDrone.Core/Parser/LanguageParser.cs b/src/NzbDrone.Core/Parser/LanguageParser.cs index 149af2329..7344bab08 100644 --- a/src/NzbDrone.Core/Parser/LanguageParser.cs +++ b/src/NzbDrone.Core/Parser/LanguageParser.cs @@ -7,6 +7,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.Languages; +using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Parser { @@ -28,7 +29,11 @@ public static class LanguageParser private static readonly Regex GermanDualLanguageRegex = new (@"(?[a-z]{2,3})([-_. ](?full|forced|foreign|default|cc|psdh|sdh))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex SubtitleLanguageRegex = new Regex(".+?([-_. ](?full|forced|foreign|default|cc|psdh|sdh))*[-_. ](?[a-z]{2,3})([-_. ](?full|forced|foreign|default|cc|psdh|sdh))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex SubtitleLanguageTitleRegex = new Regex(".+?(\\.((?full|forced|foreign|default|cc|psdh|sdh)|(?[a-z]{2,3})))*\\.(?[^.]*)(\\.((?<tags2>full|forced|foreign|default|cc|psdh|sdh)|(?<iso_code>[a-z]{2,3})))*$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static readonly Regex SubtitleTitleRegex = new Regex("((?<title>.+) - )?(?<copy>\\d+)$", RegexOptions.Compiled); public static List<Language> ParseLanguages(string title) { @@ -249,6 +254,72 @@ public static Language ParseSubtitleLanguage(string fileName) return Language.Unknown; } + public static SubtitleTitleInfo ParseBasicSubtitle(string fileName) + { + return new SubtitleTitleInfo + { + TitleFirst = false, + LanguageTags = ParseLanguageTags(fileName), + Language = ParseSubtitleLanguage(fileName) + }; + } + + public static SubtitleTitleInfo ParseSubtitleLanguageInformation(string fileName) + { + var simpleFilename = Path.GetFileNameWithoutExtension(fileName); + var matchTitle = SubtitleLanguageTitleRegex.Match(simpleFilename); + + if (!matchTitle.Groups["title"].Success || (matchTitle.Groups["iso_code"].Captures.Count is var languageCodeNumber && languageCodeNumber != 1)) + { + Logger.Debug("Could not parse a title from subtitle file: {0}. Falling back to parsing without title.", fileName); + + return ParseBasicSubtitle(fileName); + } + + var isoCode = matchTitle.Groups["iso_code"].Value; + var isoLanguage = IsoLanguages.Find(isoCode.ToLower()); + + var language = isoLanguage?.Language ?? Language.Unknown; + + var languageTags = matchTitle.Groups["tags1"].Captures + .Union(matchTitle.Groups["tags2"].Captures) + .Cast<Capture>() + .Where(tag => !tag.Value.Empty()) + .Select(tag => tag.Value.ToLower()); + var rawTitle = matchTitle.Groups["title"].Value; + + var subtitleTitleInfo = new SubtitleTitleInfo + { + TitleFirst = matchTitle.Groups["tags1"].Captures.Empty(), + LanguageTags = languageTags.ToList(), + RawTitle = rawTitle, + Language = language + }; + + UpdateTitleAndCopyFromTitle(subtitleTitleInfo); + + return subtitleTitleInfo; + } + + public static void UpdateTitleAndCopyFromTitle(SubtitleTitleInfo subtitleTitleInfo) + { + if (subtitleTitleInfo.RawTitle is null) + { + subtitleTitleInfo.Title = null; + subtitleTitleInfo.Copy = 0; + } + else if (SubtitleTitleRegex.Match(subtitleTitleInfo.RawTitle) is var match && match.Success) + { + subtitleTitleInfo.Title = match.Groups["title"].Success ? match.Groups["title"].ToString() : null; + subtitleTitleInfo.Copy = int.Parse(match.Groups["copy"].ToString()); + } + else + { + subtitleTitleInfo.Title = subtitleTitleInfo.RawTitle; + subtitleTitleInfo.Copy = 0; + } + } + public static List<string> ParseLanguageTags(string fileName) { try diff --git a/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs b/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs index 470310b7c..060a14e6e 100644 --- a/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs +++ b/src/NzbDrone.Core/Parser/Model/LocalEpisode.cs @@ -44,6 +44,7 @@ public LocalEpisode() public bool FileRenamedAfterScriptImport { get; set; } public bool ShouldImportExtras { get; set; } public List<string> PossibleExtraFiles { get; set; } + public SubtitleTitleInfo SubtitleInfo { get; set; } public int SeasonNumber { diff --git a/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs b/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs new file mode 100644 index 000000000..29ea84377 --- /dev/null +++ b/src/NzbDrone.Core/Parser/Model/SubtitleTitleInfo.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using NzbDrone.Core.Languages; + +namespace NzbDrone.Core.Parser.Model +{ + public class SubtitleTitleInfo + { + public List<string> LanguageTags { get; set; } + public Language Language { get; set; } + public string RawTitle { get; set; } + public string Title { get; set; } + public int Copy { get; set; } + public bool TitleFirst { get; set; } + } +}