Fixed: Quality Profiles resetting Custom Format scores during housekeeping

Closes #5359

src/NzbDrone.Core.Test/DecisionEngineTests/UpgradeAllowedSpecificationFixture .cs

@@ -41,13 +41,11 @@ public void Setup()
                 {
                     new ProfileFormatItem
                     {
-                        Id = 1,
                         Format = _customFormatOne,
                         Score = 50
                     },
                     new ProfileFormatItem
                     {
-                        Id = 1,
                         Format = _customFormatTwo,
                         Score = 100
                     }

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupQualityProfileFormatItemsFixture.cs

@@ -21,6 +21,9 @@ public void Setup()
         {
             Mocker.SetConstant<IQualityProfileFormatItemsCleanupRepository>(
                 new QualityProfileFormatItemsCleanupRepository(Mocker.Resolve<IMainDatabase>(), Mocker.Resolve<IEventAggregator>()));
+
+            Mocker.SetConstant<ICustomFormatRepository>(
+                new CustomFormatRepository(Mocker.Resolve<IMainDatabase>(), Mocker.Resolve<IEventAggregator>()));
         }
 
         [Test]
@@ -30,7 +33,12 @@ public void should_remove_orphaned_custom_formats()
                 .With(h => h.Items = Qualities.QualityFixture.GetDefaultQualities())
                 .With(h => h.MinFormatScore = 50)
                 .With(h => h.CutoffFormatScore = 100)
-                .With(h => h.FormatItems = Builder<ProfileFormatItem>.CreateListOfSize(1).Build().ToList())
+                .With(h => h.FormatItems = new List<ProfileFormatItem>
+                {
+                    Builder<ProfileFormatItem>.CreateNew()
+                        .With(c => c.Format = new CustomFormat("My Custom Format") { Id = 0 })
+                        .Build()
+                })
                 .BuildNew();
 
             Db.Insert(qualityProfile);
@@ -62,9 +70,10 @@ public void should_not_remove_unorphaned_custom_formats()
                 .With(h => h.CutoffFormatScore = cutoffFormatScore)
                 .With(h => h.FormatItems = new List<ProfileFormatItem>
                 {
-                    Builder<ProfileFormatItem>.CreateNew().With(f => f.Id = customFormat.Id).Build()
+                    Builder<ProfileFormatItem>.CreateNew()
+                        .With(c => c.Format = customFormat)
+                        .Build()
                 })
-
                 .BuildNew();
 
             Db.Insert(qualityProfile);
@@ -77,5 +86,49 @@ public void should_not_remove_unorphaned_custom_formats()
             result.First().MinFormatScore.Should().Be(minFormatScore);
             result.First().CutoffFormatScore.Should().Be(cutoffFormatScore);
         }
+
+        [Test]
+        public void should_add_missing_custom_formats()
+        {
+            var minFormatScore = 50;
+            var cutoffFormatScore = 100;
+
+            var customFormat1 = Builder<CustomFormat>.CreateNew()
+                .With(h => h.Id = 1)
+                .With(h => h.Name = "Custom Format 1")
+                .With(h => h.Specifications = new List<ICustomFormatSpecification>())
+                .BuildNew();
+
+            var customFormat2 = Builder<CustomFormat>.CreateNew()
+                .With(h => h.Id = 2)
+                .With(h => h.Name = "Custom Format 2")
+                .With(h => h.Specifications = new List<ICustomFormatSpecification>())
+                .BuildNew();
+
+            Db.Insert(customFormat1);
+            Db.Insert(customFormat2);
+
+            var qualityProfile = Builder<QualityProfile>.CreateNew()
+                .With(h => h.Items = Qualities.QualityFixture.GetDefaultQualities())
+                .With(h => h.MinFormatScore = minFormatScore)
+                .With(h => h.CutoffFormatScore = cutoffFormatScore)
+                .With(h => h.FormatItems = new List<ProfileFormatItem>
+                {
+                    Builder<ProfileFormatItem>.CreateNew()
+                        .With(c => c.Format = customFormat1)
+                        .Build()
+                })
+                .BuildNew();
+
+            Db.Insert(qualityProfile);
+
+            Subject.Clean();
+            var result = AllStoredModels;
+
+            result.Should().HaveCount(1);
+            result.First().FormatItems.Should().HaveCount(2);
+            result.First().MinFormatScore.Should().Be(minFormatScore);
+            result.First().CutoffFormatScore.Should().Be(cutoffFormatScore);
+        }
     }
 }

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupQualityProfileFormatItems.cs

@@ -1,35 +1,64 @@
 using System.Collections.Generic;
 using System.Linq;
-using Dapper;
 using NzbDrone.Common.Extensions;
+using NzbDrone.Core.CustomFormats;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Profiles;
 using NzbDrone.Core.Profiles.Qualities;
 
 namespace NzbDrone.Core.Housekeeping.Housekeepers
 {
     public class CleanupQualityProfileFormatItems : IHousekeepingTask
     {
-        private readonly IMainDatabase _database;
         private readonly IQualityProfileFormatItemsCleanupRepository _repository;
+        private readonly ICustomFormatRepository _customFormatRepository;
 
-        public CleanupQualityProfileFormatItems(IMainDatabase database, IQualityProfileFormatItemsCleanupRepository repository)
+        public CleanupQualityProfileFormatItems(IQualityProfileFormatItemsCleanupRepository repository,
+                                                ICustomFormatRepository customFormatRepository)
         {
-            _database = database;
             _repository = repository;
+            _customFormatRepository = customFormatRepository;
         }
 
         public void Clean()
         {
-            var customFormatIds = GetCustomFormatIds();
+            var test = _customFormatRepository.All();
+            var customFormats = _customFormatRepository.All().ToDictionary(c => c.Id);
             var profiles = _repository.All();
             var updatedProfiles = new List<QualityProfile>();
 
             foreach (var profile in profiles)
             {
-                var formatItems = profile.FormatItems.Where(f => customFormatIds.Contains(f.Id)).ToList();
+                var formatItems = new List<ProfileFormatItem>();
 
-                if (formatItems.Count != profile.FormatItems.Count)
+                // Make sure the profile doesn't include formats that have been removed
+                profile.FormatItems.ForEach(p =>
+                {
+                    if (p.Format != null && customFormats.ContainsKey(p.Format.Id))
+                    {
+                        formatItems.Add(p);
+                    }
+                });
+
+                // Make sure the profile includes all available formats
+                foreach (var customFormat in customFormats)
+                {
+                    if (formatItems.None(f => f.Format.Id == customFormat.Key))
+                    {
+                        formatItems.Insert(0, new ProfileFormatItem
+                        {
+                            Format = customFormat.Value,
+                            Score = 0
+                        });
+                    }
+                }
+
+                var previousIds = profile.FormatItems.Select(i => i.Format.Id).ToList();
+                var ids = formatItems.Select(i => i.Format.Id).ToList();
+
+                // Update the profile if any formats were added or removed
+                if (ids.Except(previousIds).Any() || previousIds.Except(ids).Any())
                 {
                     profile.FormatItems = formatItems;
 
@@ -48,14 +77,6 @@ public void Clean()
                 _repository.SetFields(updatedProfiles, p => p.FormatItems, p => p.MinFormatScore, p => p.CutoffFormatScore);
             }
         }
-
-        private HashSet<int> GetCustomFormatIds()
-        {
-            using (var mapper = _database.OpenConnection())
-            {
-                return new HashSet<int>(mapper.Query<int>("SELECT Id FROM CustomFormats"));
-            }
-        }
     }
 
     public interface IQualityProfileFormatItemsCleanupRepository : IBasicRepository<QualityProfile>

src/NzbDrone.Core/Profiles/ProfileFormatItem.cs

@@ -1,4 +1,3 @@
-using System.Text.Json.Serialization;
 using NzbDrone.Core.CustomFormats;
 using NzbDrone.Core.Datastore;
 
@@ -6,8 +5,6 @@ namespace NzbDrone.Core.Profiles
 {
     public class ProfileFormatItem : IEmbeddedDocument
     {
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-        public int Id { get; set; }
         public CustomFormat Format { get; set; }
         public int Score { get; set; }
     }

src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs

@@ -221,7 +221,6 @@ public QualityProfile GetDefaultProfile(string name, Quality cutoff = null, para
 
             var formatItems = _formatService.All().Select(format => new ProfileFormatItem
             {
-                Id = format.Id,
                 Score = 0,
                 Format = format
             }).ToList();