self-hosted/Sonarr
1
0
Fork 0
mirror of https://github.com/Sonarr/Sonarr.git synced 2024-12-14 11:23:42 +02:00
Code Issues Releases Activity

Improved Command Queue

Browse Source 
New: Limit concurrent tasks that require disk access
New: Prevent other tasks from running during application update
This commit is contained in:
Mark McDowall 2018-08-27 20:27:45 -07:00 committed by Taloth Saldono
parent 07be9cf47a
commit 36b9c51163
18 changed files with 251 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/NzbDrone.Api/Commands/CommandModule.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using Sonarr.Http.Extensions;
@ -10,7 +10,6 @@
using NzbDrone.Core.ProgressMessaging;
using NzbDrone.SignalR;
using Sonarr.Http;
using Sonarr.Http.Mapping;
using Sonarr.Http.Validation;

7
src/NzbDrone.Common/Extensions/StringExtensions.cs
View File

@ -9,6 +9,8 @@ namespace NzbDrone.Common.Extensions
{
public static class StringExtensions
{
private static readonly Regex CamelCaseRegex = new Regex("(?<!^)[A-Z]", RegexOptions.Compiled);
public static string NullSafe(this string target)
{
return ((object)target).NullSafe().ToString();
@ -133,5 +135,10 @@ public static string FromOctalString(this string octalValue)
return Encoding.ASCII.GetString(new [] { byteResult });
}
public static string SplitCamelCase(this string input)
{
return CamelCaseRegex.Replace(input, match => " " + match.Value);
}
}
}

6
src/NzbDrone.Core.Test/Messaging/Commands/CommandExecutorFixture.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
@ -15,7 +15,7 @@ namespace NzbDrone.Core.Test.Messaging.Commands
[TestFixture]
public class CommandExecutorFixture : TestBase<CommandExecutor>
{
private BlockingCollection<CommandModel> _commandQueue;
private CommandQueue _commandQueue;
private Mock<IExecute<CommandA>> _executorA;
private Mock<IExecute<CommandB>> _executorB;
private bool _commandExecuted = false;
@ -46,7 +46,7 @@ public void TearDown()
private void GivenCommandQueue()
{
_commandQueue = new BlockingCollection<CommandModel>(new CommandQueue());
_commandQueue = new CommandQueue();
Mocker.GetMock<IManageCommandQueue>()
.Setup(s => s.Queue(It.IsAny<CancellationToken>()))

7
src/NzbDrone.Core.Test/Messaging/Commands/CommandQueueManagerFixture.cs
View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using FluentAssertions;
using Moq;
@ -42,6 +43,10 @@ public void should_not_remove_commands_for_five_minutes_after_they_end()
{
var command = Subject.Push<CheckForFinishedDownloadCommand>(new CheckForFinishedDownloadCommand());
// Start the command to mimic CommandQueue's behaviour
command.StartedAt = DateTime.Now;
command.Status = CommandStatus.Started;
Subject.Start(command);
Subject.Complete(command, "All done");
Subject.CleanCommands();

4
src/NzbDrone.Core/Download/CheckForFinishedDownloadCommand.cs
View File

@ -1,9 +1,9 @@
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.Download
{
public class CheckForFinishedDownloadCommand : Command
{
public override bool RequiresDiskAccess => true;
}
}

3
src/NzbDrone.Core/MediaFiles/Commands/RenameFilesCommand.cs
View File

@ -9,6 +9,7 @@ public class RenameFilesCommand : Command
public List<int> Files { get; set; }
public override bool SendUpdatesToClient => true;
public override bool RequiresDiskAccess => true;
public RenameFilesCommand()
{
@ -20,4 +21,4 @@ public RenameFilesCommand(int seriesId, List<int> files)
Files = files;
}
}
}
}

1
src/NzbDrone.Core/MediaFiles/Commands/RenameSeriesCommand.cs
View File

@ -8,6 +8,7 @@ public class RenameSeriesCommand : Command
public List<int> SeriesIds { get; set; }
public override bool SendUpdatesToClient => true;
public override bool RequiresDiskAccess => true;
public RenameSeriesCommand()
{

3
src/NzbDrone.Core/MediaFiles/EpisodeImport/Manual/ManualImportCommand.cs
View File

@ -1,4 +1,4 @@
using System.Collections.Generic;
using System.Collections.Generic;
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.MediaFiles.EpisodeImport.Manual
@ -8,6 +8,7 @@ public class ManualImportCommand : Command
public List<ManualImportFile> Files { get; set; }
public override bool SendUpdatesToClient => true;
public override bool RequiresDiskAccess => true;
public ImportMode ImportMode { get; set; }
}

2
src/NzbDrone.Core/Messaging/Commands/Command.cs
View File

@ -21,6 +21,8 @@ public virtual bool SendUpdatesToClient
public virtual bool UpdateScheduledTask => true;
public virtual string CompletionMessage => "Completed";
public virtual bool RequiresDiskAccess => false;
public virtual bool IsExclusive => false;
public string Name { get; private set; }
public DateTime? LastExecutionTime { get; set; }

188
src/NzbDrone.Core/Messaging/Commands/CommandQueue.cs
View File

@ -1,27 +1,36 @@
using System;
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
namespace NzbDrone.Core.Messaging.Commands
{
public class CommandQueue : IProducerConsumerCollection<CommandModel>
public class CommandQueue : IEnumerable
{
private object Mutex = new object();
private List<CommandModel> _items;
private readonly object _mutex = new object();
private readonly List<CommandModel> _items;
public CommandQueue()
{
_items = new List<CommandModel>();
}
public int Count => _items.Count;
public void Add(CommandModel item)
{
lock (_mutex)
{
_items.Add(item);
}
}
public IEnumerator<CommandModel> GetEnumerator()
{
List<CommandModel> copy = null;
lock (Mutex)
lock (_mutex)
{
copy = new List<CommandModel>(_items);
}
@ -34,77 +43,140 @@ IEnumerator IEnumerable.GetEnumerator()
return GetEnumerator();
}
public void CopyTo(Array array, int index)
public List<CommandModel> All()
{
lock (Mutex)
List<CommandModel> rval = null;
lock (_mutex)
{
((ICollection)_items).CopyTo(array, index);
rval = _items;
}
return rval;
}
public CommandModel Find(int id)
{
return All().FirstOrDefault(q => q.Id == id);
}
public void RemoveMany(IEnumerable<CommandModel> commands)
{
lock (_mutex)
{
foreach (var command in commands)
{
_items.Remove(command);
}
}
}
public bool RemoveIfQueued(int id)
{
var rval = false;
lock (_mutex)
{
var command = _items.FirstOrDefault(q => q.Id == id);
if (command?.Status == CommandStatus.Queued)
{
_items.Remove(command);
rval = true;
}
}
return rval;
}
public List<CommandModel> QueuedOrStarted()
{
return All().Where(q => q.Status == CommandStatus.Queued || q.Status == CommandStatus.Started)
.ToList();
}
public IEnumerable<CommandModel> GetConsumingEnumerable()
{
return GetConsumingEnumerable(CancellationToken.None);
}
public IEnumerable<CommandModel> GetConsumingEnumerable(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
if (TryGet(out var command))
{
yield return command;
}
Thread.Sleep(10);
}
}
public int Count => _items.Count;
public object SyncRoot => Mutex;
public bool IsSynchronized => true;
public void CopyTo(CommandModel[] array, int index)
public bool TryGet(out CommandModel item)
{
lock (Mutex)
{
_items.CopyTo(array, index);
}
}
var rval = true;
item = default(CommandModel);
public bool TryAdd(CommandModel item)
{
Add(item);
return true;
}
public bool TryTake(out CommandModel item)
{
bool rval = true;
lock (Mutex)
lock (_mutex)
{
if (_items.Count == 0)
{
item = default(CommandModel);
rval = false;
}
else
{
item = _items.Where(c => c.Status == CommandStatus.Queued)
.OrderByDescending(c => c.Priority)
.ThenBy(c => c.QueuedAt)
.First();
var startedCommands = _items.Where(c => c.Status == CommandStatus.Started)
.ToList();
_items.Remove(item);
var localItem = _items.Where(c =>
{
// If an executing command requires disk access don't return a command that
// requires disk access. A lower priority or later queued task could be returned
// instead, but that will allow other tasks to execute whiule waiting for disk access.
if (startedCommands.Any(x => x.Body.RequiresDiskAccess))
{
return c.Status == CommandStatus.Queued &&
!c.Body.RequiresDiskAccess;
}
return c.Status == CommandStatus.Queued;
})
.OrderByDescending(c => c.Priority)
.ThenBy(c => c.QueuedAt)
.FirstOrDefault();
// Nothing queued that meets the requirements
if (localItem == null)
{
rval = false;
}
// If any executing command is exclusive don't want return another command until it completes.
else if (startedCommands.Any(c => c.Body.IsExclusive))
{
rval = false;
}
// If the next command to execute is exclusive wait for executing commands to complete.
// This will prevent other tasks from starting so the exclusive task executes in the order it should.
else if (localItem.Body.IsExclusive && startedCommands.Any())
{
rval = false;
}
// A command ready to execute
else
{
localItem.StartedAt = DateTime.UtcNow;
localItem.Status = CommandStatus.Started;
item = localItem;
}
}
}
return rval;
}
public CommandModel[] ToArray()
{
CommandModel[] rval = null;
lock (Mutex)
{
rval = _items.ToArray();
}
return rval;
}
public void Add(CommandModel item)
{
lock (Mutex)
{
_items.Add(item);
}
}
}
}

85
src/NzbDrone.Core/Messaging/Commands/CommandQueueManager.cs
View File

@ -1,15 +1,15 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using NLog;
using NzbDrone.Common;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnsureThat;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Events;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using NzbDrone.Core.Exceptions;
namespace NzbDrone.Core.Messaging.Commands
{
@ -19,6 +19,7 @@ public interface IManageCommandQueue
CommandModel Push<TCommand>(TCommand command, CommandPriority priority = CommandPriority.Normal, CommandTrigger trigger = CommandTrigger.Unspecified) where TCommand : Command;
CommandModel Push(string commandName, DateTime? lastExecutionTime, CommandPriority priority = CommandPriority.Normal, CommandTrigger trigger = CommandTrigger.Unspecified);
IEnumerable<CommandModel> Queue(CancellationToken cancellationToken);
List<CommandModel> All();
CommandModel Get(int id);
List<CommandModel> GetStarted();
void SetMessage(CommandModel command, string message);
@ -26,6 +27,7 @@ public interface IManageCommandQueue
void Complete(CommandModel command, string message);
void Fail(CommandModel command, string message, Exception e);
void Requeue();
void Cancel(int id);
void CleanCommands();
}
@ -35,20 +37,17 @@ public class CommandQueueManager : IManageCommandQueue, IHandle<ApplicationStart
private readonly IServiceFactory _serviceFactory;
private readonly Logger _logger;
private readonly ICached<CommandModel> _commandCache;
private readonly BlockingCollection<CommandModel> _commandQueue;
private readonly CommandQueue _commandQueue;
public CommandQueueManager(ICommandRepository repo,
IServiceFactory serviceFactory,
ICacheManager cacheManager,
Logger logger)
{
_repo = repo;
_serviceFactory = serviceFactory;
_logger = logger;
_commandCache = cacheManager.GetCache<CommandModel>(GetType());
_commandQueue = new BlockingCollection<CommandModel>(new CommandQueue());
_commandQueue = new CommandQueue();
}
public List<CommandModel> PushMany<TCommand>(List<TCommand> commands) where TCommand : Command
@ -56,8 +55,7 @@ public List<CommandModel> PushMany<TCommand>(List<TCommand> commands) where TCom
_logger.Trace("Publishing {0} commands", commands.Count);
var commandModels = new List<CommandModel>();
var existingCommands = _commandCache.Values.Where(q => q.Status == CommandStatus.Queued ||
q.Status == CommandStatus.Started).ToList();
var existingCommands = _commandQueue.QueuedOrStarted();
foreach (var command in commands)
{
@ -85,7 +83,6 @@ public List<CommandModel> PushMany<TCommand>(List<TCommand> commands) where TCom
foreach (var commandModel in commandModels)
{
_commandCache.Set(commandModel.Id.ToString(), commandModel);
_commandQueue.Add(commandModel);
}
@ -122,7 +119,6 @@ public CommandModel Push<TCommand>(TCommand command, CommandPriority priority =
_logger.Trace("Inserting new command: {0}", commandModel.Name);
_repo.Insert(commandModel);
_commandCache.Set(commandModel.Id.ToString(), commandModel);
_commandQueue.Add(commandModel);
return commandModel;
@ -142,30 +138,39 @@ public IEnumerable<CommandModel> Queue(CancellationToken cancellationToken)
return _commandQueue.GetConsumingEnumerable(cancellationToken);
}
public List<CommandModel> All()
{
_logger.Trace("Getting all commands");
return _commandQueue.All();
}
public CommandModel Get(int id)
{
return _commandCache.Get(id.ToString(), () => FindCommand(_repo.Get(id)));
var command = _commandQueue.Find(id);
if (command == null)
{
command = _repo.Get(id);
}
return command;
}
public List<CommandModel> GetStarted()
{
_logger.Trace("Getting started commands");
return _commandCache.Values.Where(c => c.Status == CommandStatus.Started).ToList();
return _commandQueue.All().Where(c => c.Status == CommandStatus.Started).ToList();
}
public void SetMessage(CommandModel command, string message)
{
command.Message = message;
_commandCache.Set(command.Id.ToString(), command);
}
public void Start(CommandModel command)
{
command.StartedAt = DateTime.UtcNow;
command.Status = CommandStatus.Started;
// Marks the command as started in the DB, the queue takes care of marking it as started on it's own
_logger.Trace("Marking command as started: {0}", command.Name);
_commandCache.Set(command.Id.ToString(), command);
_repo.Start(command);
}
@ -189,16 +194,23 @@ public void Requeue()
}
}
public void Cancel(int id)
{
if (!_commandQueue.RemoveIfQueued(id))
{
throw new NzbDroneClientException(HttpStatusCode.Conflict, "Unable to cancel task");
}
}
public void CleanCommands()
{
_logger.Trace("Cleaning up old commands");
var old = _commandCache.Values.Where(c => c.EndedAt < DateTime.UtcNow.AddMinutes(-5));
var commands = _commandQueue.All()
.Where(c => c.EndedAt < DateTime.UtcNow.AddMinutes(-5))
.ToList();
foreach (var command in old)
{
_commandCache.Remove(command.Id.ToString());
}
_commandQueue.RemoveMany(commands);
_repo.Trim();
}
@ -213,18 +225,6 @@ private dynamic GetCommand(string commandName)
return Json.Deserialize("{}", commandType);
}
private CommandModel FindCommand(CommandModel command)
{
var cachedCommand = _commandCache.Find(command.Id.ToString());
if (cachedCommand != null)
{
command.Message = cachedCommand.Message;
}
return command;
}
private void Update(CommandModel command, CommandStatus status, string message)
{
SetMessage(command, message);
@ -234,15 +234,14 @@ private void Update(CommandModel command, CommandStatus status, string message)
command.Status = status;
_logger.Trace("Updating command status");
_commandCache.Set(command.Id.ToString(), command);
_repo.End(command);
}
private List<CommandModel> QueuedOrStarted(string name)
{
return _commandCache.Values.Where(q => q.Name == name &&
(q.Status == CommandStatus.Queued ||
q.Status == CommandStatus.Started)).ToList();
return _commandQueue.QueuedOrStarted()
.Where(q => q.Name == name)
.ToList();
}
public void Handle(ApplicationStartedEvent message)

2
src/NzbDrone.Core/Messaging/Commands/CommandRepository.cs
View File

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Data.SQLite;
using NzbDrone.Core.Datastore;

1
src/NzbDrone.Core/Tv/Commands/BulkMoveSeriesCommand.cs
View File

@ -10,6 +10,7 @@ public class BulkMoveSeriesCommand : Command
public string DestinationRootFolder { get; set; }
public override bool SendUpdatesToClient => true;
public override bool RequiresDiskAccess => true;
}
public class BulkMoveSeries : IEquatable<BulkMoveSeries>

1
src/NzbDrone.Core/Tv/Commands/MoveSeriesCommand.cs
View File

@ -9,5 +9,6 @@ public class MoveSeriesCommand : Command
public string DestinationPath { get; set; }
public override bool SendUpdatesToClient => true;
public override bool RequiresDiskAccess => true;
}
}

3
src/NzbDrone.Core/Update/Commands/ApplicationUpdateCommand.cs
View File

@ -1,10 +1,11 @@
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.Update.Commands
{
public class ApplicationUpdateCommand : Command
{
public override bool SendUpdatesToClient => true;
public override bool IsExclusive => true;
public override string CompletionMessage => null;
}

45
src/Sonarr.Api.V3/Commands/CommandModule.cs
View File

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using NzbDrone.Common;
using NzbDrone.Common.TPL;
using NzbDrone.Core.Datastore.Events;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
@ -17,6 +18,8 @@ public class CommandModule : SonarrRestModuleWithSignalR<CommandResource, Comman
{
private readonly IManageCommandQueue _commandQueueManager;
private readonly IServiceFactory _serviceFactory;
private readonly Debouncer _debouncer;
private readonly Dictionary<int, CommandResource> _pendingUpdates;
public CommandModule(IManageCommandQueue commandQueueManager,
IBroadcastSignalRMessage signalRBroadcaster,
@ -26,9 +29,13 @@ public CommandModule(IManageCommandQueue commandQueueManager,
_commandQueueManager = commandQueueManager;
_serviceFactory = serviceFactory;
_debouncer = new Debouncer(SendUpdates, TimeSpan.FromSeconds(0.1));
_pendingUpdates = new Dictionary<int, CommandResource>();
GetResourceById = GetCommand;
CreateResource = StartCommand;
GetResourceAll = GetStartedCommands;
DeleteResource = CancelCommand;
PostValidator.RuleFor(c => c.Name).NotBlank();
}
@ -56,15 +63,45 @@ private int StartCommand(CommandResource commandResource)
private List<CommandResource> GetStartedCommands()
{
return _commandQueueManager.GetStarted().ToResource();
return _commandQueueManager.All().ToResource();
}
private void CancelCommand(int id)
{
_commandQueueManager.Cancel(id);
}
public void Handle(CommandUpdatedEvent message)
{
if (message.Command.Body.SendUpdatesToClient)
{
BroadcastResourceChange(ModelAction.Updated, message.Command.ToResource());
lock (_pendingUpdates)
{
_pendingUpdates[message.Command.Id] = message.Command.ToResource();
}
_debouncer.Execute();
}
}
private void SendUpdates()
{
lock (_pendingUpdates)
{
var pendingUpdates = _pendingUpdates.Values.ToArray();
_pendingUpdates.Clear();
foreach (var pendingUpdate in pendingUpdates)
{
BroadcastResourceChange(ModelAction.Updated, pendingUpdate);
if (pendingUpdate.Name == typeof(MessagingCleanupCommand).Name.Replace("Command", "") &&
pendingUpdate.Status == CommandStatus.Completed)
{
BroadcastResourceChange(ModelAction.Sync);
}
}
}
}
}
}
}

36
src/Sonarr.Api.V3/Commands/CommandResource.cs
View File

@ -1,7 +1,8 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using Newtonsoft.Json;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Messaging.Commands;
using Sonarr.Http.REST;
@ -10,6 +11,7 @@ namespace Sonarr.Api.V3.Commands
public class CommandResource : RestResource
{
public string Name { get; set; }
public string CommandName { get; set; }
public string Message { get; set; }
public Command Body { get; set; }
public CommandPriority Priority { get; set; }
@ -24,37 +26,6 @@ public class CommandResource : RestResource
[JsonIgnore]
public string CompletionMessage { get; set; }
//Legacy
public CommandStatus State
{
get
{
return Status;
}
set { }
}
public bool Manual
{
get
{
return Trigger == CommandTrigger.Manual;
}
set { }
}
public DateTime StartedOn
{
get
{
return Queued;
}
set { }
}
public DateTime? StateChangeTime
{
get
@ -106,6 +77,7 @@ public static CommandResource ToResource(this CommandModel model)
Id = model.Id,
Name = model.Name,
CommandName = model.Name.SplitCamelCase(),
Message = model.Message,
Body = model.Body,
Priority = model.Priority,

7
src/Sonarr.Api.V3/System/Tasks/TaskModule.cs
View File

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Datastore.Events;
using NzbDrone.Core.Jobs;
using NzbDrone.Core.Messaging.Events;
@ -13,8 +14,6 @@ public class TaskModule : SonarrRestModuleWithSignalR<TaskResource, ScheduledTas
{
private readonly ITaskManager _taskManager;
private static readonly Regex NameRegex = new Regex("(?<!^)[A-Z]", RegexOptions.Compiled);
public TaskModule(ITaskManager taskManager, IBroadcastSignalRMessage broadcastSignalRMessage)
: base(broadcastSignalRMessage, "system/task")
{
@ -51,7 +50,7 @@ private static TaskResource ConvertToResource(ScheduledTask scheduledTask)
return new TaskResource
{
Id = scheduledTask.Id,
Name = NameRegex.Replace(taskName, match => " " + match.Value),
Name = taskName.SplitCamelCase(),
TaskName = taskName,
Interval = scheduledTask.Interval,
LastExecution = scheduledTask.LastExecution,