Backup_Repos/Sonarr
1
0
Fork 0
mirror of https://github.com/Sonarr/Sonarr synced 2025-12-06 08:28:37 +01:00
Code Issues Projects Releases Wiki Activity

Fixed: Loading queue or processing releases slow with many Pending Releases

Browse source
This commit is contained in:
Mark McDowall 2025-10-25 16:34:17 -07:00
parent 550cf8d399
commit 064cbdbfb1
No known key found for this signature in database
8 changed files with 131 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs
View file

@ -8,6 +8,7 @@
using NzbDrone.Core.Datastore;
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities;
@ -54,8 +55,10 @@ public void Setup()
_release = Builder<ReleaseInfo>.CreateNew().Build();
_parsedEpisodeInfo = Builder<ParsedEpisodeInfo>.CreateNew().Build();
_parsedEpisodeInfo.Quality = new QualityModel(Quality.HDTV720p);
_parsedEpisodeInfo = Builder<ParsedEpisodeInfo>.CreateNew()
.With(p => p.Quality = new QualityModel(Quality.HDTV720p))
.With(p => p.AirDate = null)
.Build();
_remoteEpisode = new RemoteEpisode();
_remoteEpisode.Episodes = new List<Episode> { _episode };
@ -87,6 +90,10 @@ public void Setup()
.Setup(s => s.GetEpisodes(It.IsAny<ParsedEpisodeInfo>(), _series, true, null))
.Returns(new List<Episode> { _episode });
Mocker.GetMock<IParsingService>()
.Setup(s => s.Map(It.IsAny<ParsedEpisodeInfo>(), _series))
.Returns(_remoteEpisode);
Mocker.GetMock<IPrioritizeDownloadDecision>()
.Setup(s => s.PrioritizeDecisions(It.IsAny<List<DownloadDecision>>()))
.Returns((List<DownloadDecision> d) => d);
@ -110,9 +117,15 @@ private void GivenHeldRelease(string title, string indexer, DateTime publishDate
_heldReleases.AddRange(heldReleases);
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_add()
{
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();
@ -123,6 +136,7 @@ public void should_not_add_if_it_is_the_same_release_from_the_same_indexer()
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyNoInsert();
@ -134,6 +148,7 @@ public void should_not_add_if_it_is_the_same_release_from_the_same_indexer_twice
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable);
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyNoInsert();
@ -145,6 +160,7 @@ public void should_remove_duplicate_if_it_is_the_same_release_from_the_same_inde
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable);
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Fallback);
Mocker.GetMock<IPendingReleaseRepository>()
@ -156,6 +172,7 @@ public void should_add_if_title_is_different()
{
GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();
@ -166,6 +183,7 @@ public void should_add_if_indexer_is_different()
{
GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate);
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();
@ -176,6 +194,7 @@ public void should_add_if_publish_date_is_different()
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1));
InitializeReleases();
Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay);
VerifyInsert();

9
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs
View file

@ -7,6 +7,7 @@
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities;
@ -112,11 +113,17 @@ private void GivenHeldRelease(QualityModel quality)
_heldReleases.AddRange(heldReleases);
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_delete_if_the_grabbed_quality_is_the_same()
{
GivenHeldRelease(_parsedEpisodeInfo.Quality);
InitializeReleases();
Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode));
VerifyDelete();
@ -127,6 +134,7 @@ public void should_delete_if_the_grabbed_quality_is_the_higher()
{
GivenHeldRelease(new QualityModel(Quality.SDTV));
InitializeReleases();
Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode));
VerifyDelete();
@ -137,6 +145,7 @@ public void should_not_delete_if_the_grabbed_quality_is_the_lower()
{
GivenHeldRelease(new QualityModel(Quality.Bluray720p));
InitializeReleases();
Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode));
VerifyNoDelete();

12
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs
View file

@ -5,6 +5,7 @@
using NUnit.Framework;
using NzbDrone.Common.Crypto;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
@ -62,6 +63,11 @@ private void AddPending(int id, int seasonNumber, int[] episodes)
});
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_remove_same_release()
{
@ -69,6 +75,7 @@ public void should_remove_same_release()
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1);
@ -84,6 +91,7 @@ public void should_remove_multiple_releases_release()
var queueId = HashConverter.GetHashInt31($"pending-{3}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(3, 4);
@ -99,6 +107,7 @@ public void should_not_remove_different_season()
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1, 2);
@ -114,6 +123,7 @@ public void should_not_remove_different_episodes()
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1, 2);
@ -127,6 +137,7 @@ public void should_not_remove_multiepisodes()
var queueId = HashConverter.GetHashInt31($"pending-{1}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(1);
@ -140,6 +151,7 @@ public void should_not_remove_singleepisodes()
var queueId = HashConverter.GetHashInt31($"pending-{2}");
InitializeReleases();
Subject.RemovePendingQueueItems(queueId);
AssertRemoved(2);

12
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingObsoleteFixture.cs
View file

@ -5,6 +5,7 @@
using NUnit.Framework;
using NzbDrone.Common.Crypto;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
@ -62,6 +63,11 @@ private void AddPending(int id, int seasonNumber, int[] episodes)
});
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_remove_same_release()
{
@ -69,6 +75,7 @@ public void should_remove_same_release()
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1);
@ -84,6 +91,7 @@ public void should_remove_multiple_releases_release()
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 3, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(3, 4);
@ -99,6 +107,7 @@ public void should_not_remove_different_season()
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1, 2);
@ -114,6 +123,7 @@ public void should_not_remove_different_episodes()
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1, 2);
@ -127,6 +137,7 @@ public void should_not_remove_multiepisodes()
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(1);
@ -140,6 +151,7 @@ public void should_not_remove_singleepisodes()
var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 2, _episode.Id));
InitializeReleases();
Subject.RemovePendingQueueItemsObsolete(queueId);
AssertRemoved(2);

10
src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs
View file

@ -9,6 +9,7 @@
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.Pending;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Qualities;
@ -109,11 +110,17 @@ private void GivenHeldRelease(string title, string indexer, DateTime publishDate
.Returns(heldReleases);
}
private void InitializeReleases()
{
Subject.Handle(new ApplicationStartedEvent());
}
[Test]
public void should_remove_if_it_is_the_same_release_from_the_same_indexer()
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));
@ -126,6 +133,7 @@ public void should_not_remove_if_title_is_different()
{
GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate);
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));
@ -138,6 +146,7 @@ public void should_not_remove_if_indexer_is_different()
{
GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate);
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));
@ -150,6 +159,7 @@ public void should_not_remove_if_publish_date_is_different()
{
GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1));
InitializeReleases();
Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List<DownloadDecision>(),
new List<DownloadDecision>(),
new List<DownloadDecision> { _temporarilyRejected })));

66
src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs
View file

@ -5,15 +5,18 @@
using NzbDrone.Common.Crypto;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Configuration.Events;
using NzbDrone.Core.CustomFormats;
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download.Aggregation;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Jobs;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Profiles.Delay;
using NzbDrone.Core.Profiles.Qualities;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.Queue;
using NzbDrone.Core.Tv;
@ -37,9 +40,14 @@ public interface IPendingReleaseService
}
public class PendingReleaseService : IPendingReleaseService,
IHandle<SeriesEditedEvent>,
IHandle<SeriesUpdatedEvent>,
IHandle<SeriesDeletedEvent>,
IHandle<EpisodeGrabbedEvent>,
IHandle<RssSyncCompleteEvent>
IHandle<RssSyncCompleteEvent>,
IHandle<QualityProfileUpdatedEvent>,
IHandle<ConfigSavedEvent>,
IHandle<ApplicationStartedEvent>
{
private readonly IIndexerStatusService _indexerStatusService;
private readonly IPendingReleaseRepository _repository;
@ -55,6 +63,8 @@ public class PendingReleaseService : IPendingReleaseService,
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
private static List<PendingRelease> _pendingReleases = new();
public PendingReleaseService(IIndexerStatusService indexerStatusService,
IPendingReleaseRepository repository,
ISeriesService seriesService,
@ -91,11 +101,14 @@ public void Add(DownloadDecision decision, PendingReleaseReason reason)
public void AddMany(List<Tuple<DownloadDecision, PendingReleaseReason>> decisions)
{
var pending = _pendingReleases;
foreach (var seriesDecisions in decisions.GroupBy(v => v.Item1.RemoteEpisode.Series.Id))
{
var series = seriesDecisions.First().Item1.RemoteEpisode.Series;
var alreadyPending = _repository.AllBySeriesId(series.Id);
var alreadyPending = _pendingReleases.Where(p => p.SeriesId == series.Id).SelectList(s => s.JsonClone());
// TODO: Do we need IncludeRemoteEpisodes?
alreadyPending = IncludeRemoteEpisodes(alreadyPending, seriesDecisions.ToDictionaryIgnoreDuplicates(v => v.Item1.RemoteEpisode.Release.Title, v => v.Item1.RemoteEpisode));
var alreadyPendingByEpisode = CreateEpisodeLookup(alreadyPending);
@ -152,6 +165,8 @@ public void AddMany(List<Tuple<DownloadDecision, PendingReleaseReason>> decision
Insert(decision, reason);
}
}
UpdatePendingReleases();
}
public List<ReleaseInfo> GetPending()
@ -175,16 +190,14 @@ public List<ReleaseInfo> GetPending()
public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
{
return IncludeRemoteEpisodes(_repository.AllBySeriesId(seriesId)).Select(v => v.RemoteEpisode).ToList();
return _pendingReleases.Where(p => p.SeriesId == seriesId).Select(p => p.RemoteEpisode).ToList();
}
public List<Queue.Queue> GetPendingQueue()
{
var queued = new List<Queue.Queue>();
var nextRssSync = new Lazy<DateTime>(() => _taskManager.GetNextExecution(typeof(RssSyncCommand)));
var pendingReleases = IncludeRemoteEpisodes(_repository.WithoutFallback());
var pendingReleases = _pendingReleases.Where(p => p.Reason != PendingReleaseReason.Fallback).ToList();
foreach (var pendingRelease in pendingReleases)
{
@ -218,10 +231,8 @@ public List<RemoteEpisode> GetPendingRemoteEpisodes(int seriesId)
public List<Queue.Queue> GetPendingQueueObsolete()
{
var queued = new List<Queue.Queue>();
var nextRssSync = new Lazy<DateTime>(() => _taskManager.GetNextExecution(typeof(RssSyncCommand)));
var pendingReleases = IncludeRemoteEpisodes(_repository.WithoutFallback());
var pendingReleases = _pendingReleases.Where(p => p.Reason != PendingReleaseReason.Fallback).ToList();
foreach (var pendingRelease in pendingReleases)
{
@ -317,12 +328,12 @@ private List<ReleaseInfo> FilterBlockedIndexers(List<ReleaseInfo> releases)
private List<PendingRelease> GetPendingReleases()
{
return IncludeRemoteEpisodes(_repository.All().ToList());
return _pendingReleases;
}
private List<PendingRelease> GetPendingReleases(int seriesId)
{
return IncludeRemoteEpisodes(_repository.AllBySeriesId(seriesId).ToList());
return _pendingReleases.Where(p => p.SeriesId == seriesId).ToList();
}
private List<PendingRelease> IncludeRemoteEpisodes(List<PendingRelease> releases, Dictionary<string, RemoteEpisode> knownRemoteEpisodes = null)
@ -632,19 +643,52 @@ private int PrioritizeDownloadProtocol(Series series, DownloadProtocol downloadP
return 1;
}
private void UpdatePendingReleases()
{
_pendingReleases = IncludeRemoteEpisodes(_repository.All().ToList());
}
public void Handle(SeriesEditedEvent message)
{
UpdatePendingReleases();
}
public void Handle(SeriesUpdatedEvent message)
{
UpdatePendingReleases();
}
public void Handle(SeriesDeletedEvent message)
{
_repository.DeleteBySeriesIds(message.Series.Select(m => m.Id).ToList());
UpdatePendingReleases();
}
public void Handle(EpisodeGrabbedEvent message)
{
RemoveGrabbed(message.Episode);
UpdatePendingReleases();
}
public void Handle(RssSyncCompleteEvent message)
{
RemoveRejected(message.ProcessedDecisions.Rejected);
UpdatePendingReleases();
}
public void Handle(QualityProfileUpdatedEvent message)
{
UpdatePendingReleases();
}
public void Handle(ApplicationStartedEvent message)
{
UpdatePendingReleases();
}
public void Handle(ConfigSavedEvent message)
{
UpdatePendingReleases();
}
private static Func<PendingRelease, bool> MatchingReleasePredicate(ReleaseInfo release)

4
src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs
View file

@ -33,18 +33,21 @@ public class QualityProfileService : IQualityProfileService,
private readonly IImportListFactory _importListFactory;
private readonly ICustomFormatService _formatService;
private readonly ISeriesService _seriesService;
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
public QualityProfileService(IQualityProfileRepository qualityProfileRepository,
IImportListFactory importListFactory,
ICustomFormatService formatService,
ISeriesService seriesService,
IEventAggregator eventAggregator,
Logger logger)
{
_qualityProfileRepository = qualityProfileRepository;
_importListFactory = importListFactory;
_formatService = formatService;
_seriesService = seriesService;
_eventAggregator = eventAggregator;
_logger = logger;
}
@ -56,6 +59,7 @@ public QualityProfile Add(QualityProfile profile)
public void Update(QualityProfile profile)
{
_qualityProfileRepository.Update(profile);
_eventAggregator.PublishEvent(new QualityProfileUpdatedEvent(profile.Id));
}
public void Delete(int id)

8
src/NzbDrone.Core/Profiles/Qualities/QualityProfileUpdatedEvent.cs Normal file
View file

@ -0,0 +1,8 @@
using NzbDrone.Common.Messaging;
namespace NzbDrone.Core.Profiles.Qualities;
public class QualityProfileUpdatedEvent(int id) : IEvent
{
public int Id { get; private set; } = id;
}