From 064cbdbfb180f970b52eda202b82ed885594d931 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Sat, 25 Oct 2025 16:34:17 -0700 Subject: [PATCH] Fixed: Loading queue or processing releases slow with many Pending Releases --- .../PendingReleaseServiceTests/AddFixture.cs | 23 ++++++- .../RemoveGrabbedFixture.cs | 9 +++ .../RemovePendingFixture.cs | 12 ++++ .../RemovePendingObsoleteFixture.cs | 12 ++++ .../RemoveRejectedFixture.cs | 10 +++ .../Download/Pending/PendingReleaseService.cs | 66 +++++++++++++++---- .../Qualities/QualityProfileService.cs | 4 ++ .../Qualities/QualityProfileUpdatedEvent.cs | 8 +++ 8 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 src/NzbDrone.Core/Profiles/Qualities/QualityProfileUpdatedEvent.cs diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs index 8adcaf3cd..e726a4633 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs @@ -8,6 +8,7 @@ using NzbDrone.Core.Datastore; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Qualities; @@ -54,8 +55,10 @@ public void Setup() _release = Builder.CreateNew().Build(); - _parsedEpisodeInfo = Builder.CreateNew().Build(); - _parsedEpisodeInfo.Quality = new QualityModel(Quality.HDTV720p); + _parsedEpisodeInfo = Builder.CreateNew() + .With(p => p.Quality = new QualityModel(Quality.HDTV720p)) + .With(p => p.AirDate = null) + .Build(); _remoteEpisode = new RemoteEpisode(); _remoteEpisode.Episodes = new List { _episode }; @@ -87,6 +90,10 @@ public void Setup() .Setup(s => s.GetEpisodes(It.IsAny(), _series, true, null)) .Returns(new List { _episode }); + Mocker.GetMock() + .Setup(s => s.Map(It.IsAny(), _series)) + .Returns(_remoteEpisode); + Mocker.GetMock() .Setup(s => s.PrioritizeDecisions(It.IsAny>())) .Returns((List d) => d); @@ -110,9 +117,15 @@ private void GivenHeldRelease(string title, string indexer, DateTime publishDate _heldReleases.AddRange(heldReleases); } + private void InitializeReleases() + { + Subject.Handle(new ApplicationStartedEvent()); + } + [Test] public void should_add() { + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); @@ -123,6 +136,7 @@ public void should_not_add_if_it_is_the_same_release_from_the_same_indexer() { GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate); + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyNoInsert(); @@ -134,6 +148,7 @@ public void should_not_add_if_it_is_the_same_release_from_the_same_indexer_twice GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable); GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback); + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyNoInsert(); @@ -145,6 +160,7 @@ public void should_remove_duplicate_if_it_is_the_same_release_from_the_same_inde GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.DownloadClientUnavailable); GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate, PendingReleaseReason.Fallback); + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Fallback); Mocker.GetMock() @@ -156,6 +172,7 @@ public void should_add_if_title_is_different() { GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate); + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); @@ -166,6 +183,7 @@ public void should_add_if_indexer_is_different() { GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate); + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); @@ -176,6 +194,7 @@ public void should_add_if_publish_date_is_different() { GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1)); + InitializeReleases(); Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs index d7e806eb0..58db60016 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveGrabbedFixture.cs @@ -7,6 +7,7 @@ using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Qualities; @@ -112,11 +113,17 @@ private void GivenHeldRelease(QualityModel quality) _heldReleases.AddRange(heldReleases); } + private void InitializeReleases() + { + Subject.Handle(new ApplicationStartedEvent()); + } + [Test] public void should_delete_if_the_grabbed_quality_is_the_same() { GivenHeldRelease(_parsedEpisodeInfo.Quality); + InitializeReleases(); Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode)); VerifyDelete(); @@ -127,6 +134,7 @@ public void should_delete_if_the_grabbed_quality_is_the_higher() { GivenHeldRelease(new QualityModel(Quality.SDTV)); + InitializeReleases(); Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode)); VerifyDelete(); @@ -137,6 +145,7 @@ public void should_not_delete_if_the_grabbed_quality_is_the_lower() { GivenHeldRelease(new QualityModel(Quality.Bluray720p)); + InitializeReleases(); Subject.Handle(new EpisodeGrabbedEvent(_remoteEpisode)); VerifyNoDelete(); diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs index 37cd090a4..7ec041221 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingFixture.cs @@ -5,6 +5,7 @@ using NUnit.Framework; using NzbDrone.Common.Crypto; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; @@ -62,6 +63,11 @@ private void AddPending(int id, int seasonNumber, int[] episodes) }); } + private void InitializeReleases() + { + Subject.Handle(new ApplicationStartedEvent()); + } + [Test] public void should_remove_same_release() { @@ -69,6 +75,7 @@ public void should_remove_same_release() var queueId = HashConverter.GetHashInt31($"pending-{1}"); + InitializeReleases(); Subject.RemovePendingQueueItems(queueId); AssertRemoved(1); @@ -84,6 +91,7 @@ public void should_remove_multiple_releases_release() var queueId = HashConverter.GetHashInt31($"pending-{3}"); + InitializeReleases(); Subject.RemovePendingQueueItems(queueId); AssertRemoved(3, 4); @@ -99,6 +107,7 @@ public void should_not_remove_different_season() var queueId = HashConverter.GetHashInt31($"pending-{1}"); + InitializeReleases(); Subject.RemovePendingQueueItems(queueId); AssertRemoved(1, 2); @@ -114,6 +123,7 @@ public void should_not_remove_different_episodes() var queueId = HashConverter.GetHashInt31($"pending-{1}"); + InitializeReleases(); Subject.RemovePendingQueueItems(queueId); AssertRemoved(1, 2); @@ -127,6 +137,7 @@ public void should_not_remove_multiepisodes() var queueId = HashConverter.GetHashInt31($"pending-{1}"); + InitializeReleases(); Subject.RemovePendingQueueItems(queueId); AssertRemoved(1); @@ -140,6 +151,7 @@ public void should_not_remove_singleepisodes() var queueId = HashConverter.GetHashInt31($"pending-{2}"); + InitializeReleases(); Subject.RemovePendingQueueItems(queueId); AssertRemoved(2); diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingObsoleteFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingObsoleteFixture.cs index dc2eb9f4d..06e8d36c1 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingObsoleteFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemovePendingObsoleteFixture.cs @@ -5,6 +5,7 @@ using NUnit.Framework; using NzbDrone.Common.Crypto; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; @@ -62,6 +63,11 @@ private void AddPending(int id, int seasonNumber, int[] episodes) }); } + private void InitializeReleases() + { + Subject.Handle(new ApplicationStartedEvent()); + } + [Test] public void should_remove_same_release() { @@ -69,6 +75,7 @@ public void should_remove_same_release() var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id)); + InitializeReleases(); Subject.RemovePendingQueueItemsObsolete(queueId); AssertRemoved(1); @@ -84,6 +91,7 @@ public void should_remove_multiple_releases_release() var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 3, _episode.Id)); + InitializeReleases(); Subject.RemovePendingQueueItemsObsolete(queueId); AssertRemoved(3, 4); @@ -99,6 +107,7 @@ public void should_not_remove_different_season() var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id)); + InitializeReleases(); Subject.RemovePendingQueueItemsObsolete(queueId); AssertRemoved(1, 2); @@ -114,6 +123,7 @@ public void should_not_remove_different_episodes() var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id)); + InitializeReleases(); Subject.RemovePendingQueueItemsObsolete(queueId); AssertRemoved(1, 2); @@ -127,6 +137,7 @@ public void should_not_remove_multiepisodes() var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 1, _episode.Id)); + InitializeReleases(); Subject.RemovePendingQueueItemsObsolete(queueId); AssertRemoved(1); @@ -140,6 +151,7 @@ public void should_not_remove_singleepisodes() var queueId = HashConverter.GetHashInt31(string.Format("pending-{0}-ep{1}", 2, _episode.Id)); + InitializeReleases(); Subject.RemovePendingQueueItemsObsolete(queueId); AssertRemoved(2); diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs index 6a914208a..243d9cf0e 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/RemoveRejectedFixture.cs @@ -9,6 +9,7 @@ using NzbDrone.Core.Download; using NzbDrone.Core.Download.Pending; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Qualities; @@ -109,11 +110,17 @@ private void GivenHeldRelease(string title, string indexer, DateTime publishDate .Returns(heldReleases); } + private void InitializeReleases() + { + Subject.Handle(new ApplicationStartedEvent()); + } + [Test] public void should_remove_if_it_is_the_same_release_from_the_same_indexer() { GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate); + InitializeReleases(); Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List(), new List(), new List { _temporarilyRejected }))); @@ -126,6 +133,7 @@ public void should_not_remove_if_title_is_different() { GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate); + InitializeReleases(); Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List(), new List(), new List { _temporarilyRejected }))); @@ -138,6 +146,7 @@ public void should_not_remove_if_indexer_is_different() { GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate); + InitializeReleases(); Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List(), new List(), new List { _temporarilyRejected }))); @@ -150,6 +159,7 @@ public void should_not_remove_if_publish_date_is_different() { GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1)); + InitializeReleases(); Subject.Handle(new RssSyncCompleteEvent(new ProcessedDecisions(new List(), new List(), new List { _temporarilyRejected }))); diff --git a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs index ed12ab3cc..7d22647f7 100644 --- a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs +++ b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs @@ -5,15 +5,18 @@ using NzbDrone.Common.Crypto; using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Configuration.Events; using NzbDrone.Core.CustomFormats; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download.Aggregation; using NzbDrone.Core.Indexers; using NzbDrone.Core.Jobs; +using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Delay; +using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.Qualities; using NzbDrone.Core.Queue; using NzbDrone.Core.Tv; @@ -37,9 +40,14 @@ public interface IPendingReleaseService } public class PendingReleaseService : IPendingReleaseService, + IHandle, + IHandle, IHandle, IHandle, - IHandle + IHandle, + IHandle, + IHandle, + IHandle { private readonly IIndexerStatusService _indexerStatusService; private readonly IPendingReleaseRepository _repository; @@ -55,6 +63,8 @@ public class PendingReleaseService : IPendingReleaseService, private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; + private static List _pendingReleases = new(); + public PendingReleaseService(IIndexerStatusService indexerStatusService, IPendingReleaseRepository repository, ISeriesService seriesService, @@ -91,11 +101,14 @@ public void Add(DownloadDecision decision, PendingReleaseReason reason) public void AddMany(List> decisions) { + var pending = _pendingReleases; + foreach (var seriesDecisions in decisions.GroupBy(v => v.Item1.RemoteEpisode.Series.Id)) { var series = seriesDecisions.First().Item1.RemoteEpisode.Series; - var alreadyPending = _repository.AllBySeriesId(series.Id); + var alreadyPending = _pendingReleases.Where(p => p.SeriesId == series.Id).SelectList(s => s.JsonClone()); + // TODO: Do we need IncludeRemoteEpisodes? alreadyPending = IncludeRemoteEpisodes(alreadyPending, seriesDecisions.ToDictionaryIgnoreDuplicates(v => v.Item1.RemoteEpisode.Release.Title, v => v.Item1.RemoteEpisode)); var alreadyPendingByEpisode = CreateEpisodeLookup(alreadyPending); @@ -152,6 +165,8 @@ public void AddMany(List> decision Insert(decision, reason); } } + + UpdatePendingReleases(); } public List GetPending() @@ -175,16 +190,14 @@ public List GetPending() public List GetPendingRemoteEpisodes(int seriesId) { - return IncludeRemoteEpisodes(_repository.AllBySeriesId(seriesId)).Select(v => v.RemoteEpisode).ToList(); + return _pendingReleases.Where(p => p.SeriesId == seriesId).Select(p => p.RemoteEpisode).ToList(); } public List GetPendingQueue() { var queued = new List(); - var nextRssSync = new Lazy(() => _taskManager.GetNextExecution(typeof(RssSyncCommand))); - - var pendingReleases = IncludeRemoteEpisodes(_repository.WithoutFallback()); + var pendingReleases = _pendingReleases.Where(p => p.Reason != PendingReleaseReason.Fallback).ToList(); foreach (var pendingRelease in pendingReleases) { @@ -218,10 +231,8 @@ public List GetPendingRemoteEpisodes(int seriesId) public List GetPendingQueueObsolete() { var queued = new List(); - var nextRssSync = new Lazy(() => _taskManager.GetNextExecution(typeof(RssSyncCommand))); - - var pendingReleases = IncludeRemoteEpisodes(_repository.WithoutFallback()); + var pendingReleases = _pendingReleases.Where(p => p.Reason != PendingReleaseReason.Fallback).ToList(); foreach (var pendingRelease in pendingReleases) { @@ -317,12 +328,12 @@ private List FilterBlockedIndexers(List releases) private List GetPendingReleases() { - return IncludeRemoteEpisodes(_repository.All().ToList()); + return _pendingReleases; } private List GetPendingReleases(int seriesId) { - return IncludeRemoteEpisodes(_repository.AllBySeriesId(seriesId).ToList()); + return _pendingReleases.Where(p => p.SeriesId == seriesId).ToList(); } private List IncludeRemoteEpisodes(List releases, Dictionary knownRemoteEpisodes = null) @@ -632,19 +643,52 @@ private int PrioritizeDownloadProtocol(Series series, DownloadProtocol downloadP return 1; } + private void UpdatePendingReleases() + { + _pendingReleases = IncludeRemoteEpisodes(_repository.All().ToList()); + } + + public void Handle(SeriesEditedEvent message) + { + UpdatePendingReleases(); + } + + public void Handle(SeriesUpdatedEvent message) + { + UpdatePendingReleases(); + } + public void Handle(SeriesDeletedEvent message) { _repository.DeleteBySeriesIds(message.Series.Select(m => m.Id).ToList()); + UpdatePendingReleases(); } public void Handle(EpisodeGrabbedEvent message) { RemoveGrabbed(message.Episode); + UpdatePendingReleases(); } public void Handle(RssSyncCompleteEvent message) { RemoveRejected(message.ProcessedDecisions.Rejected); + UpdatePendingReleases(); + } + + public void Handle(QualityProfileUpdatedEvent message) + { + UpdatePendingReleases(); + } + + public void Handle(ApplicationStartedEvent message) + { + UpdatePendingReleases(); + } + + public void Handle(ConfigSavedEvent message) + { + UpdatePendingReleases(); } private static Func MatchingReleasePredicate(ReleaseInfo release) diff --git a/src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs b/src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs index 0df442393..9fdc32680 100644 --- a/src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs +++ b/src/NzbDrone.Core/Profiles/Qualities/QualityProfileService.cs @@ -33,18 +33,21 @@ public class QualityProfileService : IQualityProfileService, private readonly IImportListFactory _importListFactory; private readonly ICustomFormatService _formatService; private readonly ISeriesService _seriesService; + private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; public QualityProfileService(IQualityProfileRepository qualityProfileRepository, IImportListFactory importListFactory, ICustomFormatService formatService, ISeriesService seriesService, + IEventAggregator eventAggregator, Logger logger) { _qualityProfileRepository = qualityProfileRepository; _importListFactory = importListFactory; _formatService = formatService; _seriesService = seriesService; + _eventAggregator = eventAggregator; _logger = logger; } @@ -56,6 +59,7 @@ public QualityProfile Add(QualityProfile profile) public void Update(QualityProfile profile) { _qualityProfileRepository.Update(profile); + _eventAggregator.PublishEvent(new QualityProfileUpdatedEvent(profile.Id)); } public void Delete(int id) diff --git a/src/NzbDrone.Core/Profiles/Qualities/QualityProfileUpdatedEvent.cs b/src/NzbDrone.Core/Profiles/Qualities/QualityProfileUpdatedEvent.cs new file mode 100644 index 000000000..c02fc1ea3 --- /dev/null +++ b/src/NzbDrone.Core/Profiles/Qualities/QualityProfileUpdatedEvent.cs @@ -0,0 +1,8 @@ +using NzbDrone.Common.Messaging; + +namespace NzbDrone.Core.Profiles.Qualities; + +public class QualityProfileUpdatedEvent(int id) : IEvent +{ + public int Id { get; private set; } = id; +}