From 1f2b62b6646eb1d62f3c8842ce75f10f222c2025 Mon Sep 17 00:00:00 2001 From: PL Date: Sun, 15 Feb 2026 09:22:59 -0500 Subject: [PATCH] feat: Implement indexer download and query rate limiting and expose it via a new Newznab API controller. --- .../IndexerLimitServiceFixture.cs | 149 ++++++++++++++++++ .../Indexers/IndexerBaseSettings.cs | 3 +- .../Indexers/IndexerLimitService.cs | 50 +++--- .../Indexers/NewznabController.cs | 8 +- 4 files changed, 186 insertions(+), 24 deletions(-) create mode 100644 src/NzbDrone.Core.Test/IndexerTests/IndexerLimitServiceFixture.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerLimitServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerLimitServiceFixture.cs new file mode 100644 index 000000000..387b72473 --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerLimitServiceFixture.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.History; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.IndexerTests +{ + public class IndexerLimitServiceFixture : CoreTest + { + private IndexerDefinition CreateIndexerWithLimitsUnit(IndexerLimitsUnit unit, int id = 1) + { + return new IndexerDefinition + { + Id = id, + Settings = new TestIndexerSettings + { + BaseSettings = new IndexerBaseSettings + { + LimitsUnit = (int)unit + } + } + }; + } + + [Test] + public void should_return_1440_for_day_unit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Day); + + Subject.CalculateIntervalLimitMinutes(indexer).Should().Be(1440); + } + + [Test] + public void should_return_60_for_hour_unit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Hour); + + Subject.CalculateIntervalLimitMinutes(indexer).Should().Be(60); + } + + [Test] + public void should_return_1_for_minute_unit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Minute); + + Subject.CalculateIntervalLimitMinutes(indexer).Should().Be(1); + } + + [Test] + public void should_return_1440_for_default_when_id_is_zero() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Hour, id: 0); + + Subject.CalculateIntervalLimitMinutes(indexer).Should().Be(1440); + } + + [Test] + public void should_format_day_interval() + { + IndexerLimitService.FormatIntervalLimit(1440).Should().Be("1 day"); + } + + [Test] + public void should_format_hour_interval() + { + IndexerLimitService.FormatIntervalLimit(60).Should().Be("1 hour"); + } + + [Test] + public void should_format_minute_interval() + { + IndexerLimitService.FormatIntervalLimit(1).Should().Be("1 minute"); + } + + [Test] + public void should_format_minutes_interval() + { + IndexerLimitService.FormatIntervalLimit(5).Should().Be("5 minute(s)"); + } + + [Test] + public void should_return_true_when_at_query_limit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Minute); + ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit = 10; + + Mocker.GetMock() + .Setup(s => s.CountSince(indexer.Id, It.IsAny(), It.Is>(l => l.Contains(HistoryEventType.IndexerQuery)))) + .Returns(10); + + Subject.AtQueryLimit(indexer).Should().BeTrue(); + } + + [Test] + public void should_return_false_when_under_query_limit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Minute); + ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit = 10; + + Mocker.GetMock() + .Setup(s => s.CountSince(indexer.Id, It.IsAny(), It.Is>(l => l.Contains(HistoryEventType.IndexerQuery)))) + .Returns(9); + + Subject.AtQueryLimit(indexer).Should().BeFalse(); + } + + [Test] + public void should_return_true_when_at_download_limit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Hour); + ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit = 5; + + Mocker.GetMock() + .Setup(s => s.CountSince(indexer.Id, It.IsAny(), It.Is>(l => l.Contains(HistoryEventType.ReleaseGrabbed)))) + .Returns(5); + + Subject.AtDownloadLimit(indexer).Should().BeTrue(); + } + + [Test] + public void should_return_false_when_under_download_limit() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Hour); + ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit = 5; + + Mocker.GetMock() + .Setup(s => s.CountSince(indexer.Id, It.IsAny(), It.Is>(l => l.Contains(HistoryEventType.ReleaseGrabbed)))) + .Returns(4); + + Subject.AtDownloadLimit(indexer).Should().BeFalse(); + } + + [Test] + public void should_use_correct_time_window_for_query_limit_minutes() + { + var indexer = CreateIndexerWithLimitsUnit(IndexerLimitsUnit.Minute); + ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit = 10; + + Subject.AtQueryLimit(indexer); + + Mocker.GetMock() + .Verify(v => v.CountSince(indexer.Id, It.Is(d => d > DateTime.Now.AddMinutes(-1).AddSeconds(-5) && d < DateTime.Now.AddMinutes(-1).AddSeconds(5)), It.IsAny>()), Times.Once); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerBaseSettings.cs b/src/NzbDrone.Core/Indexers/IndexerBaseSettings.cs index 5bada9b42..4d54d5f4d 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBaseSettings.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBaseSettings.cs @@ -34,6 +34,7 @@ public class IndexerBaseSettings public enum IndexerLimitsUnit { Day = 0, - Hour = 1 + Hour = 1, + Minute = 2 } } diff --git a/src/NzbDrone.Core/Indexers/IndexerLimitService.cs b/src/NzbDrone.Core/Indexers/IndexerLimitService.cs index 112255f17..401c6d17f 100644 --- a/src/NzbDrone.Core/Indexers/IndexerLimitService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerLimitService.cs @@ -11,7 +11,7 @@ public interface IIndexerLimitService bool AtQueryLimit(IndexerDefinition indexer); int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer); int CalculateRetryAfterQueryLimit(IndexerDefinition indexer); - int CalculateIntervalLimitHours(IndexerDefinition indexer); + int CalculateIntervalLimitMinutes(IndexerDefinition indexer); } public class IndexerLimitService : IIndexerLimitService @@ -30,18 +30,18 @@ public bool AtDownloadLimit(IndexerDefinition indexer) { if (indexer is { Id: > 0 } && ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.HasValue) { - var intervalLimitHours = CalculateIntervalLimitHours(indexer); - var grabCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-1 * intervalLimitHours), new List { HistoryEventType.ReleaseGrabbed }); + var intervalLimitMinutes = CalculateIntervalLimitMinutes(indexer); + var grabCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddMinutes(-1 * intervalLimitMinutes), new List { HistoryEventType.ReleaseGrabbed }); var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit; if (grabCount >= grabLimit) { - _logger.Info("Indexer {0} has performed {1} of possible {2} grabs in last {3} hour(s), exceeding the maximum grab limit", indexer.Name, grabCount, grabLimit, intervalLimitHours); + _logger.Info("Indexer {0} has performed {1} of possible {2} grabs in last {3}, exceeding the maximum grab limit", indexer.Name, grabCount, grabLimit, FormatIntervalLimit(intervalLimitMinutes)); return true; } - _logger.Debug("Indexer {0} has performed {1} of possible {2} grabs in last {3} hour(s), proceeding", indexer.Name, grabCount, grabLimit, intervalLimitHours); + _logger.Debug("Indexer {0} has performed {1} of possible {2} grabs in last {3}, proceeding", indexer.Name, grabCount, grabLimit, FormatIntervalLimit(intervalLimitMinutes)); } return false; @@ -51,18 +51,18 @@ public bool AtQueryLimit(IndexerDefinition indexer) { if (indexer is { Id: > 0 } && ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.HasValue) { - var intervalLimitHours = CalculateIntervalLimitHours(indexer); - var queryCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-1 * intervalLimitHours), new List { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }); + var intervalLimitMinutes = CalculateIntervalLimitMinutes(indexer); + var queryCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddMinutes(-1 * intervalLimitMinutes), new List { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }); var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit; if (queryCount >= queryLimit) { - _logger.Info("Indexer {0} has performed {1} of possible {2} queries in last {3} hour(s), exceeding the maximum query limit", indexer.Name, queryCount, queryLimit, intervalLimitHours); + _logger.Info("Indexer {0} has performed {1} of possible {2} queries in last {3}, exceeding the maximum query limit", indexer.Name, queryCount, queryLimit, FormatIntervalLimit(intervalLimitMinutes)); return true; } - _logger.Debug("Indexer {0} has performed {1} of possible {2} queries in last {3} hour(s), proceeding", indexer.Name, queryCount, queryLimit, intervalLimitHours); + _logger.Debug("Indexer {0} has performed {1} of possible {2} queries in last {3}, proceeding", indexer.Name, queryCount, queryLimit, FormatIntervalLimit(intervalLimitMinutes)); } return false; @@ -72,14 +72,14 @@ public int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer) { if (indexer is { Id: > 0 } && ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.HasValue) { - var intervalLimitHours = CalculateIntervalLimitHours(indexer); + var intervalLimitMinutes = CalculateIntervalLimitMinutes(indexer); var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.GetValueOrDefault(); - var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-1 * intervalLimitHours), new List { HistoryEventType.ReleaseGrabbed }, grabLimit); + var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddMinutes(-1 * intervalLimitMinutes), new List { HistoryEventType.ReleaseGrabbed }, grabLimit); if (firstHistorySince != null) { - return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(intervalLimitHours).Subtract(DateTime.Now).TotalSeconds); + return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddMinutes(intervalLimitMinutes).Subtract(DateTime.Now).TotalSeconds); } } @@ -90,33 +90,45 @@ public int CalculateRetryAfterQueryLimit(IndexerDefinition indexer) { if (indexer is { Id: > 0 } && ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.HasValue) { - var intervalLimitHours = CalculateIntervalLimitHours(indexer); + var intervalLimitMinutes = CalculateIntervalLimitMinutes(indexer); var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.GetValueOrDefault(); - var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-1 * intervalLimitHours), new List { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }, queryLimit); + var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddMinutes(-1 * intervalLimitMinutes), new List { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }, queryLimit); if (firstHistorySince != null) { - return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(intervalLimitHours).Subtract(DateTime.Now).TotalSeconds); + return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddMinutes(intervalLimitMinutes).Subtract(DateTime.Now).TotalSeconds); } } return 0; } - public int CalculateIntervalLimitHours(IndexerDefinition indexer) + public int CalculateIntervalLimitMinutes(IndexerDefinition indexer) { if (indexer is { Id: > 0 }) { return ((IIndexerSettings)indexer.Settings).BaseSettings.LimitsUnit switch { - (int)IndexerLimitsUnit.Hour => 1, - _ => 24 + (int)IndexerLimitsUnit.Minute => 1, + (int)IndexerLimitsUnit.Hour => 60, + _ => 1440 }; } // Fallback to limits per day - return 24; + return 1440; + } + + public static string FormatIntervalLimit(int minutes) + { + return minutes switch + { + 1440 => "1 day", + 60 => "1 hour", + 1 => "1 minute", + _ => $"{minutes} minute(s)" + }; } } } diff --git a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs index cde49b789..52ae49eab 100644 --- a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs +++ b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs @@ -162,9 +162,9 @@ public async Task GetNewznabResponse(int id, [FromQuery] NewznabR AddRetryAfterHeader(retryAfterQueryLimit); var queryLimit = ((IIndexerSettings)indexer.Definition.Settings).BaseSettings.QueryLimit; - var intervalLimitHours = _indexerLimitService.CalculateIntervalLimitHours(indexerDef); + var intervalLimitMinutes = _indexerLimitService.CalculateIntervalLimitMinutes(indexerDef); - return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Query Limit of {queryLimit} in last {intervalLimitHours} hour(s) reached."), statusCode: StatusCodes.Status429TooManyRequests); + return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Query Limit of {queryLimit} in last {IndexerLimitService.FormatIntervalLimit(intervalLimitMinutes)} reached."), statusCode: StatusCodes.Status429TooManyRequests); } switch (requestType) @@ -240,9 +240,9 @@ public async Task GetDownload(int id, string link, string file) AddRetryAfterHeader(retryAfterDownloadLimit); var grabLimit = ((IIndexerSettings)indexer.Definition.Settings).BaseSettings.GrabLimit; - var intervalLimitHours = _indexerLimitService.CalculateIntervalLimitHours(indexerDef); + var intervalLimitMinutes = _indexerLimitService.CalculateIntervalLimitMinutes(indexerDef); - return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Grab Limit of {grabLimit} in last {intervalLimitHours} hour(s) reached."), statusCode: StatusCodes.Status429TooManyRequests); + return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Grab Limit of {grabLimit} in last {IndexerLimitService.FormatIntervalLimit(intervalLimitMinutes)} reached."), statusCode: StatusCodes.Status429TooManyRequests); } if (link.IsNullOrWhiteSpace() || file.IsNullOrWhiteSpace())