Fixed: (Indexers) Add SupportsPagination to prevent fetching the first page multiple times

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -55,6 +55,7 @@ public static void Map()
                   .Ignore(i => i.SupportsRss)
                   .Ignore(i => i.SupportsSearch)
                   .Ignore(i => i.SupportsRedirect)
+                  .Ignore(i => i.SupportsPagination)
                   .Ignore(i => i.Capabilities)
                   .HasOne(a => a.AppProfile, a => a.AppProfileId);
 

src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs

@@ -124,12 +124,6 @@ private IndexerPageableRequestChain GetRequestWithSearchType(SearchCriteriaBase
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            // TODO: Remove this once Prowlarr has proper support for non Pageable Indexers and can tell Sonarr that indexer doesn't support pagination in a proper way, for now just return empty release list on all request containing an offset
-            if (searchCriteria.Offset is > 0)
-            {
-                return pageableRequests;
-            }
-
             pageableRequests.Add(GetRequest(searchType, searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
 
             return pageableRequests;

src/NzbDrone.Core/Indexers/Definitions/BinSearch.cs

@@ -29,6 +29,7 @@ public class BinSearch : UsenetIndexerBase<BinSearchSettings>
         public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
         public override bool SupportsRss => false;
+        public override bool SupportsPagination => true;
         public override IndexerCapabilities Capabilities => SetCapabilities();
 
         public BinSearch(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)

src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNet.cs

@@ -15,6 +15,7 @@ public class BroadcastheNet : TorrentIndexerBase<BroadcastheNetSettings>
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
         public override bool SupportsRss => true;
         public override bool SupportsSearch => true;
+        public override bool SupportsPagination => true;
         public override int PageSize => 100;
         public override IndexerCapabilities Capabilities => SetCapabilities();
         public override TimeSpan RateLimit => TimeSpan.FromSeconds(5);

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -165,6 +165,7 @@ private IndexerDefinition GetDefinition(CardigannMetaDefinition definition)
                 SupportsRss = SupportsRss,
                 SupportsSearch = SupportsSearch,
                 SupportsRedirect = SupportsRedirect,
+                SupportsPagination = SupportsPagination,
                 Capabilities = new IndexerCapabilities(),
                 ExtraFields = settings
             };

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -1022,15 +1022,6 @@ public bool CheckIfLoginIsNeeded(HttpResponse response)
 
         private IEnumerable<IndexerRequest> GetRequest(Dictionary<string, object> variables, SearchCriteriaBase searchCriteria)
         {
-            var limit = searchCriteria.Limit ?? 100;
-            var offset = searchCriteria.Offset ?? 0;
-
-            if (offset > 0 && limit > 0 && offset / limit > 0)
-            {
-                // Pagination doesn't work yet, this is to prevent fetching the first page multiple times.
-                yield break;
-            }
-
             var search = _definition.Search;
 
             var mappedCategories = _categories.MapTorznabCapsToTrackers((int[])variables[".Query.Categories"]);

src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs

@@ -19,6 +19,7 @@ public class Headphones : UsenetIndexerBase<HeadphonesSettings>
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
         public override string[] IndexerUrls => new string[] { "https://indexer.codeshy.com" };
         public override string Description => "A Private Usenet indexer for music";
+        public override bool SupportsPagination => true;
         public override IndexerCapabilities Capabilities => SetCapabilities();
 
         public override IIndexerRequestGenerator GetRequestGenerator()

src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs

@@ -40,6 +40,7 @@ public class IPTorrents : TorrentIndexerBase<IPTorrentsSettings>
         public override string Description => "IPTorrents (IPT) is a Private Torrent Tracker for 0DAY / GENERAL.";
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+        public override bool SupportsPagination => true;
         public override IndexerCapabilities Capabilities => SetCapabilities();
 
         public IPTorrents(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@@ -175,13 +176,13 @@ public class IPTorrentsRequestGenerator : IIndexerRequestGenerator
         public IPTorrentsSettings Settings { get; set; }
         public IndexerCapabilities Capabilities { get; set; }
 
-        private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, int limit, int offset, string imdbId = null)
+        private IEnumerable<IndexerRequest> GetPagedRequests(string term, SearchCriteriaBase searchCriteria, string imdbId = null)
         {
             var searchUrl = Settings.BaseUrl + "t";
 
             var qc = new NameValueCollection();
 
-            foreach (var cat in Capabilities.Categories.MapTorznabCapsToTrackers(categories))
+            foreach (var cat in Capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories))
             {
                 qc.Add(cat, string.Empty);
             }
@@ -204,9 +205,9 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categori
                 qc.Add("q", "+(" + term + ")");
             }
 
-            if (offset > 0 && limit > 0)
+            if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
             {
-                var page = (int)(offset / limit) + 1;
+                var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
                 qc.Add("p", page.ToString());
             }
 
@@ -229,7 +230,7 @@ public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchC
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0, searchCriteria.FullImdbId));
+            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SearchTerm}", searchCriteria, searchCriteria.FullImdbId));
 
             return pageableRequests;
         }
@@ -238,7 +239,7 @@ public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchC
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
+            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria));
 
             return pageableRequests;
         }
@@ -247,7 +248,7 @@ public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCrit
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0, searchCriteria.FullImdbId));
+            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}", searchCriteria, searchCriteria.FullImdbId));
 
             return pageableRequests;
         }
@@ -256,7 +257,7 @@ public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCr
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
+            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria));
 
             return pageableRequests;
         }
@@ -265,7 +266,7 @@ public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchC
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
+            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria));
 
             return pageableRequests;
         }

src/NzbDrone.Core/Indexers/Definitions/Libble.cs

@@ -33,6 +33,7 @@ public class Libble : TorrentIndexerBase<LibbleSettings>
     public override Encoding Encoding => Encoding.UTF8;
     public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
     public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+    public override bool SupportsPagination => true;
     public override int PageSize => 50;
     public override IndexerCapabilities Capabilities => SetCapabilities();
 
@@ -206,7 +207,7 @@ private IEnumerable<IndexerRequest> GetPagedRequests(SearchCriteriaBase searchCr
             queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
         }
 
-        if (searchCriteria.Offset.HasValue && searchCriteria.Limit.HasValue && searchCriteria.Offset > 0 && searchCriteria.Limit > 0)
+        if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
         {
             var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
             parameters.Set("page", page.ToString());

src/NzbDrone.Core/Indexers/Definitions/MyAnonamouse.cs

@@ -32,6 +32,7 @@ public class MyAnonamouse : TorrentIndexerBase<MyAnonamouseSettings>
         public override string Description => "MyAnonaMouse (MAM) is a large ebook and audiobook tracker.";
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+        public override bool SupportsPagination => true;
         public override int PageSize => 100;
         public override IndexerCapabilities Capabilities => SetCapabilities();
         private readonly ICacheManager _cacheManager;

src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs

@@ -29,6 +29,7 @@ public class Nebulance : TorrentIndexerBase<NebulanceSettings>
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
         public override bool SupportsRedirect => true;
+        public override bool SupportsPagination => true;
         public override IndexerCapabilities Capabilities => SetCapabilities();
 
         public Nebulance(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)

src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs

@@ -23,6 +23,7 @@ public class Newznab : UsenetIndexerBase<NewznabSettings>
         public override string Description => "Newznab is an API search specification for Usenet";
         public override bool FollowRedirect => true;
         public override bool SupportsRedirect => true;
+        public override bool SupportsPagination => true;
 
         public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
@@ -135,6 +136,7 @@ private IndexerDefinition GetDefinition(string name, NewznabSettings settings)
                 SupportsRss = SupportsRss,
                 SupportsSearch = SupportsSearch,
                 SupportsRedirect = SupportsRedirect,
+                SupportsPagination = SupportsPagination,
                 Capabilities = Capabilities
             };
         }

src/NzbDrone.Core/Indexers/Definitions/NzbIndex.cs

@@ -25,6 +25,7 @@ public class NzbIndex : TorrentIndexerBase<NzbIndexSettings>
         public override string Description => "A Usenet Indexer";
         public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
         public override IndexerPrivacy Privacy => IndexerPrivacy.SemiPrivate;
+        public override bool SupportsPagination => true;
         public override IndexerCapabilities Capabilities => SetCapabilities();
 
         public NzbIndex(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)

src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs

@@ -30,6 +30,7 @@ public abstract class SpeedAppBase : TorrentIndexerBase<SpeedAppSettings>
         private string LoginUrl => Settings.BaseUrl + "api/login";
         public override Encoding Encoding => Encoding.UTF8;
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override bool SupportsPagination => true;
         public override int PageSize => 100;
         public override IndexerCapabilities Capabilities => SetCapabilities();
         protected virtual int MinimumSeedTime => 172800; // 48 hours
@@ -169,26 +170,23 @@ private IndexerPageableRequestChain GetSearch(SearchCriteriaBase searchCriteria,
         {
             var pageableRequests = new IndexerPageableRequestChain();
 
-            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit ?? _pageSize, searchCriteria.Offset ?? 0, imdbId, season, episode));
+            pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria, imdbId, season, episode));
 
             return pageableRequests;
         }
 
-        private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, int limit, int offset, string imdbId = null, int? season = null, string episode = null)
+        private IEnumerable<IndexerRequest> GetPagedRequests(string term, SearchCriteriaBase searchCriteria, string imdbId = null, int? season = null, string episode = null)
         {
-            limit = Math.Min(_pageSize, limit);
-            offset = Math.Max(0, offset);
-
             var parameters = new NameValueCollection
             {
-                { "itemsPerPage", limit.ToString() },
+                { "itemsPerPage", Math.Min(_pageSize, searchCriteria.Limit.GetValueOrDefault(_pageSize)).ToString() },
                 { "sort", "torrent.createdAt" },
                 { "direction", "desc" }
             };
 
-            if (limit > 0 && offset > 0)
+            if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
             {
-                var page = (offset / limit) + 1;
+                var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
                 parameters.Set("page", page.ToString());
             }
 
@@ -211,7 +209,7 @@ private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categori
                 parameters.Set("episode", episode);
             }
 
-            var cats = _capabilities.Categories.MapTorznabCapsToTrackers(categories);
+            var cats = _capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories);
             if (cats.Count > 0)
             {
                 foreach (var cat in cats)

src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs

@@ -32,7 +32,8 @@ private IndexerDefinition GetDefinition(string name, TorrentPotatoSettings setti
                 Protocol = DownloadProtocol.Torrent,
                 SupportsRss = SupportsRss,
                 SupportsSearch = SupportsSearch,
-                SupportsRedirect = SupportsRedirect
+                SupportsRedirect = SupportsRedirect,
+                SupportsPagination = SupportsPagination
             };
         }
 

src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs

@@ -113,6 +113,7 @@ private IndexerDefinition GetDefinition(string name, string description, Torznab
                 SupportsRss = SupportsRss,
                 SupportsSearch = SupportsSearch,
                 SupportsRedirect = SupportsRedirect,
+                SupportsPagination = SupportsPagination,
                 Capabilities = Capabilities
             };
         }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -33,6 +33,7 @@ public abstract class HttpIndexerBase<TSettings> : IndexerBase<TSettings>
         public override bool SupportsRss => true;
         public override bool SupportsSearch => true;
         public override bool SupportsRedirect => false;
+        public override bool SupportsPagination => false;
 
         public override Encoding Encoding => Encoding.UTF8;
         public override string Language => "en-US";
@@ -60,6 +61,11 @@ public override Task<IndexerPageableQueryResult> Fetch(MovieSearchCriteria searc
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
+            if (!SupportsPagination && searchCriteria.Offset is > 0)
+            {
+                return Task.FromResult(new IndexerPageableQueryResult());
+            }
+
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
@@ -70,6 +76,11 @@ public override Task<IndexerPageableQueryResult> Fetch(MusicSearchCriteria searc
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
+            if (!SupportsPagination && searchCriteria.Offset is > 0)
+            {
+                return Task.FromResult(new IndexerPageableQueryResult());
+            }
+
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
@@ -80,6 +91,11 @@ public override Task<IndexerPageableQueryResult> Fetch(TvSearchCriteria searchCr
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
+            if (!SupportsPagination && searchCriteria.Offset is > 0)
+            {
+                return Task.FromResult(new IndexerPageableQueryResult());
+            }
+
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
@@ -90,6 +106,11 @@ public override Task<IndexerPageableQueryResult> Fetch(BookSearchCriteria search
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
+            if (!SupportsPagination && searchCriteria.Offset is > 0)
+            {
+                return Task.FromResult(new IndexerPageableQueryResult());
+            }
+
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 
@@ -100,6 +121,11 @@ public override Task<IndexerPageableQueryResult> Fetch(BasicSearchCriteria searc
                 return Task.FromResult(new IndexerPageableQueryResult());
             }
 
+            if (!SupportsPagination && searchCriteria.Offset is > 0)
+            {
+                return Task.FromResult(new IndexerPageableQueryResult());
+            }
+
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria), searchCriteria);
         }
 

src/NzbDrone.Core/Indexers/IIndexer.cs

@@ -11,6 +11,7 @@ public interface IIndexer : IProvider
         bool SupportsRss { get; }
         bool SupportsSearch { get; }
         bool SupportsRedirect { get; }
+        bool SupportsPagination { get; }
         IndexerCapabilities Capabilities { get; }
 
         string[] IndexerUrls { get; }

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -36,6 +36,7 @@ public abstract class IndexerBase<TSettings> : IIndexer
         public abstract bool SupportsRss { get; }
         public abstract bool SupportsSearch { get; }
         public abstract bool SupportsRedirect { get; }
+        public abstract bool SupportsPagination { get; }
         public abstract IndexerCapabilities Capabilities { get; protected set; }
 
         public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@@ -51,12 +52,7 @@ public bool IsObsolete()
         {
             var attributes = GetType().GetCustomAttributes(false);
 
-            foreach (ObsoleteAttribute attribute in attributes.OfType<ObsoleteAttribute>())
-            {
-                return true;
-            }
-
-            return false;
+            return attributes.OfType<ObsoleteAttribute>().Any();
         }
 
         public virtual ProviderMessage Message => null;

src/NzbDrone.Core/Indexers/IndexerDefinition.cs

@@ -20,6 +20,7 @@ public class IndexerDefinition : ProviderDefinition
         public bool SupportsRss { get; set; }
         public bool SupportsSearch { get; set; }
         public bool SupportsRedirect { get; set; }
+        public bool SupportsPagination { get; set; }
         public IndexerCapabilities Capabilities { get; set; }
         public int Priority { get; set; } = 25;
         public bool Redirect { get; set; }

src/NzbDrone.Core/Indexers/IndexerFactory.cs

@@ -201,9 +201,10 @@ public override void SetProviderCharacteristics(IIndexer provider, IndexerDefini
             definition.SupportsRss = provider.SupportsRss;
             definition.SupportsSearch = provider.SupportsSearch;
             definition.SupportsRedirect = provider.SupportsRedirect;
+            definition.SupportsPagination = provider.SupportsPagination;
 
             //We want to use the definition Caps and Privacy for Cardigann instead of the provider.
-            if (definition.Implementation != typeof(Cardigann.Cardigann).Name)
+            if (definition.Implementation != nameof(Cardigann.Cardigann))
             {
                 definition.IndexerUrls = provider.IndexerUrls;
                 definition.LegacyUrls = provider.LegacyUrls;

src/Prowlarr.Api.V1/Indexers/IndexerResource.cs

@@ -25,6 +25,7 @@ public class IndexerResource : ProviderResource<IndexerResource>
         public bool SupportsRss { get; set; }
         public bool SupportsSearch { get; set; }
         public bool SupportsRedirect { get; set; }
+        public bool SupportsPagination { get; set; }
         public int AppProfileId { get; set; }
         public DownloadProtocol Protocol { get; set; }
         public IndexerPrivacy Privacy { get; set; }
@@ -57,9 +58,9 @@ public override IndexerResource ToResource(IndexerDefinition definition)
 
             var infoLinkName = definition.ImplementationName;
 
-            if (definition.Implementation == typeof(Cardigann).Name)
+            if (definition.Implementation == nameof(Cardigann))
             {
-                var extraFields = definition.ExtraFields?.Select((x, i) => MapField(x, i)).ToList() ?? new List<Field>();
+                var extraFields = definition.ExtraFields?.Select(MapField).ToList() ?? new List<Field>();
 
                 resource.Fields.AddRange(extraFields);
 
@@ -79,7 +80,7 @@ public override IndexerResource ToResource(IndexerDefinition definition)
                 infoLinkName = settings.DefinitionFile;
             }
 
-            resource.InfoLink = string.Format("https://wiki.servarr.com/prowlarr/supported-indexers#{0}", infoLinkName.ToLower().Replace(' ', '-'));
+            resource.InfoLink = $"https://wiki.servarr.com/prowlarr/supported-indexers#{infoLinkName.ToLower().Replace(' ', '-')}";
             resource.AppProfileId = definition.AppProfileId;
             resource.IndexerUrls = definition.IndexerUrls;
             resource.LegacyUrls = definition.LegacyUrls;
@@ -91,6 +92,7 @@ public override IndexerResource ToResource(IndexerDefinition definition)
             resource.SupportsRss = definition.SupportsRss;
             resource.SupportsSearch = definition.SupportsSearch;
             resource.SupportsRedirect = definition.SupportsRedirect;
+            resource.SupportsPagination = definition.SupportsPagination;
             resource.Capabilities = definition.Capabilities.ToResource();
             resource.Protocol = definition.Protocol;
             resource.Privacy = definition.Privacy;