diff --git a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs index 9ed678895c..0aa9fcf280 100644 --- a/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/HttpImportListBase.cs @@ -39,35 +39,41 @@ protected HttpImportListBase(IHttpClient httpClient, IImportListStatusService im public override ImportListFetchResult Fetch() { - var generator = GetRequestGenerator(); - return FetchMovies(generator.GetMovies()); + return FetchMovies(g => g.GetMovies()); } - protected virtual ImportListFetchResult FetchMovies(ImportListPageableRequestChain pageableRequestChain, bool isRecent = false) + protected virtual ImportListFetchResult FetchMovies(Func pageableRequestChainSelector, bool isRecent = false) { var movies = new List(); var url = string.Empty; - var parser = GetParser(); - var anyFailure = true; try { + var generator = GetRequestGenerator(); + var parser = GetParser(); + + var pageableRequestChain = pageableRequestChainSelector(generator); + for (var i = 0; i < pageableRequestChain.Tiers; i++) { var pageableRequests = pageableRequestChain.GetTier(i); + foreach (var pageableRequest in pageableRequests) { - var pagedReleases = new List(); + var pagedMovies = new List(); + foreach (var request in pageableRequest) { url = request.Url.FullUri; + var page = FetchPage(request, parser); - pagedReleases.AddRange(page); + + pagedMovies.AddRange(page); } - movies.AddRange(pagedReleases); + movies.AddRange(pagedMovies.Where(IsValidItem)); } if (movies.Any()) @@ -81,8 +87,7 @@ protected virtual ImportListFetchResult FetchMovies(ImportListPageableRequestCha } catch (WebException webException) { - if (webException.Status == WebExceptionStatus.NameResolutionFailure || - webException.Status == WebExceptionStatus.ConnectFailure) + if (webException.Status is WebExceptionStatus.NameResolutionFailure or WebExceptionStatus.ConnectFailure) { _importListStatusService.RecordConnectionFailure(Definition.Id); } @@ -103,14 +108,8 @@ protected virtual ImportListFetchResult FetchMovies(ImportListPageableRequestCha } catch (TooManyRequestsException ex) { - if (ex.RetryAfter != TimeSpan.Zero) - { - _importListStatusService.RecordFailure(Definition.Id, ex.RetryAfter); - } - else - { - _importListStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1)); - } + var retryTime = ex.RetryAfter != TimeSpan.Zero ? ex.RetryAfter : TimeSpan.FromHours(1); + _importListStatusService.RecordFailure(Definition.Id, retryTime); _logger.Warn("API Request Limit reached for {0}", this); } @@ -152,6 +151,16 @@ protected virtual ImportListFetchResult FetchMovies(ImportListPageableRequestCha return new ImportListFetchResult { Movies = CleanupListItems(movies), AnyFailure = anyFailure }; } + protected virtual bool IsValidItem(ImportListMovie listItem) + { + if (listItem.Title.IsNullOrWhiteSpace() && listItem.ImdbId.IsNullOrWhiteSpace() && listItem.TmdbId == 0) + { + return false; + } + + return true; + } + protected virtual IList FetchPage(ImportListRequest request, IParseImportListResponse parser) { var response = FetchImportListResponse(request); diff --git a/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs b/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs index 7a73c46f65..2d54213d6f 100644 --- a/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs +++ b/src/NzbDrone.Core/ImportLists/Plex/PlexImport.cs @@ -37,8 +37,7 @@ public override ImportListFetchResult Fetch() { Settings.Validate().Filter("AccessToken").ThrowOnError(); - var generator = GetRequestGenerator(); - return FetchMovies(generator.GetMovies()); + return FetchMovies(g => g.GetMovies()); } public override IParseImportListResponse GetParser() diff --git a/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs b/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs index 5d3c1c29cf..b6b8adeb06 100644 --- a/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/Rss/RssImportBase.cs @@ -22,9 +22,7 @@ public RssImportBase(IHttpClient httpClient, public override ImportListFetchResult Fetch() { - var generator = GetRequestGenerator(); - - return FetchMovies(generator.GetMovies()); + return FetchMovies(g => g.GetMovies()); } public override IParseImportListResponse GetParser() diff --git a/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs b/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs index f780e25567..a46f3f652a 100644 --- a/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/Simkl/SimklImportBase.cs @@ -53,9 +53,7 @@ public override ImportListFetchResult Fetch() return new ImportListFetchResult(); } - var generator = GetRequestGenerator(); - - return FetchMovies(generator.GetMovies()); + return FetchMovies(g => g.GetMovies()); } public override IParseImportListResponse GetParser() diff --git a/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs b/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs index 52e74182e0..cd1c3dbfb9 100644 --- a/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs +++ b/src/NzbDrone.Core/ImportLists/Trakt/TraktImportBase.cs @@ -40,8 +40,7 @@ public override ImportListFetchResult Fetch() RefreshToken(); } - var generator = GetRequestGenerator(); - return FetchMovies(generator.GetMovies()); + return FetchMovies(g => g.GetMovies()); } public override IParseImportListResponse GetParser()