From 3b964275ee577436e63a56066c9ec548a1a7b658 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:30:54 -0400 Subject: [PATCH 01/14] Discogs list by ID --- .gitignore | 3 + .../Discogs/DiscogsListsFixture.cs | 283 ++++++++++++++++++ .../ImportListServiceFixture.cs | 2 + .../ImportLists/Discogs/DiscogsLists.cs | 38 +++ .../ImportLists/Discogs/DiscogsListsParser.cs | 148 +++++++++ .../Discogs/DiscogsListsRequestGenerator.cs | 36 +++ .../Discogs/DiscogsListsSettings.cs | 38 +++ .../ImportLists/ImportListType.cs | 18 +- 8 files changed, 557 insertions(+), 9 deletions(-) create mode 100644 src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs diff --git a/.gitignore b/.gitignore index a5d6bb7c8..3c277c7b0 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,6 @@ node_modules.nosync # Ignore Jetbrains IntelliJ Workspace Directories .idea/ + +# Ignore Claude +**/.claude \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs new file mode 100644 index 000000000..48a395d6e --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs @@ -0,0 +1,283 @@ +using System.Linq; +using System.Net; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.ImportLists.Discogs; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests.Discogs; + +[TestFixture] +public class DiscogsListsFixture : CoreTest +{ + [SetUp] + public void Setup() + { + Subject.Definition = new ImportLists.ImportListDefinition + { + Id = 1, + Name = "Test Discogs List", + Settings = new DiscogsListsSettings + { + Token = "test_token_123456", + ListId = "123456", + BaseUrl = "https://api.discogs.com" + } + }; + } + + [Test] + public void should_parse_valid_list_response() + { + var listResponseJson = @"{ + ""items"": [ + { + ""type"": ""release"", + ""id"": 3, + ""display_title"": ""Josh Wink - Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Profound Sounds Vol. 1"", + ""artists"": [ + { + ""name"": ""Josh Wink"", + ""id"": 3 + } + ] + }"; + + var listCalled = false; + var releaseCalled = false; + var actualListUrl = ""; + var actualReleaseUrl = ""; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Callback(req => + { + if (req.Url.FullUri.Contains("/lists/")) + { + listCalled = true; + actualListUrl = req.Url.FullUri; + } + else if (req.Url.FullUri.Contains("/releases/")) + { + releaseCalled = true; + actualReleaseUrl = req.Url.FullUri; + } + }) + .Returns(req => + { + if (req.Url.FullUri.Contains("/lists/")) + { + return new HttpResponse(req, new HttpHeader(), listResponseJson, HttpStatusCode.OK); + } + else if (req.Url.FullUri.Contains("/releases/")) + { + return new HttpResponse(req, new HttpHeader(), releaseResponseJson, HttpStatusCode.OK); + } + else + { + return new HttpResponse(req, new HttpHeader(), "", HttpStatusCode.NotFound); + } + }); + + var releases = Subject.Fetch(); + + // Debug output to see what happened + System.Console.WriteLine($"List called: {listCalled}, URL: {actualListUrl}"); + System.Console.WriteLine($"Release called: {releaseCalled}, URL: {actualReleaseUrl}"); + System.Console.WriteLine($"Releases count: {releases.Count}"); + for (var i = 0; i < releases.Count; i++) + { + System.Console.WriteLine($"Release {i}: Artist='{releases[i].Artist}', Album='{releases[i].Album}'"); + } + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Josh Wink"); + releases.First().Album.Should().Be("Profound Sounds Vol. 1"); + } + + [Test] + public void should_handle_empty_list_response() + { + var responseJson = @"{""items"": []}"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/lists/123456"), new HttpHeader(), responseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void debug_test_to_see_what_url_is_called() + { + var responseJson = @"{""items"": []}"; + var actualUrl = ""; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Callback(req => actualUrl = req.Url.FullUri) + .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/lists/123456"), new HttpHeader(), responseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + // This will show us what URL is actually being called + actualUrl.Should().NotBeEmpty(); + } + + [Test] + public void should_skip_non_release_items() + { + var responseJson = @"{ + ""items"": [ + { + ""type"": ""label"", + ""id"": 1, + ""display_title"": ""Some Label"", + ""resource_url"": ""https://api.discogs.com/labels/1"" + }, + { + ""type"": ""release"", + ""id"": 3, + ""display_title"": ""Josh Wink - Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Profound Sounds Vol. 1"", + ""artists"": [ + { + ""name"": ""Josh Wink"", + ""id"": 3 + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), responseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Josh Wink"); + } + + [Test] + public void should_skip_items_when_release_fetch_fails() + { + var listResponseJson = @"{ + ""items"": [ + { + ""type"": ""release"", + ""id"": 3, + ""display_title"": ""Josh Wink - Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), listResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), "", HttpStatusCode.NotFound)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_skip_releases_with_no_artists() + { + var listResponseJson = @"{ + ""items"": [ + { + ""type"": ""release"", + ""id"": 3, + ""display_title"": ""Various - Compilation"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Compilation"", + ""artists"": [] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), listResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_use_first_artist_when_multiple_artists() + { + var listResponseJson = @"{ + ""items"": [ + { + ""type"": ""release"", + ""id"": 3, + ""display_title"": ""Artist 1 & Artist 2 - Collaboration"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Collaboration"", + ""artists"": [ + { + ""name"": ""Artist 1"", + ""id"": 1 + }, + { + ""name"": ""Artist 2"", + ""id"": 2 + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), listResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Artist 1"); + releases.First().Album.Should().Be("Collaboration"); + } +} diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs index 0be3a49cd..79e38590e 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListServiceFixture.cs @@ -3,6 +3,7 @@ using FluentAssertions; using NUnit.Framework; using NzbDrone.Core.ImportLists; +using NzbDrone.Core.ImportLists.Discogs; using NzbDrone.Core.ImportLists.LidarrLists; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Test.Framework; @@ -19,6 +20,7 @@ public void Setup() _importLists = new List(); _importLists.Add(Mocker.Resolve()); + _importLists.Add(Mocker.Resolve()); Mocker.SetConstant>(_importLists); } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs new file mode 100644 index 000000000..b5f311e70 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs @@ -0,0 +1,38 @@ +using System; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsLists : HttpImportListBase + { + public override string Name => "Discogs Lists"; + public override ImportListType ListType => ImportListType.Discogs; + public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); + public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); // 60 requests per minute limit + public override int PageSize => 0; // Discogs doesn't support pagination for lists + + public DiscogsLists(IHttpClient httpClient, + IImportListStatusService importListStatusService, + IConfigService configService, + IParsingService parsingService, + Logger logger) + : base(httpClient, importListStatusService, configService, parsingService, logger) + { + } + + public override IImportListRequestGenerator GetRequestGenerator() + { + return new DiscogsListsRequestGenerator { Settings = Settings }; + } + + public override IParseImportListResponse GetParser() + { + var parser = new DiscogsListsParser(); + parser.SetContext(_httpClient, Settings); + return parser; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs new file mode 100644 index 000000000..38261c82e --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -0,0 +1,148 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using Newtonsoft.Json; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists.Discogs; + +public class DiscogsListsParser : IParseImportListResponse +{ + private ImportListResponse _importListResponse; + private IHttpClient _httpClient; + private DiscogsListsSettings _settings; + + public DiscogsListsParser() + { + } + + public void SetContext(IHttpClient httpClient, DiscogsListsSettings settings) + { + _httpClient = httpClient; + _settings = settings; + } + + public IList ParseResponse(ImportListResponse importListResponse) + { + _importListResponse = importListResponse; + + var items = new List(); + + if (!PreProcess(_importListResponse)) + { + return items; + } + + var jsonResponse = Json.Deserialize(_importListResponse.Content); + + if (jsonResponse?.Items == null) + { + return items; + } + + foreach (var item in jsonResponse.Items) + { + if (item?.Type == "release" && item.ResourceUrl.IsNotNullOrWhiteSpace()) + { + try + { + if (_httpClient != null && _settings != null) + { + var releaseInfo = FetchReleaseDetails(item.ResourceUrl); + if (releaseInfo != null) + { + items.Add(releaseInfo); + } + } + } + catch + { + // If we can't fetch release details, skip this item + continue; + } + } + } + + return items; + } + + // Unfortunately discogs release details are nested in a given /release/N endpoint. + // We'll have to fetch each one to get proper details. + private ImportListItemInfo FetchReleaseDetails(string resourceUrl) + { + var request = new HttpRequestBuilder(resourceUrl) + .SetHeader("Authorization", $"Discogs token={_settings.Token}") + .Build(); + + var response = _httpClient.Execute(request); + + if (response.StatusCode != HttpStatusCode.OK) + { + return null; + } + + var releaseResponse = Json.Deserialize(response.Content); + + if (releaseResponse?.Artists?.Any() == true && releaseResponse.Title.IsNotNullOrWhiteSpace()) + { + return new ImportListItemInfo + { + Artist = releaseResponse.Artists.First().Name, + Album = releaseResponse.Title + }; + } + + return null; + } + + protected virtual bool PreProcess(ImportListResponse importListResponse) + { + if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(importListResponse, + "Discogs API call resulted in an unexpected StatusCode [{0}]", + importListResponse.HttpResponse.StatusCode); + } + + if (importListResponse.HttpResponse.Headers.ContentType != null && + importListResponse.HttpResponse.Headers.ContentType.Contains("text/html")) + { + throw new ImportListException(importListResponse, + "Discogs API responded with HTML content. List may be too large or API may be unavailable."); + } + + return true; + } +} + +public class DiscogsListResponse +{ + public List Items { get; set; } +} + +public class DiscogsListItem +{ + public string Type { get; set; } + public int Id { get; set; } + [JsonProperty("display_title")] + public string DisplayTitle { get; set; } + [JsonProperty("resource_url")] + public string ResourceUrl { get; set; } + public string Uri { get; set; } +} + +public class DiscogsReleaseResponse +{ + public string Title { get; set; } + public List Artists { get; set; } +} + +public class DiscogsReleaseArtist +{ + public string Name { get; set; } + public int Id { get; set; } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs new file mode 100644 index 000000000..6d5bb14f3 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsListsRequestGenerator : IImportListRequestGenerator + { + public DiscogsListsSettings Settings { get; set; } + + public int MaxPages { get; set; } + public int PageSize { get; set; } + + public DiscogsListsRequestGenerator() + { + MaxPages = 1; + PageSize = 0; // Discogs doesn't support pagination for lists currently + } + + public virtual ImportListPageableRequestChain GetListItems() + { + var pageableRequests = new ImportListPageableRequestChain(); + pageableRequests.Add(GetPagedRequests()); + return pageableRequests; + } + + private IEnumerable GetPagedRequests() + { + var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/')) + .Resource($"/lists/{Settings.ListId}") + .SetHeader("Authorization", $"Discogs token={Settings.Token}") + .Build(); + + yield return new ImportListRequest(request); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs new file mode 100644 index 000000000..68c2dc8ca --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs @@ -0,0 +1,38 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsListsSettingsValidator : AbstractValidator + { + public DiscogsListsSettingsValidator() + { + RuleFor(c => c.Token).NotEmpty(); + RuleFor(c => c.ListId).NotEmpty(); + } + } + + public class DiscogsListsSettings : IImportListSettings + { + private static readonly DiscogsListsSettingsValidator Validator = new DiscogsListsSettingsValidator(); + + public DiscogsListsSettings() + { + BaseUrl = "https://api.discogs.com"; + } + + public string BaseUrl { get; set; } + + [FieldDefinition(0, Label = "Token", Privacy = PrivacyLevel.ApiKey, HelpText = "Discogs Personal Access Token")] + public string Token { get; set; } + + [FieldDefinition(1, Label = "List ID", HelpText = "ID of the Discogs list to import")] + public string ListId { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/ImportListType.cs b/src/NzbDrone.Core/ImportLists/ImportListType.cs index 6dd76ef31..4be144c9c 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListType.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListType.cs @@ -1,11 +1,11 @@ -namespace NzbDrone.Core.ImportLists +namespace NzbDrone.Core.ImportLists; + +public enum ImportListType { - public enum ImportListType - { - Program, - Spotify, - LastFm, - Other, - Advanced - } + Program, + Spotify, + LastFm, + Discogs, + Other, + Advanced } From 16675ef44cc96c605a1898cc24730b30e1f87eb4 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:53:13 -0400 Subject: [PATCH 02/14] Wantlists --- .../Discogs/DiscogsWantlistFixture.cs | 232 ++++++++++++++++++ .../ImportLists/Discogs/DiscogsLists.cs | 2 +- .../ImportLists/Discogs/DiscogsWantlist.cs | 38 +++ .../Discogs/DiscogsWantlistParser.cs | 140 +++++++++++ .../DiscogsWantlistRequestGenerator.cs | 36 +++ .../Discogs/DiscogsWantlistSettings.cs | 38 +++ 6 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs new file mode 100644 index 000000000..af620c276 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs @@ -0,0 +1,232 @@ +using System.Linq; +using System.Net; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.ImportLists.Discogs; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests.Discogs; + +[TestFixture] +public class DiscogsWantlistFixture : CoreTest +{ + [SetUp] + public void Setup() + { + Subject.Definition = new ImportLists.ImportListDefinition + { + Id = 1, + Name = "Test Discogs Wantlist", + Settings = new DiscogsWantlistSettings + { + Token = "test_token_123456", + Username = "test_user", + BaseUrl = "https://api.discogs.com" + } + }; + } + + [Test] + public void should_parse_valid_wantlist_response() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"", + ""artists"": [ + { + ""name"": ""Josh Wink"", + ""id"": 3 + } + ] + } + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Profound Sounds Vol. 1"", + ""artists"": [ + { + ""name"": ""Josh Wink"", + ""id"": 3 + } + ] + }"; + + var wantlistCalled = false; + var releaseCalled = false; + var actualWantlistUrl = ""; + var actualReleaseUrl = ""; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Callback(req => + { + if (req.Url.FullUri.Contains("/wants")) + { + wantlistCalled = true; + actualWantlistUrl = req.Url.FullUri; + } + else if (req.Url.FullUri.Contains("/releases/")) + { + releaseCalled = true; + actualReleaseUrl = req.Url.FullUri; + } + }) + .Returns(req => + { + if (req.Url.FullUri.Contains("/wants")) + { + return new HttpResponse(req, new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK); + } + else if (req.Url.FullUri.Contains("/releases/")) + { + return new HttpResponse(req, new HttpHeader(), releaseResponseJson, HttpStatusCode.OK); + } + else + { + return new HttpResponse(req, new HttpHeader(), "", HttpStatusCode.NotFound); + } + }); + + var releases = Subject.Fetch(); + + // Debug output to see what happened + System.Console.WriteLine($"Wantlist called: {wantlistCalled}, URL: {actualWantlistUrl}"); + System.Console.WriteLine($"Release called: {releaseCalled}, URL: {actualReleaseUrl}"); + System.Console.WriteLine($"Releases count: {releases.Count}"); + for (var i = 0; i < releases.Count; i++) + { + System.Console.WriteLine($"Release {i}: Artist='{releases[i].Artist}', Album='{releases[i].Album}'"); + } + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Josh Wink"); + releases.First().Album.Should().Be("Profound Sounds Vol. 1"); + } + + [Test] + public void should_handle_empty_wantlist_response() + { + var responseJson = @"{""wants"": []}"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/users/test_user/wants"), new HttpHeader(), responseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_skip_items_when_release_fetch_fails() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), "", HttpStatusCode.NotFound)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_skip_releases_with_no_artists() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Compilation"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Compilation"", + ""artists"": [] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_use_first_artist_when_multiple_artists() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Collaboration"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Collaboration"", + ""artists"": [ + { + ""name"": ""Artist 1"", + ""id"": 1 + }, + { + ""name"": ""Artist 2"", + ""id"": 2 + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Artist 1"); + releases.First().Album.Should().Be("Collaboration"); + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs index b5f311e70..e2c950a76 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs @@ -11,7 +11,7 @@ public class DiscogsLists : HttpImportListBase public override string Name => "Discogs Lists"; public override ImportListType ListType => ImportListType.Discogs; public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); - public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); // 60 requests per minute limit + public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases public override int PageSize => 0; // Discogs doesn't support pagination for lists public DiscogsLists(IHttpClient httpClient, diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs new file mode 100644 index 000000000..6a1d138c9 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs @@ -0,0 +1,38 @@ +using System; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsWantlist : HttpImportListBase + { + public override string Name => "Discogs Wantlist"; + public override ImportListType ListType => ImportListType.Discogs; + public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); + public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases + public override int PageSize => 0; // Discogs doesn't support pagination for wantlists currently + + public DiscogsWantlist(IHttpClient httpClient, + IImportListStatusService importListStatusService, + IConfigService configService, + IParsingService parsingService, + Logger logger) + : base(httpClient, importListStatusService, configService, parsingService, logger) + { + } + + public override IImportListRequestGenerator GetRequestGenerator() + { + return new DiscogsWantlistRequestGenerator { Settings = Settings }; + } + + public override IParseImportListResponse GetParser() + { + var parser = new DiscogsWantlistParser(); + parser.SetContext(_httpClient, Settings); + return parser; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs new file mode 100644 index 000000000..b17068824 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs @@ -0,0 +1,140 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using Newtonsoft.Json; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists.Discogs; + +public class DiscogsWantlistParser : IParseImportListResponse +{ + private ImportListResponse _importListResponse; + private IHttpClient _httpClient; + private DiscogsWantlistSettings _settings; + + public DiscogsWantlistParser() + { + } + + public void SetContext(IHttpClient httpClient, DiscogsWantlistSettings settings) + { + _httpClient = httpClient; + _settings = settings; + } + + public IList ParseResponse(ImportListResponse importListResponse) + { + _importListResponse = importListResponse; + + var items = new List(); + + if (!PreProcess(_importListResponse)) + { + return items; + } + + var jsonResponse = Json.Deserialize(_importListResponse.Content); + + if (jsonResponse?.Wants == null) + { + return items; + } + + foreach (var want in jsonResponse.Wants) + { + if (want?.BasicInformation != null) + { + try + { + if (_httpClient != null && _settings != null) + { + var releaseInfo = FetchReleaseDetails(want.BasicInformation.ResourceUrl); + if (releaseInfo != null) + { + items.Add(releaseInfo); + } + } + } + catch + { + // If we can't fetch release details, skip this item + continue; + } + } + } + + return items; + } + + // Unfortunately discogs release details are nested in a given /release/N endpoint. + // We'll have to fetch each one to get proper details. + private ImportListItemInfo FetchReleaseDetails(string resourceUrl) + { + var request = new HttpRequestBuilder(resourceUrl) + .SetHeader("Authorization", $"Discogs token={_settings.Token}") + .Build(); + + var response = _httpClient.Execute(request); + + if (response.StatusCode != HttpStatusCode.OK) + { + return null; + } + + var releaseResponse = Json.Deserialize(response.Content); + + if (releaseResponse?.Artists?.Any() == true && releaseResponse.Title.IsNotNullOrWhiteSpace()) + { + return new ImportListItemInfo + { + Artist = releaseResponse.Artists.First().Name, + Album = releaseResponse.Title + }; + } + + return null; + } + + protected virtual bool PreProcess(ImportListResponse importListResponse) + { + if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(importListResponse, + "Discogs API call resulted in an unexpected StatusCode [{0}]", + importListResponse.HttpResponse.StatusCode); + } + + if (importListResponse.HttpResponse.Headers.ContentType != null && + importListResponse.HttpResponse.Headers.ContentType.Contains("text/html")) + { + throw new ImportListException(importListResponse, + "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable."); + } + + return true; + } +} + +public class DiscogsWantlistResponse +{ + public List Wants { get; set; } +} + +public class DiscogsWantlistItem +{ + [JsonProperty("basic_information")] + public DiscogsBasicInformation BasicInformation { get; set; } +} + +public class DiscogsBasicInformation +{ + public int Id { get; set; } + public string Title { get; set; } + [JsonProperty("resource_url")] + public string ResourceUrl { get; set; } + public List Artists { get; set; } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs new file mode 100644 index 000000000..6be4820e7 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsWantlistRequestGenerator : IImportListRequestGenerator + { + public DiscogsWantlistSettings Settings { get; set; } + + public int MaxPages { get; set; } + public int PageSize { get; set; } + + public DiscogsWantlistRequestGenerator() + { + MaxPages = 1; + PageSize = 0; // Discogs doesn't support pagination for wantlists currently + } + + public virtual ImportListPageableRequestChain GetListItems() + { + var pageableRequests = new ImportListPageableRequestChain(); + pageableRequests.Add(GetPagedRequests()); + return pageableRequests; + } + + private IEnumerable GetPagedRequests() + { + var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/')) + .Resource($"/users/{Settings.Username}/wants") + .SetHeader("Authorization", $"Discogs token={Settings.Token}") + .Build(); + + yield return new ImportListRequest(request); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs new file mode 100644 index 000000000..8279e0355 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs @@ -0,0 +1,38 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsWantlistSettingsValidator : AbstractValidator + { + public DiscogsWantlistSettingsValidator() + { + RuleFor(c => c.Token).NotEmpty(); + RuleFor(c => c.Username).NotEmpty(); + } + } + + public class DiscogsWantlistSettings : IImportListSettings + { + private static readonly DiscogsWantlistSettingsValidator Validator = new DiscogsWantlistSettingsValidator(); + + public DiscogsWantlistSettings() + { + BaseUrl = "https://api.discogs.com"; + } + + public string BaseUrl { get; set; } + + [FieldDefinition(0, Label = "Token", Privacy = PrivacyLevel.ApiKey, HelpText = "Discogs Personal Access Token")] + public string Token { get; set; } + + [FieldDefinition(1, Label = "Username", HelpText = "Discogs username whose wantlist to import")] + public string Username { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} From 5bea444bad694a2edfcde40e7a54350de5014b61 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 12:45:59 -0500 Subject: [PATCH 03/14] Clean up and don't fetch release details --- .gitignore | 11 +- .../Discogs/DiscogsListsFixture.cs | 282 ++++-------------- .../Discogs/DiscogsWantlistFixture.cs | 233 +++++---------- .../ImportLists/Discogs/DiscogsListsParser.cs | 89 ++---- .../Discogs/DiscogsParserHelper.cs | 55 ++++ .../Discogs/DiscogsWantlistParser.cs | 104 ++----- 6 files changed, 245 insertions(+), 529 deletions(-) create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs diff --git a/.gitignore b/.gitignore index 3c277c7b0..cba2cec00 100644 --- a/.gitignore +++ b/.gitignore @@ -169,4 +169,13 @@ node_modules.nosync .idea/ # Ignore Claude -**/.claude \ No newline at end of file +**/.claude + +# Ignore markdown files (except essential tracked ones) +*.md +!README.md +!CONTRIBUTING.md +!SECURITY.md +!LICENSE.md +!CODE_OF_CONDUCT.md +!CLA.md \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs index 48a395d6e..98769a012 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs @@ -4,34 +4,38 @@ using Moq; using NUnit.Framework; using NzbDrone.Common.Http; +using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Discogs; -using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ImportLists.Exceptions; namespace NzbDrone.Core.Test.ImportListTests.Discogs; [TestFixture] -public class DiscogsListsFixture : CoreTest +public class DiscogsListsFixture { + private readonly HttpHeader _defaultHeaders = new () { ContentType = "application/json" }; + + private DiscogsListsParser _parser; + private Mock _httpClient; + private DiscogsListsSettings _settings; + [SetUp] - public void Setup() + public void SetUp() { - Subject.Definition = new ImportLists.ImportListDefinition - { - Id = 1, - Name = "Test Discogs List", - Settings = new DiscogsListsSettings - { - Token = "test_token_123456", - ListId = "123456", - BaseUrl = "https://api.discogs.com" - } - }; + _httpClient = new Mock(); + _parser = new DiscogsListsParser(); + _settings = new DiscogsListsSettings { Token = "token", ListId = "123", BaseUrl = "https://api.discogs.com" }; + + _parser.SetContext(_httpClient.Object, _settings); } [Test] - public void should_parse_valid_list_response() + public void should_parse_release_items() { - var listResponseJson = @"{ + const string resourceUrl = "https://api.discogs.com/releases/3"; + GivenReleaseDetails(resourceUrl, BuildReleaseResponse("Josh Wink", "Profound Sounds Vol. 1")); + + var response = BuildListResponse(@"{ ""items"": [ { ""type"": ""release"", @@ -40,149 +44,39 @@ public void should_parse_valid_list_response() ""resource_url"": ""https://api.discogs.com/releases/3"" } ] - }"; + }"); - var releaseResponseJson = @"{ - ""title"": ""Profound Sounds Vol. 1"", - ""artists"": [ - { - ""name"": ""Josh Wink"", - ""id"": 3 - } - ] - }"; + var items = _parser.ParseResponse(response); - var listCalled = false; - var releaseCalled = false; - var actualListUrl = ""; - var actualReleaseUrl = ""; - - Mocker.GetMock() - .Setup(s => s.Execute(It.IsAny())) - .Callback(req => - { - if (req.Url.FullUri.Contains("/lists/")) - { - listCalled = true; - actualListUrl = req.Url.FullUri; - } - else if (req.Url.FullUri.Contains("/releases/")) - { - releaseCalled = true; - actualReleaseUrl = req.Url.FullUri; - } - }) - .Returns(req => - { - if (req.Url.FullUri.Contains("/lists/")) - { - return new HttpResponse(req, new HttpHeader(), listResponseJson, HttpStatusCode.OK); - } - else if (req.Url.FullUri.Contains("/releases/")) - { - return new HttpResponse(req, new HttpHeader(), releaseResponseJson, HttpStatusCode.OK); - } - else - { - return new HttpResponse(req, new HttpHeader(), "", HttpStatusCode.NotFound); - } - }); - - var releases = Subject.Fetch(); - - // Debug output to see what happened - System.Console.WriteLine($"List called: {listCalled}, URL: {actualListUrl}"); - System.Console.WriteLine($"Release called: {releaseCalled}, URL: {actualReleaseUrl}"); - System.Console.WriteLine($"Releases count: {releases.Count}"); - for (var i = 0; i < releases.Count; i++) - { - System.Console.WriteLine($"Release {i}: Artist='{releases[i].Artist}', Album='{releases[i].Album}'"); - } - - releases.Should().HaveCount(1); - releases.First().Artist.Should().Be("Josh Wink"); - releases.First().Album.Should().Be("Profound Sounds Vol. 1"); + items.Should().HaveCount(1); + items.First().Artist.Should().Be("Josh Wink"); + items.First().Album.Should().Be("Profound Sounds Vol. 1"); } [Test] - public void should_handle_empty_list_response() + public void should_ignore_non_release_items() { - var responseJson = @"{""items"": []}"; - - Mocker.GetMock() - .Setup(s => s.Execute(It.IsAny())) - .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/lists/123456"), new HttpHeader(), responseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().BeEmpty(); - } - - [Test] - public void debug_test_to_see_what_url_is_called() - { - var responseJson = @"{""items"": []}"; - var actualUrl = ""; - - Mocker.GetMock() - .Setup(s => s.Execute(It.IsAny())) - .Callback(req => actualUrl = req.Url.FullUri) - .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/lists/123456"), new HttpHeader(), responseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - // This will show us what URL is actually being called - actualUrl.Should().NotBeEmpty(); - } - - [Test] - public void should_skip_non_release_items() - { - var responseJson = @"{ + var response = BuildListResponse(@"{ ""items"": [ { ""type"": ""label"", - ""id"": 1, - ""display_title"": ""Some Label"", - ""resource_url"": ""https://api.discogs.com/labels/1"" - }, - { - ""type"": ""release"", - ""id"": 3, - ""display_title"": ""Josh Wink - Profound Sounds Vol. 1"", - ""resource_url"": ""https://api.discogs.com/releases/3"" + ""id"": 7, + ""display_title"": ""Ignore me"", + ""resource_url"": ""https://api.discogs.com/labels/7"" } ] - }"; + }"); - var releaseResponseJson = @"{ - ""title"": ""Profound Sounds Vol. 1"", - ""artists"": [ - { - ""name"": ""Josh Wink"", - ""id"": 3 - } - ] - }"; + var items = _parser.ParseResponse(response); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), responseJson, HttpStatusCode.OK)); - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().HaveCount(1); - releases.First().Artist.Should().Be("Josh Wink"); + items.Should().BeEmpty(); + _httpClient.Verify(c => c.Execute(It.IsAny()), Times.Never); } [Test] - public void should_skip_items_when_release_fetch_fails() + public void should_skip_release_when_details_fail() { - var listResponseJson = @"{ + var response = BuildListResponse(@"{ ""items"": [ { ""type"": ""release"", @@ -191,93 +85,49 @@ public void should_skip_items_when_release_fetch_fails() ""resource_url"": ""https://api.discogs.com/releases/3"" } ] - }"; + }"); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), listResponseJson, HttpStatusCode.OK)); + _httpClient.Setup(c => c.Execute(It.IsAny())) + .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/releases/3"), _defaultHeaders, string.Empty, HttpStatusCode.NotFound)); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), "", HttpStatusCode.NotFound)); + var items = _parser.ParseResponse(response); - var releases = Subject.Fetch(); - - releases.Should().BeEmpty(); + items.Should().BeEmpty(); } [Test] - public void should_skip_releases_with_no_artists() + public void should_throw_when_discogs_returns_html() { - var listResponseJson = @"{ - ""items"": [ - { - ""type"": ""release"", - ""id"": 3, - ""display_title"": ""Various - Compilation"", - ""resource_url"": ""https://api.discogs.com/releases/3"" - } - ] - }"; + var response = BuildListResponse("", contentType: "text/html"); - var releaseResponseJson = @"{ - ""title"": ""Compilation"", - ""artists"": [] - }"; - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), listResponseJson, HttpStatusCode.OK)); - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().BeEmpty(); + _parser.Invoking(p => p.ParseResponse(response)) + .Should().Throw() + .WithMessage("*HTML content*"); } - [Test] - public void should_use_first_artist_when_multiple_artists() + private ImportListResponse BuildListResponse(string content, HttpStatusCode statusCode = HttpStatusCode.OK, string contentType = "application/json") { - var listResponseJson = @"{ - ""items"": [ - { - ""type"": ""release"", - ""id"": 3, - ""display_title"": ""Artist 1 & Artist 2 - Collaboration"", - ""resource_url"": ""https://api.discogs.com/releases/3"" - } - ] - }"; + var httpRequest = new HttpRequest("https://api.discogs.com/lists/123"); + var importListRequest = new ImportListRequest(httpRequest); + var headers = new HttpHeader { ContentType = contentType }; + var httpResponse = new HttpResponse(httpRequest, headers, content, statusCode); - var releaseResponseJson = @"{ - ""title"": ""Collaboration"", + return new ImportListResponse(importListRequest, httpResponse); + } + + private void GivenReleaseDetails(string resourceUrl, string payload, HttpStatusCode statusCode = HttpStatusCode.OK) + { + _httpClient.Setup(c => c.Execute(It.Is(r => r.Url.FullUri == resourceUrl))) + .Returns(new HttpResponse(new HttpRequest(resourceUrl), _defaultHeaders, payload, statusCode)); + } + + private static string BuildReleaseResponse(string artist, string title) + { + return $@"{{ + ""title"": ""{title}"", ""artists"": [ - { - ""name"": ""Artist 1"", - ""id"": 1 - }, - { - ""name"": ""Artist 2"", - ""id"": 2 - } + {{ ""name"": ""{artist}"", ""id"": 3 }} ] - }"; - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/lists/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), listResponseJson, HttpStatusCode.OK)); - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().HaveCount(1); - releases.First().Artist.Should().Be("Artist 1"); - releases.First().Album.Should().Be("Collaboration"); + }}"; } } diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs index af620c276..0874c6e93 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs @@ -4,34 +4,35 @@ using Moq; using NUnit.Framework; using NzbDrone.Common.Http; +using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Discogs; -using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ImportLists.Exceptions; namespace NzbDrone.Core.Test.ImportListTests.Discogs; [TestFixture] -public class DiscogsWantlistFixture : CoreTest +public class DiscogsWantlistFixture { + private readonly HttpHeader _defaultHeaders = new () { ContentType = "application/json" }; + + private DiscogsWantlistParser _parser; + private Mock _httpClient; + private DiscogsWantlistSettings _settings; + [SetUp] - public void Setup() + public void SetUp() { - Subject.Definition = new ImportLists.ImportListDefinition - { - Id = 1, - Name = "Test Discogs Wantlist", - Settings = new DiscogsWantlistSettings - { - Token = "test_token_123456", - Username = "test_user", - BaseUrl = "https://api.discogs.com" - } - }; + _httpClient = new Mock(); + _parser = new DiscogsWantlistParser(); + _settings = new DiscogsWantlistSettings { Token = "token", Username = "user", BaseUrl = "https://api.discogs.com" }; + + _parser.SetContext(_httpClient.Object, _settings); } [Test] - public void should_parse_valid_wantlist_response() + public void should_parse_wantlist_items() { - var wantlistResponseJson = @"{ + var response = BuildWantlistResponse(@"{ ""wants"": [ { ""basic_information"": { @@ -39,194 +40,98 @@ public void should_parse_valid_wantlist_response() ""title"": ""Profound Sounds Vol. 1"", ""resource_url"": ""https://api.discogs.com/releases/3"", ""artists"": [ - { - ""name"": ""Josh Wink"", - ""id"": 3 - } + { ""name"": ""Josh Wink"", ""id"": 123 } ] } } ] - }"; + }"); - var releaseResponseJson = @"{ - ""title"": ""Profound Sounds Vol. 1"", - ""artists"": [ - { - ""name"": ""Josh Wink"", - ""id"": 3 - } - ] - }"; + var items = _parser.ParseResponse(response); - var wantlistCalled = false; - var releaseCalled = false; - var actualWantlistUrl = ""; - var actualReleaseUrl = ""; - - Mocker.GetMock() - .Setup(s => s.Execute(It.IsAny())) - .Callback(req => - { - if (req.Url.FullUri.Contains("/wants")) - { - wantlistCalled = true; - actualWantlistUrl = req.Url.FullUri; - } - else if (req.Url.FullUri.Contains("/releases/")) - { - releaseCalled = true; - actualReleaseUrl = req.Url.FullUri; - } - }) - .Returns(req => - { - if (req.Url.FullUri.Contains("/wants")) - { - return new HttpResponse(req, new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK); - } - else if (req.Url.FullUri.Contains("/releases/")) - { - return new HttpResponse(req, new HttpHeader(), releaseResponseJson, HttpStatusCode.OK); - } - else - { - return new HttpResponse(req, new HttpHeader(), "", HttpStatusCode.NotFound); - } - }); - - var releases = Subject.Fetch(); - - // Debug output to see what happened - System.Console.WriteLine($"Wantlist called: {wantlistCalled}, URL: {actualWantlistUrl}"); - System.Console.WriteLine($"Release called: {releaseCalled}, URL: {actualReleaseUrl}"); - System.Console.WriteLine($"Releases count: {releases.Count}"); - for (var i = 0; i < releases.Count; i++) - { - System.Console.WriteLine($"Release {i}: Artist='{releases[i].Artist}', Album='{releases[i].Album}'"); - } - - releases.Should().HaveCount(1); - releases.First().Artist.Should().Be("Josh Wink"); - releases.First().Album.Should().Be("Profound Sounds Vol. 1"); + items.Should().HaveCount(1); + items.First().Artist.Should().Be("Josh Wink"); + items.First().Album.Should().Be("Profound Sounds Vol. 1"); + _httpClient.Verify(c => c.Execute(It.IsAny()), Times.Never); } [Test] - public void should_handle_empty_wantlist_response() + public void should_skip_entries_without_artists() { - var responseJson = @"{""wants"": []}"; - - Mocker.GetMock() - .Setup(s => s.Execute(It.IsAny())) - .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/users/test_user/wants"), new HttpHeader(), responseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().BeEmpty(); - } - - [Test] - public void should_skip_items_when_release_fetch_fails() - { - var wantlistResponseJson = @"{ + var response = BuildWantlistResponse(@"{ ""wants"": [ { ""basic_information"": { ""id"": 3, - ""title"": ""Profound Sounds Vol. 1"", - ""resource_url"": ""https://api.discogs.com/releases/3"" + ""title"": ""Missing artists"" } } ] - }"; + }"); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + var items = _parser.ParseResponse(response); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), "", HttpStatusCode.NotFound)); - - var releases = Subject.Fetch(); - - releases.Should().BeEmpty(); + items.Should().BeEmpty(); + _httpClient.Verify(c => c.Execute(It.IsAny()), Times.Never); } [Test] - public void should_skip_releases_with_no_artists() + public void should_skip_entries_without_title() { - var wantlistResponseJson = @"{ + var response = BuildWantlistResponse(@"{ ""wants"": [ { ""basic_information"": { ""id"": 3, - ""title"": ""Compilation"", - ""resource_url"": ""https://api.discogs.com/releases/3"" + ""artists"": [ + { ""name"": ""Test Artist"", ""id"": 123 } + ] } } ] - }"; + }"); - var releaseResponseJson = @"{ - ""title"": ""Compilation"", - ""artists"": [] - }"; + var items = _parser.ParseResponse(response); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().BeEmpty(); + items.Should().BeEmpty(); + _httpClient.Verify(c => c.Execute(It.IsAny()), Times.Never); } [Test] - public void should_use_first_artist_when_multiple_artists() + public void should_skip_entries_with_null_basic_information() { - var wantlistResponseJson = @"{ + var response = BuildWantlistResponse(@"{ ""wants"": [ { - ""basic_information"": { - ""id"": 3, - ""title"": ""Collaboration"", - ""resource_url"": ""https://api.discogs.com/releases/3"" - } + ""basic_information"": null } ] - }"; + }"); - var releaseResponseJson = @"{ - ""title"": ""Collaboration"", - ""artists"": [ - { - ""name"": ""Artist 1"", - ""id"": 1 - }, - { - ""name"": ""Artist 2"", - ""id"": 2 - } - ] - }"; + var items = _parser.ParseResponse(response); - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); - - Mocker.GetMock() - .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) - .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); - - var releases = Subject.Fetch(); - - releases.Should().HaveCount(1); - releases.First().Artist.Should().Be("Artist 1"); - releases.First().Album.Should().Be("Collaboration"); + items.Should().BeEmpty(); + _httpClient.Verify(c => c.Execute(It.IsAny()), Times.Never); } -} \ No newline at end of file + + [Test] + public void should_throw_when_discogs_returns_html() + { + var response = BuildWantlistResponse("", contentType: "text/html"); + + _parser.Invoking(p => p.ParseResponse(response)) + .Should().Throw() + .WithMessage("*HTML content*"); + } + + private ImportListResponse BuildWantlistResponse(string content, HttpStatusCode statusCode = HttpStatusCode.OK, string contentType = "application/json") + { + var httpRequest = new HttpRequest("https://api.discogs.com/users/user/wants"); + var importListRequest = new ImportListRequest(httpRequest); + var headers = new HttpHeader { ContentType = contentType }; + var httpResponse = new HttpResponse(httpRequest, headers, content, statusCode); + + return new ImportListResponse(importListRequest, httpResponse); + } + +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs index 38261c82e..27fbe7a66 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -1,18 +1,15 @@ using System.Collections.Generic; using System.Linq; -using System.Net; using Newtonsoft.Json; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; -using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.ImportLists.Discogs; public class DiscogsListsParser : IParseImportListResponse { - private ImportListResponse _importListResponse; private IHttpClient _httpClient; private DiscogsListsSettings _settings; @@ -28,94 +25,52 @@ public void SetContext(IHttpClient httpClient, DiscogsListsSettings settings) public IList ParseResponse(ImportListResponse importListResponse) { - _importListResponse = importListResponse; + DiscogsParserHelper.EnsureValidResponse(importListResponse, + "Discogs API responded with HTML content. List may be too large or API may be unavailable."); - var items = new List(); - - if (!PreProcess(_importListResponse)) - { - return items; - } - - var jsonResponse = Json.Deserialize(_importListResponse.Content); + var jsonResponse = Json.Deserialize(importListResponse.Content); if (jsonResponse?.Items == null) { - return items; + return new List(); } - foreach (var item in jsonResponse.Items) + var items = new List(); + + foreach (var resourceUrl in jsonResponse.Items.Where(IsReleaseItem).Select(item => item.ResourceUrl)) { - if (item?.Type == "release" && item.ResourceUrl.IsNotNullOrWhiteSpace()) + var releaseInfo = TryFetchRelease(resourceUrl); + + if (releaseInfo != null) { - try - { - if (_httpClient != null && _settings != null) - { - var releaseInfo = FetchReleaseDetails(item.ResourceUrl); - if (releaseInfo != null) - { - items.Add(releaseInfo); - } - } - } - catch - { - // If we can't fetch release details, skip this item - continue; - } + items.Add(releaseInfo); } } return items; } - // Unfortunately discogs release details are nested in a given /release/N endpoint. - // We'll have to fetch each one to get proper details. - private ImportListItemInfo FetchReleaseDetails(string resourceUrl) + private static bool IsReleaseItem(DiscogsListItem item) { - var request = new HttpRequestBuilder(resourceUrl) - .SetHeader("Authorization", $"Discogs token={_settings.Token}") - .Build(); + return item?.Type == "release" && item.ResourceUrl.IsNotNullOrWhiteSpace(); + } - var response = _httpClient.Execute(request); - - if (response.StatusCode != HttpStatusCode.OK) + private ImportListItemInfo TryFetchRelease(string resourceUrl) + { + if (_httpClient == null || _settings == null) { return null; } - var releaseResponse = Json.Deserialize(response.Content); - - if (releaseResponse?.Artists?.Any() == true && releaseResponse.Title.IsNotNullOrWhiteSpace()) + try { - return new ImportListItemInfo - { - Artist = releaseResponse.Artists.First().Name, - Album = releaseResponse.Title - }; + return DiscogsParserHelper.FetchReleaseDetails(_httpClient, _settings.Token, resourceUrl); } - - return null; - } - - protected virtual bool PreProcess(ImportListResponse importListResponse) - { - if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + catch { - throw new ImportListException(importListResponse, - "Discogs API call resulted in an unexpected StatusCode [{0}]", - importListResponse.HttpResponse.StatusCode); + // If we can't fetch release details, skip this item + return null; } - - if (importListResponse.HttpResponse.Headers.ContentType != null && - importListResponse.HttpResponse.Headers.ContentType.Contains("text/html")) - { - throw new ImportListException(importListResponse, - "Discogs API responded with HTML content. List may be too large or API may be unavailable."); - } - - return true; } } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs new file mode 100644 index 000000000..e16a2750a --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs @@ -0,0 +1,55 @@ +using System.Linq; +using System.Net; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists.Discogs; + +internal static class DiscogsParserHelper +{ + public static ImportListItemInfo FetchReleaseDetails(IHttpClient httpClient, string token, string resourceUrl) + { + var request = new HttpRequestBuilder(resourceUrl) + .SetHeader("Authorization", $"Discogs token={token}") + .Build(); + + var response = httpClient.Execute(request); + + if (response.StatusCode != HttpStatusCode.OK) + { + return null; + } + + var releaseResponse = Json.Deserialize(response.Content); + + if (releaseResponse?.Artists?.Any() != true || releaseResponse.Title.IsNullOrWhiteSpace()) + { + return null; + } + + return new ImportListItemInfo + { + Artist = releaseResponse.Artists.First().Name, + Album = releaseResponse.Title + }; + } + + public static void EnsureValidResponse(ImportListResponse importListResponse, string htmlContentMessage) + { + if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(importListResponse, + "Discogs API call resulted in an unexpected StatusCode [{0}]", + importListResponse.HttpResponse.StatusCode); + } + + if (importListResponse.HttpResponse.Headers.ContentType != null && + importListResponse.HttpResponse.Headers.ContentType.Contains("text/html")) + { + throw new ImportListException(importListResponse, htmlContentMessage); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs index b17068824..85ccb939f 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs @@ -1,21 +1,15 @@ using System.Collections.Generic; using System.Linq; -using System.Net; using Newtonsoft.Json; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; -using NzbDrone.Core.ImportLists.Exceptions; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.ImportLists.Discogs; public class DiscogsWantlistParser : IParseImportListResponse { - private ImportListResponse _importListResponse; - private IHttpClient _httpClient; - private DiscogsWantlistSettings _settings; - public DiscogsWantlistParser() { } @@ -28,95 +22,43 @@ public void SetContext(IHttpClient httpClient, DiscogsWantlistSettings settings) public IList ParseResponse(ImportListResponse importListResponse) { - _importListResponse = importListResponse; + DiscogsParserHelper.EnsureValidResponse(importListResponse, + "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable."); - var items = new List(); - - if (!PreProcess(_importListResponse)) - { - return items; - } - - var jsonResponse = Json.Deserialize(_importListResponse.Content); + var jsonResponse = Json.Deserialize(importListResponse.Content); if (jsonResponse?.Wants == null) { - return items; + return new List(); } + var items = new List(); + foreach (var want in jsonResponse.Wants) { - if (want?.BasicInformation != null) + var basicInfo = want?.BasicInformation; + + if (basicInfo == null) { - try - { - if (_httpClient != null && _settings != null) - { - var releaseInfo = FetchReleaseDetails(want.BasicInformation.ResourceUrl); - if (releaseInfo != null) - { - items.Add(releaseInfo); - } - } - } - catch - { - // If we can't fetch release details, skip this item - continue; - } + continue; } + + // The wantlist API includes artists and title in basic_information, so no need to fetch release details + // If you want is artists.First().Name and title, then fetching the release details is redundant according to their API. + if (basicInfo.Artists?.Any() != true || basicInfo.Title.IsNullOrWhiteSpace()) + { + continue; + } + + items.Add(new ImportListItemInfo + { + Artist = basicInfo.Artists.First().Name, + Album = basicInfo.Title + }); } return items; } - - // Unfortunately discogs release details are nested in a given /release/N endpoint. - // We'll have to fetch each one to get proper details. - private ImportListItemInfo FetchReleaseDetails(string resourceUrl) - { - var request = new HttpRequestBuilder(resourceUrl) - .SetHeader("Authorization", $"Discogs token={_settings.Token}") - .Build(); - - var response = _httpClient.Execute(request); - - if (response.StatusCode != HttpStatusCode.OK) - { - return null; - } - - var releaseResponse = Json.Deserialize(response.Content); - - if (releaseResponse?.Artists?.Any() == true && releaseResponse.Title.IsNotNullOrWhiteSpace()) - { - return new ImportListItemInfo - { - Artist = releaseResponse.Artists.First().Name, - Album = releaseResponse.Title - }; - } - - return null; - } - - protected virtual bool PreProcess(ImportListResponse importListResponse) - { - if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) - { - throw new ImportListException(importListResponse, - "Discogs API call resulted in an unexpected StatusCode [{0}]", - importListResponse.HttpResponse.StatusCode); - } - - if (importListResponse.HttpResponse.Headers.ContentType != null && - importListResponse.HttpResponse.Headers.ContentType.Contains("text/html")) - { - throw new ImportListException(importListResponse, - "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable."); - } - - return true; - } } public class DiscogsWantlistResponse From a39512cf45fe7e4dded069d4898cd88ed3d2d42c Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 12:47:15 -0500 Subject: [PATCH 04/14] Undo extra gitignore --- .gitignore | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.gitignore b/.gitignore index cba2cec00..a5d6bb7c8 100644 --- a/.gitignore +++ b/.gitignore @@ -167,15 +167,3 @@ node_modules.nosync # Ignore Jetbrains IntelliJ Workspace Directories .idea/ - -# Ignore Claude -**/.claude - -# Ignore markdown files (except essential tracked ones) -*.md -!README.md -!CONTRIBUTING.md -!SECURITY.md -!LICENSE.md -!CODE_OF_CONDUCT.md -!CLA.md \ No newline at end of file From 9792c298d92ba6bb202463c1ab0590770720b6b7 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 12:55:23 -0500 Subject: [PATCH 05/14] Add logging on list parser errors --- src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs | 2 +- .../ImportLists/Discogs/DiscogsListsParser.cs | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs index e2c950a76..1a9c31cf3 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs @@ -31,7 +31,7 @@ public override IImportListRequestGenerator GetRequestGenerator() public override IParseImportListResponse GetParser() { var parser = new DiscogsListsParser(); - parser.SetContext(_httpClient, Settings); + parser.SetContext(_httpClient, Settings, _logger); return parser; } } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs index 27fbe7a66..479fe8008 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -1,5 +1,7 @@ +using System; using System.Collections.Generic; using System.Linq; +using NLog; using Newtonsoft.Json; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; @@ -12,15 +14,17 @@ public class DiscogsListsParser : IParseImportListResponse { private IHttpClient _httpClient; private DiscogsListsSettings _settings; + private Logger _logger; public DiscogsListsParser() { } - public void SetContext(IHttpClient httpClient, DiscogsListsSettings settings) + public void SetContext(IHttpClient httpClient, DiscogsListsSettings settings, Logger logger = null) { _httpClient = httpClient; _settings = settings; + _logger = logger ?? LogManager.GetCurrentClassLogger(); } public IList ParseResponse(ImportListResponse importListResponse) @@ -66,9 +70,9 @@ private ImportListItemInfo TryFetchRelease(string resourceUrl) { return DiscogsParserHelper.FetchReleaseDetails(_httpClient, _settings.Token, resourceUrl); } - catch + catch (Exception ex) { - // If we can't fetch release details, skip this item + _logger?.Error(ex, "Failed to fetch release details from Discogs API for resource URL: {0}. Skipping this item.", resourceUrl); return null; } } From 245a23946d192925ea12e6b704a571075a9b30ed Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 12:56:05 -0500 Subject: [PATCH 06/14] Update src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs Co-authored-by: Bogdan --- src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs index 1a9c31cf3..656efcc1f 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs @@ -11,8 +11,7 @@ public class DiscogsLists : HttpImportListBase public override string Name => "Discogs Lists"; public override ImportListType ListType => ImportListType.Discogs; public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); - public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases - public override int PageSize => 0; // Discogs doesn't support pagination for lists + public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); public DiscogsLists(IHttpClient httpClient, IImportListStatusService importListStatusService, From f8dbad254ff3ef27759b322f2f97d0300ebbd745 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:02:13 -0500 Subject: [PATCH 07/14] move API DTOs to separate file --- .../ImportLists/Discogs/DiscogsApi.cs | 52 +++++++++++++++++++ .../ImportLists/Discogs/DiscogsListsParser.cs | 29 ----------- .../Discogs/DiscogsWantlistParser.cs | 24 ++------- 3 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs new file mode 100644 index 000000000..f9a1b6c65 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs @@ -0,0 +1,52 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NzbDrone.Core.ImportLists.Discogs; + +public class DiscogsListResponse +{ + public List Items { get; set; } +} + +public class DiscogsListItem +{ + public string Type { get; set; } + public int Id { get; set; } + [JsonProperty("display_title")] + public string DisplayTitle { get; set; } + [JsonProperty("resource_url")] + public string ResourceUrl { get; set; } + public string Uri { get; set; } +} + +public class DiscogsReleaseResponse +{ + public string Title { get; set; } + public List Artists { get; set; } +} + +public class DiscogsReleaseArtist +{ + public string Name { get; set; } + public int Id { get; set; } +} + +public class DiscogsWantlistResponse +{ + public List Wants { get; set; } +} + +public class DiscogsWantlistItem +{ + [JsonProperty("basic_information")] + public DiscogsBasicInformation BasicInformation { get; set; } +} + +public class DiscogsBasicInformation +{ + public int Id { get; set; } + public string Title { get; set; } + [JsonProperty("resource_url")] + public string ResourceUrl { get; set; } + public List Artists { get; set; } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs index 479fe8008..0e6a960f1 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using NLog; -using Newtonsoft.Json; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; @@ -77,31 +76,3 @@ private ImportListItemInfo TryFetchRelease(string resourceUrl) } } } - -public class DiscogsListResponse -{ - public List Items { get; set; } -} - -public class DiscogsListItem -{ - public string Type { get; set; } - public int Id { get; set; } - [JsonProperty("display_title")] - public string DisplayTitle { get; set; } - [JsonProperty("resource_url")] - public string ResourceUrl { get; set; } - public string Uri { get; set; } -} - -public class DiscogsReleaseResponse -{ - public string Title { get; set; } - public List Artists { get; set; } -} - -public class DiscogsReleaseArtist -{ - public string Name { get; set; } - public int Id { get; set; } -} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs index 85ccb939f..415e64c2b 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; using System.Linq; -using Newtonsoft.Json; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Common.Serializer; @@ -10,6 +9,9 @@ namespace NzbDrone.Core.ImportLists.Discogs; public class DiscogsWantlistParser : IParseImportListResponse { + private IHttpClient _httpClient; + private DiscogsWantlistSettings _settings; + public DiscogsWantlistParser() { } @@ -60,23 +62,3 @@ public IList ParseResponse(ImportListResponse importListResp return items; } } - -public class DiscogsWantlistResponse -{ - public List Wants { get; set; } -} - -public class DiscogsWantlistItem -{ - [JsonProperty("basic_information")] - public DiscogsBasicInformation BasicInformation { get; set; } -} - -public class DiscogsBasicInformation -{ - public int Id { get; set; } - public string Title { get; set; } - [JsonProperty("resource_url")] - public string ResourceUrl { get; set; } - public List Artists { get; set; } -} From c6f14c397579ea78038ccb0454120dcd6ecbb59f Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:01:46 -0500 Subject: [PATCH 08/14] Update src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs Co-authored-by: Bogdan --- src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs index 656efcc1f..5fdecd611 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs @@ -24,14 +24,12 @@ public DiscogsLists(IHttpClient httpClient, public override IImportListRequestGenerator GetRequestGenerator() { - return new DiscogsListsRequestGenerator { Settings = Settings }; + return new DiscogsListsRequestGenerator(Settings); } public override IParseImportListResponse GetParser() { - var parser = new DiscogsListsParser(); - parser.SetContext(_httpClient, Settings, _logger); - return parser; + return new DiscogsListsParser(Settings, _httpClient, _logger); } } } From 55c9b5428db237d1ee33577a52ff99d366fd6d09 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:02:12 -0500 Subject: [PATCH 09/14] Update src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs Co-authored-by: Bogdan --- .../Discogs/DiscogsListsRequestGenerator.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs index 6d5bb14f3..97eb139b1 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsRequestGenerator.cs @@ -5,29 +5,26 @@ namespace NzbDrone.Core.ImportLists.Discogs { public class DiscogsListsRequestGenerator : IImportListRequestGenerator { - public DiscogsListsSettings Settings { get; set; } + private readonly DiscogsListsSettings _settings; - public int MaxPages { get; set; } - public int PageSize { get; set; } - - public DiscogsListsRequestGenerator() + public DiscogsListsRequestGenerator(DiscogsListsSettings settings) { - MaxPages = 1; - PageSize = 0; // Discogs doesn't support pagination for lists currently + _settings = settings; } public virtual ImportListPageableRequestChain GetListItems() { var pageableRequests = new ImportListPageableRequestChain(); pageableRequests.Add(GetPagedRequests()); + return pageableRequests; } private IEnumerable GetPagedRequests() { - var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/')) - .Resource($"/lists/{Settings.ListId}") - .SetHeader("Authorization", $"Discogs token={Settings.Token}") + var request = new HttpRequestBuilder(_settings.BaseUrl.TrimEnd('/')) + .Resource($"/lists/{_settings.ListId}") + .SetHeader("Authorization", $"Discogs token={_settings.Token}") .Build(); yield return new ImportListRequest(request); From 4326ecf7d20be31f9652f3ef912a1154b1c6b48c Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 13:02:47 -0500 Subject: [PATCH 10/14] Update src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs Co-authored-by: Bogdan --- src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs index 68c2dc8ca..ab3b5771d 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsSettings.cs @@ -15,7 +15,7 @@ public DiscogsListsSettingsValidator() public class DiscogsListsSettings : IImportListSettings { - private static readonly DiscogsListsSettingsValidator Validator = new DiscogsListsSettingsValidator(); + private static readonly DiscogsListsSettingsValidator Validator = new (); public DiscogsListsSettings() { From c19643e321ae2d3fc11f5dcf279e3c9ad81a2dbd Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 16:17:03 -0500 Subject: [PATCH 11/14] More cleanup and DI --- .../Discogs/DiscogsListsFixture.cs | 7 +- .../Discogs/DiscogsWantlistFixture.cs | 5 +- .../ImportLists/Discogs/DiscogsListsParser.cs | 68 ++++++++----------- .../ImportLists/Discogs/DiscogsWantlist.cs | 4 +- .../Discogs/DiscogsWantlistParser.cs | 33 +++++---- 5 files changed, 55 insertions(+), 62 deletions(-) diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs index 98769a012..f1fc83181 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs @@ -2,6 +2,7 @@ using System.Net; using FluentAssertions; using Moq; +using NLog; using NUnit.Framework; using NzbDrone.Common.Http; using NzbDrone.Core.ImportLists; @@ -18,15 +19,15 @@ public class DiscogsListsFixture private DiscogsListsParser _parser; private Mock _httpClient; private DiscogsListsSettings _settings; + private Logger _logger; [SetUp] public void SetUp() { _httpClient = new Mock(); - _parser = new DiscogsListsParser(); _settings = new DiscogsListsSettings { Token = "token", ListId = "123", BaseUrl = "https://api.discogs.com" }; - - _parser.SetContext(_httpClient.Object, _settings); + _logger = LogManager.GetCurrentClassLogger(); + _parser = new DiscogsListsParser(_settings, _httpClient.Object, _logger); } [Test] diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs index 0874c6e93..0e54e6e71 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs @@ -23,10 +23,8 @@ public class DiscogsWantlistFixture public void SetUp() { _httpClient = new Mock(); - _parser = new DiscogsWantlistParser(); _settings = new DiscogsWantlistSettings { Token = "token", Username = "user", BaseUrl = "https://api.discogs.com" }; - - _parser.SetContext(_httpClient.Object, _settings); + _parser = new DiscogsWantlistParser(_settings, _httpClient.Object); } [Test] @@ -133,5 +131,4 @@ private ImportListResponse BuildWantlistResponse(string content, HttpStatusCode return new ImportListResponse(importListRequest, httpResponse); } - } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs index 0e6a960f1..79f10526b 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; @@ -11,68 +10,61 @@ namespace NzbDrone.Core.ImportLists.Discogs; public class DiscogsListsParser : IParseImportListResponse { - private IHttpClient _httpClient; - private DiscogsListsSettings _settings; - private Logger _logger; + private readonly DiscogsListsSettings _settings; + private readonly IHttpClient _httpClient; + private readonly Logger _logger; - public DiscogsListsParser() + public DiscogsListsParser(DiscogsListsSettings settings, IHttpClient httpClient, Logger logger) { - } - - public void SetContext(IHttpClient httpClient, DiscogsListsSettings settings, Logger logger = null) - { - _httpClient = httpClient; _settings = settings; - _logger = logger ?? LogManager.GetCurrentClassLogger(); + _httpClient = httpClient; + _logger = logger; } public IList ParseResponse(ImportListResponse importListResponse) { - DiscogsParserHelper.EnsureValidResponse(importListResponse, - "Discogs API responded with HTML content. List may be too large or API may be unavailable."); + var items = new List(); + + if (!PreProcess(importListResponse)) + { + return items; + } var jsonResponse = Json.Deserialize(importListResponse.Content); if (jsonResponse?.Items == null) { - return new List(); + return items; } - var items = new List(); - - foreach (var resourceUrl in jsonResponse.Items.Where(IsReleaseItem).Select(item => item.ResourceUrl)) + foreach (var item in jsonResponse.Items) { - var releaseInfo = TryFetchRelease(resourceUrl); - - if (releaseInfo != null) + if (item.Type == "release" && item.ResourceUrl.IsNotNullOrWhiteSpace()) { - items.Add(releaseInfo); + try + { + var releaseInfo = FetchReleaseDetails(item.ResourceUrl); + items.AddIfNotNull(releaseInfo); + } + catch (Exception ex) + { + _logger.Error(ex, "Discogs release details API call resulted in an unexpected exception"); + } } } return items; } - private static bool IsReleaseItem(DiscogsListItem item) + private bool PreProcess(ImportListResponse importListResponse) { - return item?.Type == "release" && item.ResourceUrl.IsNotNullOrWhiteSpace(); + DiscogsParserHelper.EnsureValidResponse(importListResponse, + "Discogs API responded with HTML content. List may be too large or API may be unavailable."); + return true; } - private ImportListItemInfo TryFetchRelease(string resourceUrl) + private ImportListItemInfo FetchReleaseDetails(string resourceUrl) { - if (_httpClient == null || _settings == null) - { - return null; - } - - try - { - return DiscogsParserHelper.FetchReleaseDetails(_httpClient, _settings.Token, resourceUrl); - } - catch (Exception ex) - { - _logger?.Error(ex, "Failed to fetch release details from Discogs API for resource URL: {0}. Skipping this item.", resourceUrl); - return null; - } + return DiscogsParserHelper.FetchReleaseDetails(_httpClient, _settings.Token, resourceUrl); } } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs index 6a1d138c9..e9358b82d 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs @@ -30,9 +30,7 @@ public override IImportListRequestGenerator GetRequestGenerator() public override IParseImportListResponse GetParser() { - var parser = new DiscogsWantlistParser(); - parser.SetContext(_httpClient, Settings); - return parser; + return new DiscogsWantlistParser(Settings, _httpClient); } } } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs index 415e64c2b..8074ea91f 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs @@ -9,33 +9,31 @@ namespace NzbDrone.Core.ImportLists.Discogs; public class DiscogsWantlistParser : IParseImportListResponse { - private IHttpClient _httpClient; - private DiscogsWantlistSettings _settings; + private readonly DiscogsWantlistSettings _settings; + private readonly IHttpClient _httpClient; - public DiscogsWantlistParser() + public DiscogsWantlistParser(DiscogsWantlistSettings settings, IHttpClient httpClient) { - } - - public void SetContext(IHttpClient httpClient, DiscogsWantlistSettings settings) - { - _httpClient = httpClient; _settings = settings; + _httpClient = httpClient; } public IList ParseResponse(ImportListResponse importListResponse) { - DiscogsParserHelper.EnsureValidResponse(importListResponse, - "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable."); + var items = new List(); + + if (!PreProcess(importListResponse)) + { + return items; + } var jsonResponse = Json.Deserialize(importListResponse.Content); if (jsonResponse?.Wants == null) { - return new List(); + return items; } - var items = new List(); - foreach (var want in jsonResponse.Wants) { var basicInfo = want?.BasicInformation; @@ -52,7 +50,7 @@ public IList ParseResponse(ImportListResponse importListResp continue; } - items.Add(new ImportListItemInfo + items.AddIfNotNull(new ImportListItemInfo { Artist = basicInfo.Artists.First().Name, Album = basicInfo.Title @@ -61,4 +59,11 @@ public IList ParseResponse(ImportListResponse importListResp return items; } + + private bool PreProcess(ImportListResponse importListResponse) + { + DiscogsParserHelper.EnsureValidResponse(importListResponse, + "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable."); + return true; + } } From 5122e8e2f312218270bc63ef154dd30c92048daa Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 16:21:57 -0500 Subject: [PATCH 12/14] Support pagination --- .../ImportLists/Discogs/DiscogsWantlist.cs | 2 +- .../DiscogsWantlistRequestGenerator.cs | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs index e9358b82d..7c80df13f 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs @@ -12,7 +12,7 @@ public class DiscogsWantlist : HttpImportListBase public override ImportListType ListType => ImportListType.Discogs; public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases - public override int PageSize => 0; // Discogs doesn't support pagination for wantlists currently + public override int PageSize => 50; // Discogs API supports pagination with page and per_page parameters public DiscogsWantlist(IHttpClient httpClient, IImportListStatusService importListStatusService, diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs index 6be4820e7..0e06ca64b 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs @@ -12,8 +12,8 @@ public class DiscogsWantlistRequestGenerator : IImportListRequestGenerator public DiscogsWantlistRequestGenerator() { - MaxPages = 1; - PageSize = 0; // Discogs doesn't support pagination for wantlists currently + MaxPages = 10; // Allow fetching up to 10 pages + PageSize = 50; // Discogs API supports pagination with page and per_page parameters (max 100 per page) } public virtual ImportListPageableRequestChain GetListItems() @@ -25,12 +25,17 @@ public virtual ImportListPageableRequestChain GetListItems() private IEnumerable GetPagedRequests() { - var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/')) - .Resource($"/users/{Settings.Username}/wants") - .SetHeader("Authorization", $"Discogs token={Settings.Token}") - .Build(); + for (var page = 1; page <= MaxPages; page++) + { + var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/')) + .Resource($"/users/{Settings.Username}/wants") + .AddQueryParam("page", page) + .AddQueryParam("per_page", PageSize) + .SetHeader("Authorization", $"Discogs token={Settings.Token}") + .Build(); - yield return new ImportListRequest(request); + yield return new ImportListRequest(request); + } } } } From d900890e6ba033e73151a64cf56ee0b22d18fc5a Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:28:06 -0500 Subject: [PATCH 13/14] Support artist type from discogs api --- .../Discogs/DiscogsListsFixture.cs | 118 +++++++++++++++++- .../ImportLists/Discogs/DiscogsApi.cs | 6 + .../ImportLists/Discogs/DiscogsListsParser.cs | 44 ++++--- .../Discogs/DiscogsParserHelper.cs | 27 ++++ 4 files changed, 177 insertions(+), 18 deletions(-) diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs index f1fc83181..390b4f57c 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsListsFixture.cs @@ -55,7 +55,65 @@ public void should_parse_release_items() } [Test] - public void should_ignore_non_release_items() + public void should_parse_artist_items() + { + const string resourceUrl = "https://api.discogs.com/artists/3227"; + GivenArtistDetails(resourceUrl, BuildArtistResponse("Silent Phase")); + + var response = BuildListResponse(@"{ + ""items"": [ + { + ""type"": ""artist"", + ""id"": 3227, + ""display_title"": ""Silent Phase"", + ""resource_url"": ""https://api.discogs.com/artists/3227"" + } + ] + }"); + + var items = _parser.ParseResponse(response); + + items.Should().HaveCount(1); + items.First().Artist.Should().Be("Silent Phase"); + items.First().Album.Should().BeNull(); + } + + [Test] + public void should_parse_mixed_release_and_artist_items() + { + const string releaseUrl = "https://api.discogs.com/releases/4674"; + const string artistUrl = "https://api.discogs.com/artists/3227"; + GivenReleaseDetails(releaseUrl, BuildReleaseResponse("Silent Phase", "The Rewired Mixes")); + GivenArtistDetails(artistUrl, BuildArtistResponse("Silent Phase")); + + var response = BuildListResponse(@"{ + ""items"": [ + { + ""type"": ""release"", + ""id"": 4674, + ""display_title"": ""Silent Phase - The Rewired Mixes"", + ""resource_url"": ""https://api.discogs.com/releases/4674"" + }, + { + ""type"": ""artist"", + ""id"": 3227, + ""display_title"": ""Silent Phase"", + ""resource_url"": ""https://api.discogs.com/artists/3227"" + } + ] + }"); + + var items = _parser.ParseResponse(response); + + items.Should().HaveCount(2); + items.First().Artist.Should().Be("Silent Phase"); + items.First().Album.Should().Be("The Rewired Mixes"); + items.Last().Artist.Should().Be("Silent Phase"); + items.Last().Album.Should().BeNull(); + } + + [Test] + public void should_ignore_non_release_and_non_artist_items() { var response = BuildListResponse(@"{ ""items"": [ @@ -96,6 +154,50 @@ public void should_skip_release_when_details_fail() items.Should().BeEmpty(); } + [Test] + public void should_skip_artist_when_details_fail() + { + var response = BuildListResponse(@"{ + ""items"": [ + { + ""type"": ""artist"", + ""id"": 3227, + ""display_title"": ""Silent Phase"", + ""resource_url"": ""https://api.discogs.com/artists/3227"" + } + ] + }"); + + _httpClient.Setup(c => c.Execute(It.IsAny())) + .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/artists/3227"), _defaultHeaders, string.Empty, HttpStatusCode.NotFound)); + + var items = _parser.ParseResponse(response); + + items.Should().BeEmpty(); + } + + [Test] + public void should_skip_artist_when_name_is_missing() + { + const string resourceUrl = "https://api.discogs.com/artists/3227"; + GivenArtistDetails(resourceUrl, @"{ ""id"": 3227 }"); + + var response = BuildListResponse(@"{ + ""items"": [ + { + ""type"": ""artist"", + ""id"": 3227, + ""display_title"": ""Silent Phase"", + ""resource_url"": ""https://api.discogs.com/artists/3227"" + } + ] + }"); + + var items = _parser.ParseResponse(response); + + items.Should().BeEmpty(); + } + [Test] public void should_throw_when_discogs_returns_html() { @@ -122,6 +224,12 @@ private void GivenReleaseDetails(string resourceUrl, string payload, HttpStatusC .Returns(new HttpResponse(new HttpRequest(resourceUrl), _defaultHeaders, payload, statusCode)); } + private void GivenArtistDetails(string resourceUrl, string payload, HttpStatusCode statusCode = HttpStatusCode.OK) + { + _httpClient.Setup(c => c.Execute(It.Is(r => r.Url.FullUri == resourceUrl))) + .Returns(new HttpResponse(new HttpRequest(resourceUrl), _defaultHeaders, payload, statusCode)); + } + private static string BuildReleaseResponse(string artist, string title) { return $@"{{ @@ -131,4 +239,12 @@ private static string BuildReleaseResponse(string artist, string title) ] }}"; } + + private static string BuildArtistResponse(string name) + { + return $@"{{ + ""name"": ""{name}"", + ""id"": 3227 + }}"; + } } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs index f9a1b6c65..a23e50cd5 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsApi.cs @@ -31,6 +31,12 @@ public class DiscogsReleaseArtist public int Id { get; set; } } +public class DiscogsArtistResponse +{ + public string Name { get; set; } + public int Id { get; set; } +} + public class DiscogsWantlistResponse { public List Wants { get; set; } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs index 79f10526b..4f36069b6 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -8,18 +8,11 @@ namespace NzbDrone.Core.ImportLists.Discogs; -public class DiscogsListsParser : IParseImportListResponse +public class DiscogsListsParser(DiscogsListsSettings settings, IHttpClient httpClient, Logger logger) : IParseImportListResponse { - private readonly DiscogsListsSettings _settings; - private readonly IHttpClient _httpClient; - private readonly Logger _logger; - - public DiscogsListsParser(DiscogsListsSettings settings, IHttpClient httpClient, Logger logger) - { - _settings = settings; - _httpClient = httpClient; - _logger = logger; - } + private readonly DiscogsListsSettings _settings = settings; + private readonly IHttpClient _httpClient = httpClient; + private readonly Logger _logger = logger; public IList ParseResponse(ImportListResponse importListResponse) { @@ -39,17 +32,29 @@ public IList ParseResponse(ImportListResponse importListResp foreach (var item in jsonResponse.Items) { - if (item.Type == "release" && item.ResourceUrl.IsNotNullOrWhiteSpace()) + if (item.ResourceUrl.IsNullOrWhiteSpace()) { - try + continue; + } + + try + { + ImportListItemInfo itemInfo = null; + + if (item.Type == "release") { - var releaseInfo = FetchReleaseDetails(item.ResourceUrl); - items.AddIfNotNull(releaseInfo); + itemInfo = FetchReleaseDetails(item.ResourceUrl); } - catch (Exception ex) + else if (item.Type == "artist") { - _logger.Error(ex, "Discogs release details API call resulted in an unexpected exception"); + itemInfo = FetchArtistDetails(item.ResourceUrl); } + + items.AddIfNotNull(itemInfo); + } + catch (Exception ex) + { + _logger.Error(ex, "Discogs API call resulted in an unexpected exception for {0} type item", item.Type ?? "unknown"); } } @@ -67,4 +72,9 @@ private ImportListItemInfo FetchReleaseDetails(string resourceUrl) { return DiscogsParserHelper.FetchReleaseDetails(_httpClient, _settings.Token, resourceUrl); } + + private ImportListItemInfo FetchArtistDetails(string resourceUrl) + { + return DiscogsParserHelper.FetchArtistDetails(_httpClient, _settings.Token, resourceUrl); + } } diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs index e16a2750a..3765aaf28 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsParserHelper.cs @@ -37,6 +37,33 @@ public static ImportListItemInfo FetchReleaseDetails(IHttpClient httpClient, str }; } + public static ImportListItemInfo FetchArtistDetails(IHttpClient httpClient, string token, string resourceUrl) + { + var request = new HttpRequestBuilder(resourceUrl) + .SetHeader("Authorization", $"Discogs token={token}") + .Build(); + + var response = httpClient.Execute(request); + + if (response.StatusCode != HttpStatusCode.OK) + { + return null; + } + + var artistResponse = Json.Deserialize(response.Content); + + if (artistResponse?.Name.IsNullOrWhiteSpace() == true) + { + return null; + } + + return new ImportListItemInfo + { + Artist = artistResponse.Name, + Album = null // Artists don't have a specific album, just the artist name + }; + } + public static void EnsureValidResponse(ImportListResponse importListResponse, string htmlContentMessage) { if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) From b526c76feeab24603478479a71f572251cdae165 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:41:55 -0500 Subject: [PATCH 14/14] Stylecop fixes --- src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs index 4f36069b6..3aa745187 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsListsParser.cs @@ -8,7 +8,8 @@ namespace NzbDrone.Core.ImportLists.Discogs; -public class DiscogsListsParser(DiscogsListsSettings settings, IHttpClient httpClient, Logger logger) : IParseImportListResponse +public class DiscogsListsParser(DiscogsListsSettings settings, IHttpClient httpClient, Logger logger) + : IParseImportListResponse { private readonly DiscogsListsSettings _settings = settings; private readonly IHttpClient _httpClient = httpClient;