From 16675ef44cc96c605a1898cc24730b30e1f87eb4 Mon Sep 17 00:00:00 2001 From: aglowinthefield <146008217+aglowinthefield@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:53:13 -0400 Subject: [PATCH] Wantlists --- .../Discogs/DiscogsWantlistFixture.cs | 232 ++++++++++++++++++ .../ImportLists/Discogs/DiscogsLists.cs | 2 +- .../ImportLists/Discogs/DiscogsWantlist.cs | 38 +++ .../Discogs/DiscogsWantlistParser.cs | 140 +++++++++++ .../DiscogsWantlistRequestGenerator.cs | 36 +++ .../Discogs/DiscogsWantlistSettings.cs | 38 +++ 6 files changed, 485 insertions(+), 1 deletion(-) create mode 100644 src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs create mode 100644 src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs diff --git a/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs new file mode 100644 index 000000000..af620c276 --- /dev/null +++ b/src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs @@ -0,0 +1,232 @@ +using System.Linq; +using System.Net; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.ImportLists.Discogs; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ImportListTests.Discogs; + +[TestFixture] +public class DiscogsWantlistFixture : CoreTest +{ + [SetUp] + public void Setup() + { + Subject.Definition = new ImportLists.ImportListDefinition + { + Id = 1, + Name = "Test Discogs Wantlist", + Settings = new DiscogsWantlistSettings + { + Token = "test_token_123456", + Username = "test_user", + BaseUrl = "https://api.discogs.com" + } + }; + } + + [Test] + public void should_parse_valid_wantlist_response() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"", + ""artists"": [ + { + ""name"": ""Josh Wink"", + ""id"": 3 + } + ] + } + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Profound Sounds Vol. 1"", + ""artists"": [ + { + ""name"": ""Josh Wink"", + ""id"": 3 + } + ] + }"; + + var wantlistCalled = false; + var releaseCalled = false; + var actualWantlistUrl = ""; + var actualReleaseUrl = ""; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Callback(req => + { + if (req.Url.FullUri.Contains("/wants")) + { + wantlistCalled = true; + actualWantlistUrl = req.Url.FullUri; + } + else if (req.Url.FullUri.Contains("/releases/")) + { + releaseCalled = true; + actualReleaseUrl = req.Url.FullUri; + } + }) + .Returns(req => + { + if (req.Url.FullUri.Contains("/wants")) + { + return new HttpResponse(req, new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK); + } + else if (req.Url.FullUri.Contains("/releases/")) + { + return new HttpResponse(req, new HttpHeader(), releaseResponseJson, HttpStatusCode.OK); + } + else + { + return new HttpResponse(req, new HttpHeader(), "", HttpStatusCode.NotFound); + } + }); + + var releases = Subject.Fetch(); + + // Debug output to see what happened + System.Console.WriteLine($"Wantlist called: {wantlistCalled}, URL: {actualWantlistUrl}"); + System.Console.WriteLine($"Release called: {releaseCalled}, URL: {actualReleaseUrl}"); + System.Console.WriteLine($"Releases count: {releases.Count}"); + for (var i = 0; i < releases.Count; i++) + { + System.Console.WriteLine($"Release {i}: Artist='{releases[i].Artist}', Album='{releases[i].Album}'"); + } + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Josh Wink"); + releases.First().Album.Should().Be("Profound Sounds Vol. 1"); + } + + [Test] + public void should_handle_empty_wantlist_response() + { + var responseJson = @"{""wants"": []}"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.IsAny())) + .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/users/test_user/wants"), new HttpHeader(), responseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_skip_items_when_release_fetch_fails() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Profound Sounds Vol. 1"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), "", HttpStatusCode.NotFound)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_skip_releases_with_no_artists() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Compilation"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Compilation"", + ""artists"": [] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().BeEmpty(); + } + + [Test] + public void should_use_first_artist_when_multiple_artists() + { + var wantlistResponseJson = @"{ + ""wants"": [ + { + ""basic_information"": { + ""id"": 3, + ""title"": ""Collaboration"", + ""resource_url"": ""https://api.discogs.com/releases/3"" + } + } + ] + }"; + + var releaseResponseJson = @"{ + ""title"": ""Collaboration"", + ""artists"": [ + { + ""name"": ""Artist 1"", + ""id"": 1 + }, + { + ""name"": ""Artist 2"", + ""id"": 2 + } + ] + }"; + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/wants")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK)); + + Mocker.GetMock() + .Setup(s => s.Execute(It.Is(r => r.Url.ToString().Contains("/releases/")))) + .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK)); + + var releases = Subject.Fetch(); + + releases.Should().HaveCount(1); + releases.First().Artist.Should().Be("Artist 1"); + releases.First().Album.Should().Be("Collaboration"); + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs index b5f311e70..e2c950a76 100644 --- a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs @@ -11,7 +11,7 @@ public class DiscogsLists : HttpImportListBase public override string Name => "Discogs Lists"; public override ImportListType ListType => ImportListType.Discogs; public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); - public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); // 60 requests per minute limit + public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases public override int PageSize => 0; // Discogs doesn't support pagination for lists public DiscogsLists(IHttpClient httpClient, diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs new file mode 100644 index 000000000..6a1d138c9 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs @@ -0,0 +1,38 @@ +using System; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsWantlist : HttpImportListBase + { + public override string Name => "Discogs Wantlist"; + public override ImportListType ListType => ImportListType.Discogs; + public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12); + public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases + public override int PageSize => 0; // Discogs doesn't support pagination for wantlists currently + + public DiscogsWantlist(IHttpClient httpClient, + IImportListStatusService importListStatusService, + IConfigService configService, + IParsingService parsingService, + Logger logger) + : base(httpClient, importListStatusService, configService, parsingService, logger) + { + } + + public override IImportListRequestGenerator GetRequestGenerator() + { + return new DiscogsWantlistRequestGenerator { Settings = Settings }; + } + + public override IParseImportListResponse GetParser() + { + var parser = new DiscogsWantlistParser(); + parser.SetContext(_httpClient, Settings); + return parser; + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs new file mode 100644 index 000000000..b17068824 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs @@ -0,0 +1,140 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using Newtonsoft.Json; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.ImportLists.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.ImportLists.Discogs; + +public class DiscogsWantlistParser : IParseImportListResponse +{ + private ImportListResponse _importListResponse; + private IHttpClient _httpClient; + private DiscogsWantlistSettings _settings; + + public DiscogsWantlistParser() + { + } + + public void SetContext(IHttpClient httpClient, DiscogsWantlistSettings settings) + { + _httpClient = httpClient; + _settings = settings; + } + + public IList ParseResponse(ImportListResponse importListResponse) + { + _importListResponse = importListResponse; + + var items = new List(); + + if (!PreProcess(_importListResponse)) + { + return items; + } + + var jsonResponse = Json.Deserialize(_importListResponse.Content); + + if (jsonResponse?.Wants == null) + { + return items; + } + + foreach (var want in jsonResponse.Wants) + { + if (want?.BasicInformation != null) + { + try + { + if (_httpClient != null && _settings != null) + { + var releaseInfo = FetchReleaseDetails(want.BasicInformation.ResourceUrl); + if (releaseInfo != null) + { + items.Add(releaseInfo); + } + } + } + catch + { + // If we can't fetch release details, skip this item + continue; + } + } + } + + return items; + } + + // Unfortunately discogs release details are nested in a given /release/N endpoint. + // We'll have to fetch each one to get proper details. + private ImportListItemInfo FetchReleaseDetails(string resourceUrl) + { + var request = new HttpRequestBuilder(resourceUrl) + .SetHeader("Authorization", $"Discogs token={_settings.Token}") + .Build(); + + var response = _httpClient.Execute(request); + + if (response.StatusCode != HttpStatusCode.OK) + { + return null; + } + + var releaseResponse = Json.Deserialize(response.Content); + + if (releaseResponse?.Artists?.Any() == true && releaseResponse.Title.IsNotNullOrWhiteSpace()) + { + return new ImportListItemInfo + { + Artist = releaseResponse.Artists.First().Name, + Album = releaseResponse.Title + }; + } + + return null; + } + + protected virtual bool PreProcess(ImportListResponse importListResponse) + { + if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new ImportListException(importListResponse, + "Discogs API call resulted in an unexpected StatusCode [{0}]", + importListResponse.HttpResponse.StatusCode); + } + + if (importListResponse.HttpResponse.Headers.ContentType != null && + importListResponse.HttpResponse.Headers.ContentType.Contains("text/html")) + { + throw new ImportListException(importListResponse, + "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable."); + } + + return true; + } +} + +public class DiscogsWantlistResponse +{ + public List Wants { get; set; } +} + +public class DiscogsWantlistItem +{ + [JsonProperty("basic_information")] + public DiscogsBasicInformation BasicInformation { get; set; } +} + +public class DiscogsBasicInformation +{ + public int Id { get; set; } + public string Title { get; set; } + [JsonProperty("resource_url")] + public string ResourceUrl { get; set; } + public List Artists { get; set; } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs new file mode 100644 index 000000000..6be4820e7 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsWantlistRequestGenerator : IImportListRequestGenerator + { + public DiscogsWantlistSettings Settings { get; set; } + + public int MaxPages { get; set; } + public int PageSize { get; set; } + + public DiscogsWantlistRequestGenerator() + { + MaxPages = 1; + PageSize = 0; // Discogs doesn't support pagination for wantlists currently + } + + public virtual ImportListPageableRequestChain GetListItems() + { + var pageableRequests = new ImportListPageableRequestChain(); + pageableRequests.Add(GetPagedRequests()); + return pageableRequests; + } + + private IEnumerable GetPagedRequests() + { + var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/')) + .Resource($"/users/{Settings.Username}/wants") + .SetHeader("Authorization", $"Discogs token={Settings.Token}") + .Build(); + + yield return new ImportListRequest(request); + } + } +} diff --git a/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs new file mode 100644 index 000000000..8279e0355 --- /dev/null +++ b/src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs @@ -0,0 +1,38 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.ImportLists.Discogs +{ + public class DiscogsWantlistSettingsValidator : AbstractValidator + { + public DiscogsWantlistSettingsValidator() + { + RuleFor(c => c.Token).NotEmpty(); + RuleFor(c => c.Username).NotEmpty(); + } + } + + public class DiscogsWantlistSettings : IImportListSettings + { + private static readonly DiscogsWantlistSettingsValidator Validator = new DiscogsWantlistSettingsValidator(); + + public DiscogsWantlistSettings() + { + BaseUrl = "https://api.discogs.com"; + } + + public string BaseUrl { get; set; } + + [FieldDefinition(0, Label = "Token", Privacy = PrivacyLevel.ApiKey, HelpText = "Discogs Personal Access Token")] + public string Token { get; set; } + + [FieldDefinition(1, Label = "Username", HelpText = "Discogs username whose wantlist to import")] + public string Username { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +}