Wantlists

src/NzbDrone.Core.Test/ImportListTests/Discogs/DiscogsWantlistFixture.cs

@@ -0,0 +1,232 @@
+using System.Linq;
+using System.Net;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.ImportLists.Discogs;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.ImportListTests.Discogs;
+
+[TestFixture]
+public class DiscogsWantlistFixture : CoreTest<DiscogsWantlist>
+{
+    [SetUp]
+    public void Setup()
+    {
+        Subject.Definition = new ImportLists.ImportListDefinition
+        {
+            Id = 1,
+            Name = "Test Discogs Wantlist",
+            Settings = new DiscogsWantlistSettings
+            {
+                Token = "test_token_123456",
+                Username = "test_user",
+                BaseUrl = "https://api.discogs.com"
+            }
+        };
+    }
+
+    [Test]
+    public void should_parse_valid_wantlist_response()
+    {
+        var wantlistResponseJson = @"{
+            ""wants"": [
+                {
+                    ""basic_information"": {
+                        ""id"": 3,
+                        ""title"": ""Profound Sounds Vol. 1"",
+                        ""resource_url"": ""https://api.discogs.com/releases/3"",
+                        ""artists"": [
+                            {
+                                ""name"": ""Josh Wink"",
+                                ""id"": 3
+                            }
+                        ]
+                    }
+                }
+            ]
+        }";
+
+        var releaseResponseJson = @"{
+            ""title"": ""Profound Sounds Vol. 1"",
+            ""artists"": [
+                {
+                    ""name"": ""Josh Wink"",
+                    ""id"": 3
+                }
+            ]
+        }";
+
+        var wantlistCalled = false;
+        var releaseCalled = false;
+        var actualWantlistUrl = "";
+        var actualReleaseUrl = "";
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.IsAny<HttpRequest>()))
+              .Callback<HttpRequest>(req =>
+              {
+                  if (req.Url.FullUri.Contains("/wants"))
+                  {
+                      wantlistCalled = true;
+                      actualWantlistUrl = req.Url.FullUri;
+                  }
+                  else if (req.Url.FullUri.Contains("/releases/"))
+                  {
+                      releaseCalled = true;
+                      actualReleaseUrl = req.Url.FullUri;
+                  }
+              })
+              .Returns<HttpRequest>(req =>
+              {
+                  if (req.Url.FullUri.Contains("/wants"))
+                  {
+                      return new HttpResponse(req, new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK);
+                  }
+                  else if (req.Url.FullUri.Contains("/releases/"))
+                  {
+                      return new HttpResponse(req, new HttpHeader(), releaseResponseJson, HttpStatusCode.OK);
+                  }
+                  else
+                  {
+                      return new HttpResponse(req, new HttpHeader(), "", HttpStatusCode.NotFound);
+                  }
+              });
+
+        var releases = Subject.Fetch();
+
+        // Debug output to see what happened
+        System.Console.WriteLine($"Wantlist called: {wantlistCalled}, URL: {actualWantlistUrl}");
+        System.Console.WriteLine($"Release called: {releaseCalled}, URL: {actualReleaseUrl}");
+        System.Console.WriteLine($"Releases count: {releases.Count}");
+        for (var i = 0; i < releases.Count; i++)
+        {
+            System.Console.WriteLine($"Release {i}: Artist='{releases[i].Artist}', Album='{releases[i].Album}'");
+        }
+
+        releases.Should().HaveCount(1);
+        releases.First().Artist.Should().Be("Josh Wink");
+        releases.First().Album.Should().Be("Profound Sounds Vol. 1");
+    }
+
+    [Test]
+    public void should_handle_empty_wantlist_response()
+    {
+        var responseJson = @"{""wants"": []}";
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.IsAny<HttpRequest>()))
+              .Returns(new HttpResponse(new HttpRequest("https://api.discogs.com/users/test_user/wants"), new HttpHeader(), responseJson, HttpStatusCode.OK));
+
+        var releases = Subject.Fetch();
+
+        releases.Should().BeEmpty();
+    }
+
+    [Test]
+    public void should_skip_items_when_release_fetch_fails()
+    {
+        var wantlistResponseJson = @"{
+            ""wants"": [
+                {
+                    ""basic_information"": {
+                        ""id"": 3,
+                        ""title"": ""Profound Sounds Vol. 1"",
+                        ""resource_url"": ""https://api.discogs.com/releases/3""
+                    }
+                }
+            ]
+        }";
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.Is<HttpRequest>(r => r.Url.ToString().Contains("/wants"))))
+              .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK));
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.Is<HttpRequest>(r => r.Url.ToString().Contains("/releases/"))))
+              .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), "", HttpStatusCode.NotFound));
+
+        var releases = Subject.Fetch();
+
+        releases.Should().BeEmpty();
+    }
+
+    [Test]
+    public void should_skip_releases_with_no_artists()
+    {
+        var wantlistResponseJson = @"{
+            ""wants"": [
+                {
+                    ""basic_information"": {
+                        ""id"": 3,
+                        ""title"": ""Compilation"",
+                        ""resource_url"": ""https://api.discogs.com/releases/3""
+                    }
+                }
+            ]
+        }";
+
+        var releaseResponseJson = @"{
+            ""title"": ""Compilation"",
+            ""artists"": []
+        }";
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.Is<HttpRequest>(r => r.Url.ToString().Contains("/wants"))))
+              .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK));
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.Is<HttpRequest>(r => r.Url.ToString().Contains("/releases/"))))
+              .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK));
+
+        var releases = Subject.Fetch();
+
+        releases.Should().BeEmpty();
+    }
+
+    [Test]
+    public void should_use_first_artist_when_multiple_artists()
+    {
+        var wantlistResponseJson = @"{
+            ""wants"": [
+                {
+                    ""basic_information"": {
+                        ""id"": 3,
+                        ""title"": ""Collaboration"",
+                        ""resource_url"": ""https://api.discogs.com/releases/3""
+                    }
+                }
+            ]
+        }";
+
+        var releaseResponseJson = @"{
+            ""title"": ""Collaboration"",
+            ""artists"": [
+                {
+                    ""name"": ""Artist 1"",
+                    ""id"": 1
+                },
+                {
+                    ""name"": ""Artist 2"",
+                    ""id"": 2
+                }
+            ]
+        }";
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.Is<HttpRequest>(r => r.Url.ToString().Contains("/wants"))))
+              .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), wantlistResponseJson, HttpStatusCode.OK));
+
+        Mocker.GetMock<IHttpClient>()
+              .Setup(s => s.Execute(It.Is<HttpRequest>(r => r.Url.ToString().Contains("/releases/"))))
+              .Returns(new HttpResponse(new HttpRequest("http://my.indexer.com"), new HttpHeader(), releaseResponseJson, HttpStatusCode.OK));
+
+        var releases = Subject.Fetch();
+
+        releases.Should().HaveCount(1);
+        releases.First().Artist.Should().Be("Artist 1");
+        releases.First().Album.Should().Be("Collaboration");
+    }
+}

src/NzbDrone.Core/ImportLists/Discogs/DiscogsLists.cs

@@ -11,7 +11,7 @@ public class DiscogsLists : HttpImportListBase<DiscogsListsSettings>
         public override string Name => "Discogs Lists";
         public override ImportListType ListType => ImportListType.Discogs;
         public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12);
-        public override TimeSpan RateLimit => TimeSpan.FromSeconds(1); // 60 requests per minute limit
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases
         public override int PageSize => 0; // Discogs doesn't support pagination for lists
 
         public DiscogsLists(IHttpClient httpClient,

src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlist.cs

@@ -0,0 +1,38 @@
+using System;
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Parser;
+
+namespace NzbDrone.Core.ImportLists.Discogs
+{
+    public class DiscogsWantlist : HttpImportListBase<DiscogsWantlistSettings>
+    {
+        public override string Name => "Discogs Wantlist";
+        public override ImportListType ListType => ImportListType.Discogs;
+        public override TimeSpan MinRefreshInterval => TimeSpan.FromHours(12);
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(3); // Conservative rate limiting to avoid 429 errors when fetching many releases
+        public override int PageSize => 0; // Discogs doesn't support pagination for wantlists currently
+
+        public DiscogsWantlist(IHttpClient httpClient,
+            IImportListStatusService importListStatusService,
+            IConfigService configService,
+            IParsingService parsingService,
+            Logger logger)
+            : base(httpClient, importListStatusService, configService, parsingService, logger)
+        {
+        }
+
+        public override IImportListRequestGenerator GetRequestGenerator()
+        {
+            return new DiscogsWantlistRequestGenerator { Settings = Settings };
+        }
+
+        public override IParseImportListResponse GetParser()
+        {
+            var parser = new DiscogsWantlistParser();
+            parser.SetContext(_httpClient, Settings);
+            return parser;
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistParser.cs

@@ -0,0 +1,140 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using Newtonsoft.Json;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.ImportLists.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.ImportLists.Discogs;
+
+public class DiscogsWantlistParser : IParseImportListResponse
+{
+    private ImportListResponse _importListResponse;
+    private IHttpClient _httpClient;
+    private DiscogsWantlistSettings _settings;
+
+    public DiscogsWantlistParser()
+    {
+    }
+
+    public void SetContext(IHttpClient httpClient, DiscogsWantlistSettings settings)
+    {
+        _httpClient = httpClient;
+        _settings = settings;
+    }
+
+    public IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse)
+    {
+        _importListResponse = importListResponse;
+
+        var items = new List<ImportListItemInfo>();
+
+        if (!PreProcess(_importListResponse))
+        {
+            return items;
+        }
+
+        var jsonResponse = Json.Deserialize<DiscogsWantlistResponse>(_importListResponse.Content);
+
+        if (jsonResponse?.Wants == null)
+        {
+            return items;
+        }
+
+        foreach (var want in jsonResponse.Wants)
+        {
+            if (want?.BasicInformation != null)
+            {
+                try
+                {
+                    if (_httpClient != null && _settings != null)
+                    {
+                        var releaseInfo = FetchReleaseDetails(want.BasicInformation.ResourceUrl);
+                        if (releaseInfo != null)
+                        {
+                            items.Add(releaseInfo);
+                        }
+                    }
+                }
+                catch
+                {
+                    // If we can't fetch release details, skip this item
+                    continue;
+                }
+            }
+        }
+
+        return items;
+    }
+
+    // Unfortunately discogs release details are nested in a given /release/N endpoint.
+    // We'll have to fetch each one to get proper details.
+    private ImportListItemInfo FetchReleaseDetails(string resourceUrl)
+    {
+        var request = new HttpRequestBuilder(resourceUrl)
+            .SetHeader("Authorization", $"Discogs token={_settings.Token}")
+            .Build();
+
+        var response = _httpClient.Execute(request);
+
+        if (response.StatusCode != HttpStatusCode.OK)
+        {
+            return null;
+        }
+
+        var releaseResponse = Json.Deserialize<DiscogsReleaseResponse>(response.Content);
+
+        if (releaseResponse?.Artists?.Any() == true && releaseResponse.Title.IsNotNullOrWhiteSpace())
+        {
+            return new ImportListItemInfo
+            {
+                Artist = releaseResponse.Artists.First().Name,
+                Album = releaseResponse.Title
+            };
+        }
+
+        return null;
+    }
+
+    protected virtual bool PreProcess(ImportListResponse importListResponse)
+    {
+        if (importListResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+        {
+            throw new ImportListException(importListResponse,
+                "Discogs API call resulted in an unexpected StatusCode [{0}]",
+                importListResponse.HttpResponse.StatusCode);
+        }
+
+        if (importListResponse.HttpResponse.Headers.ContentType != null &&
+            importListResponse.HttpResponse.Headers.ContentType.Contains("text/html"))
+        {
+            throw new ImportListException(importListResponse,
+                "Discogs API responded with HTML content. Wantlist may be too large or API may be unavailable.");
+        }
+
+        return true;
+    }
+}
+
+public class DiscogsWantlistResponse
+{
+    public List<DiscogsWantlistItem> Wants { get; set; }
+}
+
+public class DiscogsWantlistItem
+{
+    [JsonProperty("basic_information")]
+    public DiscogsBasicInformation BasicInformation { get; set; }
+}
+
+public class DiscogsBasicInformation
+{
+    public int Id { get; set; }
+    public string Title { get; set; }
+    [JsonProperty("resource_url")]
+    public string ResourceUrl { get; set; }
+    public List<DiscogsReleaseArtist> Artists { get; set; }
+}

src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistRequestGenerator.cs

@@ -0,0 +1,36 @@
+using System.Collections.Generic;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.ImportLists.Discogs
+{
+    public class DiscogsWantlistRequestGenerator : IImportListRequestGenerator
+    {
+        public DiscogsWantlistSettings Settings { get; set; }
+
+        public int MaxPages { get; set; }
+        public int PageSize { get; set; }
+
+        public DiscogsWantlistRequestGenerator()
+        {
+            MaxPages = 1;
+            PageSize = 0; // Discogs doesn't support pagination for wantlists currently
+        }
+
+        public virtual ImportListPageableRequestChain GetListItems()
+        {
+            var pageableRequests = new ImportListPageableRequestChain();
+            pageableRequests.Add(GetPagedRequests());
+            return pageableRequests;
+        }
+
+        private IEnumerable<ImportListRequest> GetPagedRequests()
+        {
+            var request = new HttpRequestBuilder(Settings.BaseUrl.TrimEnd('/'))
+                .Resource($"/users/{Settings.Username}/wants")
+                .SetHeader("Authorization", $"Discogs token={Settings.Token}")
+                .Build();
+
+            yield return new ImportListRequest(request);
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/Discogs/DiscogsWantlistSettings.cs

@@ -0,0 +1,38 @@
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.ImportLists.Discogs
+{
+    public class DiscogsWantlistSettingsValidator : AbstractValidator<DiscogsWantlistSettings>
+    {
+        public DiscogsWantlistSettingsValidator()
+        {
+            RuleFor(c => c.Token).NotEmpty();
+            RuleFor(c => c.Username).NotEmpty();
+        }
+    }
+
+    public class DiscogsWantlistSettings : IImportListSettings
+    {
+        private static readonly DiscogsWantlistSettingsValidator Validator = new DiscogsWantlistSettingsValidator();
+
+        public DiscogsWantlistSettings()
+        {
+            BaseUrl = "https://api.discogs.com";
+        }
+
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(0, Label = "Token", Privacy = PrivacyLevel.ApiKey, HelpText = "Discogs Personal Access Token")]
+        public string Token { get; set; }
+
+        [FieldDefinition(1, Label = "Username", HelpText = "Discogs username whose wantlist to import")]
+        public string Username { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}