From f57d14fb1604bebc30b1544e979cb80e839bb6f8 Mon Sep 17 00:00:00 2001 From: Meyn Date: Sat, 24 Jan 2026 20:09:02 +0100 Subject: [PATCH] Implement release year detection --- .../PrioritizeDownloadDecisionFixture.cs | 42 +++ .../YearMatchSpecificationFixture.cs | 146 ++++++++++ .../MusicTests/AlbumServiceFixture.cs | 2 + .../AlbumServiceYearMatchingFixture.cs | 237 ++++++++++++++++ .../MusicTests/AlbumYearMatcherFixture.cs | 203 ++++++++++++++ .../ParsingServiceTests/GetAlbumsFixture.cs | 49 +++- .../GetAlbumsWithYearFixture.cs | 257 ++++++++++++++++++ .../ParserTests/YearParsingFixture.cs | 138 ++++++++++ .../DownloadDecisionComparer.cs | 45 ++- .../DownloadDecisionPriorizationService.cs | 7 +- .../Specifications/YearMatchSpecification.cs | 51 ++++ src/NzbDrone.Core/Music/AlbumYearMatcher.cs | 97 +++++++ .../Music/AlbumYearMatchingOptions.cs | 18 ++ src/NzbDrone.Core/Music/IAlbumYearMatcher.cs | 53 ++++ .../Music/Services/AlbumService.cs | 121 ++++++++- .../Parser/Model/ParsedAlbumInfo.cs | 7 +- src/NzbDrone.Core/Parser/Parser.cs | 61 ++++- src/NzbDrone.Core/Parser/ParsingService.cs | 70 ++++- 18 files changed, 1566 insertions(+), 38 deletions(-) create mode 100644 src/NzbDrone.Core.Test/DecisionEngineTests/YearMatchSpecificationFixture.cs create mode 100644 src/NzbDrone.Core.Test/MusicTests/AlbumServiceYearMatchingFixture.cs create mode 100644 src/NzbDrone.Core.Test/MusicTests/AlbumYearMatcherFixture.cs create mode 100644 src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsWithYearFixture.cs create mode 100644 src/NzbDrone.Core.Test/ParserTests/YearParsingFixture.cs create mode 100644 src/NzbDrone.Core/DecisionEngine/Specifications/YearMatchSpecification.cs create mode 100644 src/NzbDrone.Core/Music/AlbumYearMatcher.cs create mode 100644 src/NzbDrone.Core/Music/AlbumYearMatchingOptions.cs create mode 100644 src/NzbDrone.Core/Music/IAlbumYearMatcher.cs diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs index c9c432b84..b6c673097 100644 --- a/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/PrioritizeDownloadDecisionFixture.cs @@ -29,6 +29,8 @@ public void Setup() Mocker.GetMock() .Setup(s => s.Get(It.IsAny())) .Returns(new QualityDefinition { PreferredSize = null }); + + Mocker.SetConstant(new AlbumYearMatcher()); } private void GivenPreferredSize(double? size) @@ -605,5 +607,45 @@ public void ensure_download_decisions_indexer_priority_is_not_perfered_over_qual qualifiedReports.Skip(2).First().RemoteAlbum.Should().Be(remoteAlbum1); qualifiedReports.Last().RemoteAlbum.Should().Be(remoteAlbum3); } + + [Test] + public void should_prefer_release_with_matching_year() + { + var album2020 = GivenAlbum(1); + album2020.ReleaseDate = new System.DateTime(2020, 6, 15); + + var remoteAlbum1 = GivenRemoteAlbum(new List { album2020 }, new QualityModel(Quality.FLAC)); + var remoteAlbum2 = GivenRemoteAlbum(new List { album2020 }, new QualityModel(Quality.FLAC)); + + remoteAlbum1.ParsedAlbumInfo.ReleaseYear = 2015; + remoteAlbum2.ParsedAlbumInfo.ReleaseYear = 2020; + + var decisions = new List(); + decisions.Add(new DownloadDecision(remoteAlbum1)); + decisions.Add(new DownloadDecision(remoteAlbum2)); + + var qualifiedReports = Subject.PrioritizeDecisions(decisions); + qualifiedReports.First().RemoteAlbum.ParsedAlbumInfo.ReleaseYear.Should().Be(2020); + } + + [Test] + public void should_prefer_quality_over_year_match() + { + var album2020 = GivenAlbum(1); + album2020.ReleaseDate = new System.DateTime(2020, 6, 15); + + var remoteAlbum1 = GivenRemoteAlbum(new List { album2020 }, new QualityModel(Quality.FLAC)); + var remoteAlbum2 = GivenRemoteAlbum(new List { album2020 }, new QualityModel(Quality.MP3_256)); + + remoteAlbum1.ParsedAlbumInfo.ReleaseYear = 2015; + remoteAlbum2.ParsedAlbumInfo.ReleaseYear = 2020; + + var decisions = new List(); + decisions.Add(new DownloadDecision(remoteAlbum1)); + decisions.Add(new DownloadDecision(remoteAlbum2)); + + var qualifiedReports = Subject.PrioritizeDecisions(decisions); + qualifiedReports.First().RemoteAlbum.ParsedAlbumInfo.Quality.Quality.Should().Be(Quality.FLAC); + } } } diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/YearMatchSpecificationFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/YearMatchSpecificationFixture.cs new file mode 100644 index 000000000..aa9e41579 --- /dev/null +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/YearMatchSpecificationFixture.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.DecisionEngine.Specifications; +using NzbDrone.Core.Music; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.DecisionEngineTests +{ + [TestFixture] + public class YearMatchSpecificationFixture : CoreTest + { + private Artist _artist; + private Album _album; + private RemoteAlbum _remoteAlbum; + + [SetUp] + public void Setup() + { + _artist = Builder.CreateNew().With(s => s.Id = 1).Build(); + _album = Builder.CreateNew() + .With(s => s.ReleaseDate = new DateTime(2020, 6, 15)) + .Build(); + + _remoteAlbum = new RemoteAlbum + { + Artist = _artist, + Albums = new List { _album }, + ParsedAlbumInfo = new ParsedAlbumInfo + { + AlbumTitle = "Test Album", + ReleaseYear = 2020 + }, + Release = new ReleaseInfo + { + Title = "Artist - Test Album (2020) FLAC" + } + }; + + Mocker.SetConstant(new AlbumYearMatcher()); + } + + [Test] + public void should_accept_when_no_year_in_parsed_info() + { + _remoteAlbum.ParsedAlbumInfo.ReleaseYear = null; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_when_no_albums() + { + _remoteAlbum.Albums = new List(); + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_when_year_matches_exactly() + { + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_when_year_is_within_tolerance() + { + _remoteAlbum.ParsedAlbumInfo.ReleaseYear = 2021; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_accept_when_year_difference_is_within_fuzzy_range() + { + _remoteAlbum.ParsedAlbumInfo.ReleaseYear = 2023; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_reject_when_year_difference_exceeds_hard_limit() + { + _remoteAlbum.ParsedAlbumInfo.ReleaseYear = 2010; + + var result = Subject.IsSatisfiedBy(_remoteAlbum, null); + + result.Accepted.Should().BeFalse(); + result.Reason.Should().Contain("does not match"); + } + + [Test] + public void should_accept_when_album_has_no_release_date() + { + _album.ReleaseDate = null; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_check_all_albums_in_release() + { + var album2 = Builder.CreateNew() + .With(s => s.ReleaseDate = new DateTime(2010, 1, 1)) + .Build(); + + _remoteAlbum.Albums = new List { _album, album2 }; + _remoteAlbum.ParsedAlbumInfo.ReleaseYear = 2020; + + var result = Subject.IsSatisfiedBy(_remoteAlbum, null); + + result.Accepted.Should().BeFalse(); + } + + [Test] + public void should_accept_multi_album_when_all_years_match() + { + var album2 = Builder.CreateNew() + .With(s => s.ReleaseDate = new DateTime(2020, 8, 20)) + .Build(); + + _remoteAlbum.Albums = new List { _album, album2 }; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [TestCase(2020, 2020, true)] + [TestCase(2020, 2021, true)] + [TestCase(2020, 2019, true)] + [TestCase(2020, 2023, true)] + [TestCase(2020, 2025, true)] + [TestCase(2020, 2026, false)] + [TestCase(2020, 2014, false)] + [TestCase(2020, 2005, false)] + public void should_handle_various_year_differences(int albumYear, int parsedYear, bool expectedAccepted) + { + _album.ReleaseDate = new DateTime(albumYear, 6, 15); + _remoteAlbum.ParsedAlbumInfo.ReleaseYear = parsedYear; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().Be(expectedAccepted); + } + } +} diff --git a/src/NzbDrone.Core.Test/MusicTests/AlbumServiceFixture.cs b/src/NzbDrone.Core.Test/MusicTests/AlbumServiceFixture.cs index 6faa5ebd1..890d6e8da 100644 --- a/src/NzbDrone.Core.Test/MusicTests/AlbumServiceFixture.cs +++ b/src/NzbDrone.Core.Test/MusicTests/AlbumServiceFixture.cs @@ -31,6 +31,8 @@ public void Setup() Mocker.GetMock() .Setup(s => s.GetAlbumsByArtistMetadataId(It.IsAny())) .Returns(_albums); + + Mocker.SetConstant(new AlbumYearMatcher()); } private void GivenSimilarAlbum() diff --git a/src/NzbDrone.Core.Test/MusicTests/AlbumServiceYearMatchingFixture.cs b/src/NzbDrone.Core.Test/MusicTests/AlbumServiceYearMatchingFixture.cs new file mode 100644 index 000000000..bb6f825e7 --- /dev/null +++ b/src/NzbDrone.Core.Test/MusicTests/AlbumServiceYearMatchingFixture.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Music; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.MusicTests.AlbumRepositoryTests +{ + [TestFixture] + public class AlbumServiceYearMatchingFixture : CoreTest + { + private List _albums; + + [SetUp] + public void Setup() + { + _albums = new List + { + new Album + { + Id = 1, + Title = "Greatest Hits", + CleanTitle = "greatesthits", + ReleaseDate = new DateTime(2010, 6, 15) + }, + new Album + { + Id = 2, + Title = "Greatest Hits", + CleanTitle = "greatesthits", + ReleaseDate = new DateTime(2020, 3, 20) + }, + new Album + { + Id = 3, + Title = "Greatest Hits Vol. 2", + CleanTitle = "greatesthitsvol2", + ReleaseDate = new DateTime(2015, 8, 10) + }, + new Album + { + Id = 4, + Title = "Peppermint Winter", + CleanTitle = "peppermintwinter", + ReleaseDate = new DateTime(2012, 12, 1) + }, + new Album + { + Id = 5, + Title = "Peppermint Winter (Remastered)", + CleanTitle = "peppermintwinterremastered", + ReleaseDate = new DateTime(2022, 12, 1) + }, + new Album + { + Id = 6, + Title = "Album Without Date", + CleanTitle = "albumwithoutdate", + ReleaseDate = null + } + }; + + Mocker.GetMock() + .Setup(s => s.GetAlbumsByArtistMetadataId(It.IsAny())) + .Returns(_albums); + + Mocker.GetMock() + .Setup(s => s.FindByTitle(It.IsAny(), "Peppermint Winter")) + .Returns(_albums[3]); + + Mocker.GetMock() + .Setup(s => s.FindByTitle(It.IsAny(), "Greatest Hits")) + .Returns(_albums[0]); + + Mocker.SetConstant(new AlbumYearMatcher()); + } + + [Test] + public void should_return_album_when_year_matches_exactly() + { + var album = Subject.FindByTitleAndYear(0, "Peppermint Winter", 2012); + + album.Should().NotBeNull(); + album.Id.Should().Be(4); + album.ReleaseDate.Value.Year.Should().Be(2012); + } + + [Test] + public void should_return_album_when_year_is_within_acceptable_range() + { + var album = Subject.FindByTitleAndYear(0, "Peppermint Winter", 2013); + + album.Should().NotBeNull(); + album.Id.Should().Be(4); + } + + [Test] + public void should_return_null_when_year_differs_significantly() + { + var album = Subject.FindByTitleAndYear(0, "Peppermint Winter", 2020); + + album.Should().BeNull(); + } + + [Test] + public void should_return_album_when_year_is_null() + { + var album = Subject.FindByTitleAndYear(0, "Peppermint Winter", null); + + album.Should().NotBeNull(); + album.Id.Should().Be(4); + } + + [Test] + public void should_return_null_when_album_not_found() + { + Mocker.GetMock() + .Setup(s => s.FindByTitle(It.IsAny(), "Nonexistent Album")) + .Returns((Album)null); + + var album = Subject.FindByTitleAndYear(0, "Nonexistent Album", 2012); + + album.Should().BeNull(); + } + + [Test] + public void should_find_album_by_inexact_title_and_matching_year() + { + var album = Subject.FindByTitleAndYearInexact(0, "Peppermint Wintr", 2012); + + album.Should().NotBeNull(); + album.Id.Should().Be(4); + } + + [Test] + public void should_prefer_album_with_matching_year_when_similar_titles_exist() + { + var album = Subject.FindByTitleAndYearInexact(0, "Greatest Hits", 2010); + + album.Should().NotBeNull(); + album.Id.Should().Be(1); + album.ReleaseDate.Value.Year.Should().Be(2010); + } + + [Test] + public void should_prefer_album_with_matching_year_2020() + { + var album = Subject.FindByTitleAndYearInexact(0, "Greatest Hits", 2020); + + album.Should().NotBeNull(); + album.Id.Should().Be(2); + album.ReleaseDate.Value.Year.Should().Be(2020); + } + + [Test] + public void should_distinguish_original_from_remastered_by_year() + { + var album = Subject.FindByTitleAndYearInexact(0, "Peppermint Winter", 2012); + + album.Should().NotBeNull(); + album.Id.Should().Be(4); + album.Title.Should().Be("Peppermint Winter"); + } + + [Test] + public void should_find_remastered_version_by_year() + { + var album = Subject.FindByTitleAndYearInexact(0, "Peppermint Winter", 2022); + + album.Should().NotBeNull(); + album.Id.Should().Be(5); + album.Title.Should().Be("Peppermint Winter (Remastered)"); + } + + [Test] + public void should_fall_back_to_title_only_matching_when_year_is_null() + { + var album = Subject.FindByTitleAndYearInexact(0, "Peppermint Wintr", null); + + album.Should().NotBeNull(); + } + + [Test] + public void should_return_null_when_no_matching_albums() + { + var album = Subject.FindByTitleAndYearInexact(0, "Completely Unknown Album Title XYZ", 2012); + + album.Should().BeNull(); + } + + [Test] + public void should_handle_album_without_release_date() + { + var album = Subject.FindByTitleAndYearInexact(0, "Album Without Date", 2015); + + album.Should().NotBeNull(); + album.Id.Should().Be(6); + } + + [Test] + public void should_return_candidates_filtered_by_year() + { + var candidates = Subject.GetCandidates(0, "Greatest Hits", 2010); + + candidates.Should().NotBeEmpty(); + candidates[0].ReleaseDate.Value.Year.Should().Be(2010); + } + + [Test] + public void should_return_candidates_without_year_filter() + { + var candidates = Subject.GetCandidates(0, "Greatest Hits", null); + + candidates.Should().NotBeEmpty(); + } + + [Test] + public void should_give_small_bonus_for_one_year_difference() + { + var album = Subject.FindByTitleAndYearInexact(0, "Greatest Hits", 2011); + + album.Should().NotBeNull(); + album.Id.Should().Be(1); + } + + [Test] + public void should_penalize_large_year_differences() + { + var album = Subject.FindByTitleAndYearInexact(0, "Greatest Hits Vol. 2", 2015); + + album.Should().NotBeNull(); + album.Id.Should().Be(3); + } + } +} diff --git a/src/NzbDrone.Core.Test/MusicTests/AlbumYearMatcherFixture.cs b/src/NzbDrone.Core.Test/MusicTests/AlbumYearMatcherFixture.cs new file mode 100644 index 000000000..5d69a7ec8 --- /dev/null +++ b/src/NzbDrone.Core.Test/MusicTests/AlbumYearMatcherFixture.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Music; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.MusicTests +{ + [TestFixture] + public class AlbumYearMatcherFixture : CoreTest + { + [Test] + public void should_return_match_when_parsed_year_is_null() + { + var result = Subject.Match(new DateTime(2020, 1, 1), null); + + result.IsMatch.Should().BeTrue(); + result.YearDifference.Should().BeNull(); + result.ScoreAdjustment.Should().Be(0); + } + + [Test] + public void should_return_match_when_album_date_is_null() + { + var result = Subject.Match((DateTime?)null, 2020); + + result.IsMatch.Should().BeTrue(); + result.YearDifference.Should().Be(0); + result.Confidence.Should().Be(YearMatchConfidence.Low); + } + + [Test] + public void should_give_high_bonus_for_exact_year_match() + { + var result = Subject.Match(new DateTime(2020, 6, 15), 2020); + + result.IsMatch.Should().BeTrue(); + result.YearDifference.Should().Be(0); + result.ScoreAdjustment.Should().Be(AlbumYearMatchingOptions.ExactYearBonus); + result.Confidence.Should().Be(YearMatchConfidence.High); + } + + [Test] + public void should_give_medium_bonus_for_one_year_difference() + { + var result = Subject.Match(new DateTime(2020, 6, 15), 2021); + + result.IsMatch.Should().BeTrue(); + result.YearDifference.Should().Be(1); + result.ScoreAdjustment.Should().Be(AlbumYearMatchingOptions.CloseYearBonus); + result.Confidence.Should().Be(YearMatchConfidence.High); + } + + [Test] + public void should_apply_penalty_for_two_year_difference() + { + var result = Subject.Match(new DateTime(2020, 6, 15), 2022); + + result.IsMatch.Should().BeTrue(); + result.YearDifference.Should().Be(2); + result.ScoreAdjustment.Should().BeLessThan(0); + } + + [Test] + public void should_apply_increasing_penalty_for_larger_differences() + { + var result3 = Subject.Match(new DateTime(2020, 6, 15), 2023); + var result4 = Subject.Match(new DateTime(2020, 6, 15), 2024); + + result3.ScoreAdjustment.Should().BeGreaterThan(result4.ScoreAdjustment); + } + + [Test] + public void should_reject_when_year_difference_exceeds_hard_limit() + { + var result = Subject.Match(new DateTime(2020, 6, 15), 2010); + + result.IsMatch.Should().BeFalse(); + result.YearDifference.Should().Be(10); + result.RejectionReason.Should().Contain("does not match"); + } + + [Test] + public void should_reject_at_boundary_of_hard_limit() + { + var result = Subject.Match(new DateTime(2020, 6, 15), 2014); + + result.IsMatch.Should().BeFalse(); + result.YearDifference.Should().Be(6); + } + + [Test] + public void should_accept_at_fuzzy_match_boundary() + { + var result = Subject.Match(new DateTime(2020, 6, 15), 2015); + + result.IsMatch.Should().BeTrue(); + result.YearDifference.Should().Be(5); + result.Confidence.Should().Be(YearMatchConfidence.Low); + } + + [TestCase(2020, 2020, true)] + [TestCase(2020, 2019, true)] + [TestCase(2020, 2021, true)] + [TestCase(2020, 2018, true)] + [TestCase(2020, 2023, true)] + [TestCase(2020, 2025, true)] + [TestCase(2020, 2026, false)] + [TestCase(2020, 2014, false)] + [TestCase(2020, 2010, false)] + public void should_handle_year_boundaries_correctly(int albumYear, int parsedYear, bool expectedMatch) + { + var result = Subject.Match(new DateTime(albumYear, 1, 1), parsedYear); + + result.IsMatch.Should().Be(expectedMatch); + } + + [Test] + public void calculate_year_score_should_return_bonus_for_exact_match() + { + var score = Subject.CalculateYearScore(new DateTime(2020, 1, 1), 2020); + + score.Should().Be(AlbumYearMatchingOptions.ExactYearBonus); + } + + [Test] + public void calculate_year_score_should_return_zero_when_no_year_provided() + { + var score = Subject.CalculateYearScore(new DateTime(2020, 1, 1), null); + + score.Should().Be(0); + } + + [Test] + public void calculate_year_score_should_return_zero_when_no_album_date() + { + var score = Subject.CalculateYearScore(null, 2020); + + score.Should().Be(0); + } + + [Test] + public void should_match_album_with_null_year() + { + var album = new Album { Title = "Test", ReleaseDate = new DateTime(2020, 1, 1) }; + + var result = Subject.Match(album, null); + + result.IsMatch.Should().BeTrue(); + } + + [Test] + public void should_match_album_with_exact_year() + { + var album = new Album { Title = "Test", ReleaseDate = new DateTime(2020, 1, 1) }; + + var result = Subject.Match(album, 2020); + + result.IsMatch.Should().BeTrue(); + result.ScoreAdjustment.Should().Be(AlbumYearMatchingOptions.ExactYearBonus); + } + + [Test] + public void should_be_lenient_for_compilation_albums() + { + var album = new Album + { + Title = "Greatest Hits", + ReleaseDate = new DateTime(2020, 1, 1), + SecondaryTypes = new List + { + new SecondaryAlbumType { Name = "Compilation" } + }, + AlbumReleases = new LazyLoaded>(new List()) + }; + + var result = Subject.Match(album, 2015); + + result.IsMatch.Should().BeTrue(); + } + + [Test] + public void should_be_lenient_for_live_albums() + { + var album = new Album + { + Title = "Live at Wembley", + ReleaseDate = new DateTime(2020, 1, 1), + SecondaryTypes = new List + { + new SecondaryAlbumType { Name = "Live" } + }, + AlbumReleases = new LazyLoaded>(new List()) + }; + + var result = Subject.Match(album, 2015); + + result.IsMatch.Should().BeTrue(); + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsFixture.cs index 5973e1b8f..b27935058 100644 --- a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsFixture.cs +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsFixture.cs @@ -15,6 +15,12 @@ namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests [TestFixture] public class GetAlbumsFixture : CoreTest { + [SetUp] + public void Setup() + { + Mocker.SetConstant(new AlbumYearMatcher()); + } + [Test] public void should_not_fail_if_search_criteria_contains_multiple_albums_with_the_same_name() { @@ -28,13 +34,52 @@ public void should_not_fail_if_search_criteria_contains_multiple_albums_with_the var parsed = new ParsedAlbumInfo { - AlbumTitle = "IdenticalTitle" + AlbumTitle = "IdenticalTitle", + ReleaseYear = null }; Subject.GetAlbums(parsed, artist, criteria).Should().BeEquivalentTo(new List()); Mocker.GetMock() - .Verify(s => s.FindByTitle(artist.ArtistMetadataId, "IdenticalTitle"), Times.Once()); + .Verify(s => s.FindByTitleAndYear(artist.ArtistMetadataId, "IdenticalTitle", null), Times.Once()); + } + + [Test] + public void should_return_empty_when_album_title_is_null() + { + var artist = Builder.CreateNew().Build(); + var parsed = new ParsedAlbumInfo + { + AlbumTitle = null + }; + + var result = Subject.GetAlbums(parsed, artist, null); + + result.Should().BeEmpty(); + } + + [Test] + public void should_use_year_for_disambiguation() + { + var artist = Builder.CreateNew().Build(); + var album = Builder.CreateNew() + .With(x => x.Title = "TestAlbum") + .Build(); + + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "TestAlbum", + ReleaseYear = 2020 + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(artist.ArtistMetadataId, "TestAlbum", 2020)) + .Returns(album); + + var result = Subject.GetAlbums(parsed, artist, null); + + result.Should().HaveCount(1); + result[0].Should().Be(album); } } } diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsWithYearFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsWithYearFixture.cs new file mode 100644 index 000000000..adad7aaaf --- /dev/null +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/GetAlbumsWithYearFixture.cs @@ -0,0 +1,257 @@ +using System; +using System.Collections.Generic; +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Music; +using NzbDrone.Core.Parser; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests +{ + [TestFixture] + public class GetAlbumsWithYearFixture : CoreTest + { + private Artist _artist; + private List _albums; + + [SetUp] + public void Setup() + { + _artist = Builder.CreateNew() + .With(a => a.ArtistMetadataId = 1) + .Build(); + + _albums = new List + { + new Album + { + Id = 1, + Title = "Greatest Hits", + CleanTitle = "greatesthits", + ReleaseDate = new DateTime(2010, 6, 15) + }, + new Album + { + Id = 2, + Title = "Greatest Hits", + CleanTitle = "greatesthits", + ReleaseDate = new DateTime(2020, 3, 20) + } + }; + + Mocker.SetConstant(new AlbumYearMatcher()); + } + + [Test] + public void should_use_year_from_parsed_album_info_for_matching() + { + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2010, + ReleaseDate = "2010" + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", 2010)) + .Returns(_albums[0]); + + var result = Subject.GetAlbums(parsed, _artist, null); + + result.Should().HaveCount(1); + result[0].Id.Should().Be(1); + result[0].ReleaseDate.Value.Year.Should().Be(2010); + } + + [Test] + public void should_match_2020_album_when_year_is_2020() + { + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2020, + ReleaseDate = "2020" + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", 2020)) + .Returns(_albums[1]); + + var result = Subject.GetAlbums(parsed, _artist, null); + + result.Should().HaveCount(1); + result[0].Id.Should().Be(2); + result[0].ReleaseDate.Value.Year.Should().Be(2020); + } + + [Test] + public void should_fall_back_to_inexact_matching_when_exact_match_not_found() + { + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2010, + ReleaseDate = "2010" + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", 2010)) + .Returns((Album)null); + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYearInexact(_artist.ArtistMetadataId, "Greatest Hits", 2010)) + .Returns(_albums[0]); + + var result = Subject.GetAlbums(parsed, _artist, null); + + result.Should().HaveCount(1); + result[0].Id.Should().Be(1); + } + + [Test] + public void should_work_without_year_in_parsed_info() + { + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = null, + ReleaseDate = null + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", null)) + .Returns(_albums[0]); + + var result = Subject.GetAlbums(parsed, _artist, null); + + result.Should().HaveCount(1); + } + + [Test] + public void should_use_release_year_property_directly() + { + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2010, + ReleaseDate = "2010" + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", 2010)) + .Returns(_albums[0]); + + Subject.GetAlbums(parsed, _artist, null); + + Mocker.GetMock() + .Verify(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", 2010), Times.Once()); + } + + [Test] + public void should_verify_year_when_using_search_criteria_with_matching_title() + { + var criteria = new AlbumSearchCriteria + { + Artist = _artist, + Albums = _albums + }; + + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2020, + ReleaseDate = "2020" + }; + + var result = Subject.GetAlbums(parsed, _artist, criteria); + + result.Should().HaveCount(1); + result[0].Id.Should().Be(2); + } + + [Test] + public void should_reject_search_criteria_match_when_year_differs_significantly() + { + var criteria = new AlbumSearchCriteria + { + Artist = _artist, + Albums = new List { _albums[0] } + }; + + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2020, + ReleaseDate = "2020" + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", 2020)) + .Returns(_albums[1]); + + var result = Subject.GetAlbums(parsed, _artist, criteria); + + result.Should().HaveCount(1); + result[0].Id.Should().Be(2); + } + + [Test] + public void should_accept_search_criteria_match_when_year_is_close() + { + var album2011 = new Album + { + Id = 3, + Title = "Greatest Hits", + CleanTitle = "greatesthits", + ReleaseDate = new DateTime(2011, 1, 1) + }; + + var criteria = new AlbumSearchCriteria + { + Artist = _artist, + Albums = new List { album2011 } + }; + + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = 2010, + ReleaseDate = "2010" + }; + + var result = Subject.GetAlbums(parsed, _artist, criteria); + + result.Should().HaveCount(1); + result[0].Id.Should().Be(3); + } + + [Test] + public void should_handle_null_release_year_with_search_criteria() + { + var criteria = new AlbumSearchCriteria + { + Artist = _artist, + Albums = _albums + }; + + var parsed = new ParsedAlbumInfo + { + AlbumTitle = "Greatest Hits", + ReleaseYear = null, + ReleaseDate = null + }; + + Mocker.GetMock() + .Setup(s => s.FindByTitleAndYear(_artist.ArtistMetadataId, "Greatest Hits", null)) + .Returns(_albums[0]); + + var result = Subject.GetAlbums(parsed, _artist, criteria); + + result.Should().HaveCount(1); + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/YearParsingFixture.cs b/src/NzbDrone.Core.Test/ParserTests/YearParsingFixture.cs new file mode 100644 index 000000000..fcdd0dd45 --- /dev/null +++ b/src/NzbDrone.Core.Test/ParserTests/YearParsingFixture.cs @@ -0,0 +1,138 @@ +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Music; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.ParserTests +{ + [TestFixture] + public class YearParsingFixture : CoreTest + { + [TestCase("Artist - Album (2020) FLAC", 2020, YearMatchConfidence.High)] + [TestCase("Artist - Album - 2018 - FLAC", 2018, YearMatchConfidence.High)] + [TestCase("Artist-Album-WEB-2021-GROUP", 2021, YearMatchConfidence.High)] + [TestCase("Artist - Album 2022", 2022, YearMatchConfidence.High)] + [TestCase("(Rock) Artist - Album - 2015 FLAC", 2015, YearMatchConfidence.High)] + public void should_parse_year_from_standard_formats(string title, int expectedYear, YearMatchConfidence expectedConfidence) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + result.Should().NotBeNull(); + result.ReleaseYear.Should().Be(expectedYear); + result.YearConfidence.Should().Be(expectedConfidence); + } + + [TestCase("Artist - Album (2020) [FLAC]", 2020)] + [TestCase("Artist - Album (Deluxe) (2020)", 2020)] + public void should_parse_year_from_parentheses_formats(string title, int expectedYear) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + result.Should().NotBeNull(); + result.ReleaseYear.Should().Be(expectedYear); + } + + [TestCase("Artist - Album FLAC")] + [TestCase("Artist - Album [FLAC]")] + [TestCase("Artist - Album (Deluxe Edition)")] + public void should_have_null_year_when_not_present(string title) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + if (result != null) + { + result.ReleaseYear.Should().BeNull(); + } + } + + [TestCase("Artist-Album-Deluxe Edition-2CD-FLAC-2015-GROUP", 2015)] + [TestCase("Artist-Album-WEB-2017-FURY", 2017)] + public void should_parse_year_from_scene_formats(string title, int expectedYear) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + result.Should().NotBeNull(); + result.ReleaseYear.Should().Be(expectedYear); + } + + [TestCase("Artist - 2020 - Album Title", 2020)] + public void should_parse_year_from_artist_year_album_format(string title, int expectedYear) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + result.Should().NotBeNull(); + result.ReleaseYear.Should().Be(expectedYear); + } + + [Test] + public void should_set_release_date_string_from_year() + { + var result = Parser.Parser.ParseAlbumTitle("Artist - Album (2020) FLAC"); + + result.Should().NotBeNull(); + result.ReleaseDate.Should().Be("2020"); + result.ReleaseYear.Should().Be(2020); + } + + [Test] + public void should_handle_empty_release_date_when_no_year() + { + var result = Parser.Parser.ParseAlbumTitle("Artist - Album [FLAC]"); + + if (result != null && !result.ReleaseYear.HasValue) + { + result.ReleaseDate.Should().BeEmpty(); + } + } + + [TestCase("Artist - Discography 1990-2020", null)] + [TestCase("Artist Discography 2010-2023", null)] + public void should_not_set_release_year_for_discography(string title, int? expectedYear) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + result.Should().NotBeNull(); + result.Discography.Should().BeTrue(); + result.ReleaseYear.Should().Be(expectedYear); + } + + [TestCase("Artist - Album 1899 FLAC", null)] + [TestCase("Artist - Album 2099 FLAC", null)] + public void should_ignore_years_outside_valid_range(string title, int? expectedYear) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + if (result != null) + { + if (result.ReleaseYear.HasValue && (result.ReleaseYear < 1900 || result.ReleaseYear > 2100)) + { + result.YearConfidence.Should().Be(YearMatchConfidence.Low); + } + } + } + + [TestCase("Coldplay - Music of the Spheres (2021) [MP3 256kbps] [ df1975 ] [WEB]", 2021)] + [TestCase("Coldplay - Music of the Spheres [MP3 320kbps] [ Frederic1986 ] [WEB]", null)] + [TestCase("Artist - Album (2020) [ user2001 ] [FLAC]", 2020)] + [TestCase("Artist - Album (2019) [MP3] [ oldschool1975 ]", 2019)] + [TestCase("Artist - Album [FLAC] [ john1990 ] [⚡ fast ]", null)] + public void should_not_parse_year_from_username_in_brackets(string title, int? expectedYear) + { + var result = Parser.Parser.ParseAlbumTitle(title); + + if (expectedYear.HasValue) + { + result.Should().NotBeNull(); + result.ReleaseYear.Should().Be(expectedYear); + } + else + { + // If no expected year, either result is null or year is null + if (result != null) + { + result.ReleaseYear.Should().BeNull(); + } + } + } + } +} diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs index f03ab8d64..a62171bb1 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionComparer.cs @@ -4,6 +4,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Delay; using NzbDrone.Core.Qualities; @@ -15,15 +16,20 @@ public class DownloadDecisionComparer : IComparer private readonly IConfigService _configService; private readonly IDelayProfileService _delayProfileService; private readonly IQualityDefinitionService _qualityDefinitionService; + private readonly IAlbumYearMatcher _yearMatcher; public delegate int CompareDelegate(DownloadDecision x, DownloadDecision y); public delegate int CompareDelegate(DownloadDecision x, DownloadDecision y); - public DownloadDecisionComparer(IConfigService configService, IDelayProfileService delayProfileService, IQualityDefinitionService qualityDefinitionService) + public DownloadDecisionComparer(IConfigService configService, + IDelayProfileService delayProfileService, + IQualityDefinitionService qualityDefinitionService, + IAlbumYearMatcher yearMatcher) { _configService = configService; _delayProfileService = delayProfileService; _qualityDefinitionService = qualityDefinitionService; + _yearMatcher = yearMatcher; } public int Compare(DownloadDecision x, DownloadDecision y) @@ -32,6 +38,7 @@ public int Compare(DownloadDecision x, DownloadDecision y) { CompareQuality, CompareCustomFormatScore, + CompareYearMatch, CompareProtocol, CompareIndexerPriority, ComparePeersIfTorrent, @@ -84,6 +91,33 @@ private int CompareCustomFormatScore(DownloadDecision x, DownloadDecision y) return CompareBy(x.RemoteAlbum, y.RemoteAlbum, remoteAlbum => remoteAlbum.CustomFormatScore); } + private int CompareYearMatch(DownloadDecision x, DownloadDecision y) + { + return CompareBy(x.RemoteAlbum, y.RemoteAlbum, CalculateYearMatchScore); + } + + private double CalculateYearMatchScore(RemoteAlbum remoteAlbum) + { + if (remoteAlbum.Albums == null || !remoteAlbum.Albums.Any()) + { + return 0; + } + + var parsedYear = remoteAlbum.ParsedAlbumInfo?.ReleaseYear; + if (!parsedYear.HasValue) + { + return 0; + } + + var totalScore = 0.0; + foreach (var album in remoteAlbum.Albums) + { + totalScore += _yearMatcher.CalculateYearScore(album.ReleaseDate, parsedYear); + } + + return totalScore / remoteAlbum.Albums.Count; + } + private int CompareProtocol(DownloadDecision x, DownloadDecision y) { var result = CompareBy(x.RemoteAlbum, y.RemoteAlbum, remoteAlbum => @@ -170,11 +204,16 @@ private int CompareAgeIfUsenet(DownloadDecision x, DownloadDecision y) private int CompareSize(DownloadDecision x, DownloadDecision y) { - var sizeCompare = CompareBy(x.RemoteAlbum, y.RemoteAlbum, remoteAlbum => + var sizeCompare = CompareBy(x.RemoteAlbum, y.RemoteAlbum, remoteAlbum => { var preferredSize = _qualityDefinitionService.Get(remoteAlbum.ParsedAlbumInfo.Quality.Quality).PreferredSize; - var releaseDuration = remoteAlbum.Albums.Select(a => a.AlbumReleases.Value.Where(r => r.Monitored || a.AnyReleaseOk).Select(r => r.Duration).MaxOrDefault()).Sum() / 1000; + var releaseDuration = remoteAlbum.Albums + .Select(a => a.AlbumReleases.Value + .Where(r => r.Monitored || a.AnyReleaseOk) + .Select(r => r.Duration) + .MaxOrDefault()) + .Sum() / 1000; // If no value for preferred it means unlimited so fallback to sort largest is best if (preferredSize.HasValue && releaseDuration > 0) diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs index af1914814..e496d8b99 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionPriorizationService.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Music; using NzbDrone.Core.Profiles.Delay; using NzbDrone.Core.Qualities; @@ -16,12 +17,14 @@ public class DownloadDecisionPriorizationService : IPrioritizeDownloadDecision private readonly IConfigService _configService; private readonly IDelayProfileService _delayProfileService; private readonly IQualityDefinitionService _qualityDefinitionService; + private readonly IAlbumYearMatcher _albumYearMatcher; - public DownloadDecisionPriorizationService(IConfigService configService, IDelayProfileService delayProfileService, IQualityDefinitionService qualityDefinitionService) + public DownloadDecisionPriorizationService(IConfigService configService, IDelayProfileService delayProfileService, IQualityDefinitionService qualityDefinitionService, IAlbumYearMatcher albumYearMatcher) { _configService = configService; _delayProfileService = delayProfileService; _qualityDefinitionService = qualityDefinitionService; + _albumYearMatcher = albumYearMatcher; } public List PrioritizeDecisions(List decisions) @@ -29,7 +32,7 @@ public List PrioritizeDecisions(List decisio return decisions.Where(c => c.RemoteAlbum.DownloadAllowed) .GroupBy(c => c.RemoteAlbum.Artist.Id, (artistId, downloadDecisions) => { - return downloadDecisions.OrderByDescending(decision => decision, new DownloadDecisionComparer(_configService, _delayProfileService, _qualityDefinitionService)); + return downloadDecisions.OrderByDescending(decision => decision, new DownloadDecisionComparer(_configService, _delayProfileService, _qualityDefinitionService, _albumYearMatcher)); }) .SelectMany(c => c) .Union(decisions.Where(c => !c.RemoteAlbum.DownloadAllowed)) diff --git a/src/NzbDrone.Core/DecisionEngine/Specifications/YearMatchSpecification.cs b/src/NzbDrone.Core/DecisionEngine/Specifications/YearMatchSpecification.cs new file mode 100644 index 000000000..3d3ee9518 --- /dev/null +++ b/src/NzbDrone.Core/DecisionEngine/Specifications/YearMatchSpecification.cs @@ -0,0 +1,51 @@ +using System.Linq; +using NLog; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Music; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.DecisionEngine.Specifications +{ + public class YearMatchSpecification : IDecisionEngineSpecification + { + private readonly IAlbumYearMatcher _yearMatcher; + private readonly Logger _logger; + + public YearMatchSpecification(IAlbumYearMatcher yearMatcher, Logger logger) + { + _yearMatcher = yearMatcher; + _logger = logger; + } + + public SpecificationPriority Priority => SpecificationPriority.Default; + public RejectionType Type => RejectionType.Permanent; + + public Decision IsSatisfiedBy(RemoteAlbum subject, SearchCriteriaBase searchCriteria) + { + var parsedYear = subject.ParsedAlbumInfo?.ReleaseYear; + + if (!parsedYear.HasValue || subject.Albums == null || !subject.Albums.Any()) + { + return Decision.Accept(); + } + + foreach (var album in subject.Albums) + { + var result = _yearMatcher.Match(album, parsedYear); + + if (!result.IsMatch) + { + _logger.Debug("Rejecting release {0}: {1}", subject.Release.Title, result.RejectionReason); + return Decision.Reject(result.RejectionReason); + } + + if (result.YearDifference > AlbumYearMatchingOptions.FuzzyMatchMaxYearDiff) + { + _logger.Debug("Release {0} has year difference of {1} years, accepting with warning", subject.Release.Title, result.YearDifference); + } + } + + return Decision.Accept(); + } + } +} diff --git a/src/NzbDrone.Core/Music/AlbumYearMatcher.cs b/src/NzbDrone.Core/Music/AlbumYearMatcher.cs new file mode 100644 index 000000000..a9ab2fc40 --- /dev/null +++ b/src/NzbDrone.Core/Music/AlbumYearMatcher.cs @@ -0,0 +1,97 @@ +using System; +using System.Linq; + +namespace NzbDrone.Core.Music +{ + public class AlbumYearMatcher : IAlbumYearMatcher + { + public AlbumYearMatchResult Match(DateTime? albumReleaseDate, int? parsedYear) + { + if (!parsedYear.HasValue) + { + return AlbumYearMatchResult.NoYearProvided(); + } + + if (!albumReleaseDate.HasValue) + { + return AlbumYearMatchResult.Match(0, 0, YearMatchConfidence.Low); + } + + var albumYear = albumReleaseDate.Value.Year; + var yearDiff = Math.Abs(albumYear - parsedYear.Value); + + if (yearDiff == 0) + { + return AlbumYearMatchResult.Match(0, AlbumYearMatchingOptions.ExactYearBonus, YearMatchConfidence.High); + } + + if (yearDiff <= AlbumYearMatchingOptions.ExactMatchYearTolerance) + { + return AlbumYearMatchResult.Match(yearDiff, AlbumYearMatchingOptions.CloseYearBonus, YearMatchConfidence.High); + } + + if (yearDiff <= AlbumYearMatchingOptions.FuzzyMatchMaxYearDiff) + { + var penalty = -AlbumYearMatchingOptions.YearPenaltyPerYear * (yearDiff - AlbumYearMatchingOptions.ExactMatchYearTolerance); + return AlbumYearMatchResult.Match(yearDiff, penalty, YearMatchConfidence.Medium); + } + + if (yearDiff <= AlbumYearMatchingOptions.HardRejectYearDiff) + { + var penalty = -AlbumYearMatchingOptions.YearPenaltyPerYear * (yearDiff - AlbumYearMatchingOptions.ExactMatchYearTolerance); + return AlbumYearMatchResult.Match(yearDiff, penalty, YearMatchConfidence.Low); + } + + return AlbumYearMatchResult.Reject(yearDiff, $"Release year {parsedYear.Value} does not match album year {albumYear} {yearDiff}"); + } + + public AlbumYearMatchResult Match(Album album, int? parsedYear) + { + if (!parsedYear.HasValue) + { + return AlbumYearMatchResult.NoYearProvided(); + } + + var primaryResult = Match(album.ReleaseDate, parsedYear); + if (primaryResult.IsMatch && primaryResult.YearDifference <= AlbumYearMatchingOptions.ExactMatchYearTolerance) + { + return primaryResult; + } + + // Check album releases for remasters/editions with different years + if (album.AlbumReleases != null && album.AlbumReleases.IsLoaded) + { + var releases = album.AlbumReleases.Value; + if (releases != null && releases.Any()) + { + foreach (var release in releases.Where(r => r.Monitored || album.AnyReleaseOk)) + { + var releaseResult = Match(release.ReleaseDate, parsedYear); + if (releaseResult.IsMatch && + releaseResult.YearDifference <= AlbumYearMatchingOptions.ExactMatchYearTolerance) + { + return releaseResult; + } + } + } + } + + // Be more lenient for compilations and live albums + if (album.SecondaryTypes != null && album.SecondaryTypes.Any(t => t.Name == "Compilation" || t.Name == "Live")) + { + if (primaryResult.YearDifference <= AlbumYearMatchingOptions.HardRejectYearDiff) + { + return AlbumYearMatchResult.Match(primaryResult.YearDifference ?? 0, -AlbumYearMatchingOptions.YearPenaltyPerYear, YearMatchConfidence.Low); + } + } + + return primaryResult; + } + + public double CalculateYearScore(DateTime? albumReleaseDate, int? expectedYear) + { + var result = Match(albumReleaseDate, expectedYear); + return result.ScoreAdjustment; + } + } +} diff --git a/src/NzbDrone.Core/Music/AlbumYearMatchingOptions.cs b/src/NzbDrone.Core/Music/AlbumYearMatchingOptions.cs new file mode 100644 index 000000000..d1a0e4733 --- /dev/null +++ b/src/NzbDrone.Core/Music/AlbumYearMatchingOptions.cs @@ -0,0 +1,18 @@ +namespace NzbDrone.Core.Music +{ + public static class AlbumYearMatchingOptions + { + public const int ExactMatchYearTolerance = 1; + public const int FuzzyMatchMaxYearDiff = 3; + public const int HardRejectYearDiff = 5; + + public const double ExactYearBonus = 0.20; + public const double CloseYearBonus = 0.10; + public const double YearPenaltyPerYear = 0.05; + + public const double TitleMinThresholdWithYear = 0.55; + public const double TitleMinThresholdNoYear = 0.70; + public const double TitleFuzzThreshold = 0.70; + public const double TitleFuzzGap = 0.40; + } +} diff --git a/src/NzbDrone.Core/Music/IAlbumYearMatcher.cs b/src/NzbDrone.Core/Music/IAlbumYearMatcher.cs new file mode 100644 index 000000000..e1f9db376 --- /dev/null +++ b/src/NzbDrone.Core/Music/IAlbumYearMatcher.cs @@ -0,0 +1,53 @@ +using System; + +namespace NzbDrone.Core.Music +{ + public enum YearMatchConfidence + { + None = 0, + Low = 1, + Medium = 2, + High = 3 + } + + public class AlbumYearMatchResult + { + public bool IsMatch { get; set; } + public int? YearDifference { get; set; } + public double ScoreAdjustment { get; set; } + public string RejectionReason { get; set; } + public YearMatchConfidence Confidence { get; set; } + + public static AlbumYearMatchResult NoYearProvided() => new AlbumYearMatchResult + { + IsMatch = true, + YearDifference = null, + ScoreAdjustment = 0, + Confidence = YearMatchConfidence.None + }; + + public static AlbumYearMatchResult Match(int yearDiff, double scoreAdjustment, YearMatchConfidence confidence) => new AlbumYearMatchResult + { + IsMatch = true, + YearDifference = yearDiff, + ScoreAdjustment = scoreAdjustment, + Confidence = confidence + }; + + public static AlbumYearMatchResult Reject(int yearDiff, string reason) => new AlbumYearMatchResult + { + IsMatch = false, + YearDifference = yearDiff, + ScoreAdjustment = -AlbumYearMatchingOptions.YearPenaltyPerYear * AlbumYearMatchingOptions.HardRejectYearDiff, + RejectionReason = reason, + Confidence = YearMatchConfidence.High + }; + } + + public interface IAlbumYearMatcher + { + AlbumYearMatchResult Match(DateTime? albumReleaseDate, int? parsedYear); + AlbumYearMatchResult Match(Album album, int? parsedYear); + double CalculateYearScore(DateTime? albumReleaseDate, int? expectedYear); + } +} diff --git a/src/NzbDrone.Core/Music/Services/AlbumService.cs b/src/NzbDrone.Core/Music/Services/AlbumService.cs index 69b1794d3..656813de1 100644 --- a/src/NzbDrone.Core/Music/Services/AlbumService.cs +++ b/src/NzbDrone.Core/Music/Services/AlbumService.cs @@ -24,7 +24,10 @@ public interface IAlbumService Album FindById(string foreignId); Album FindByTitle(int artistMetadataId, string title); Album FindByTitleInexact(int artistMetadataId, string title); + Album FindByTitleAndYear(int artistMetadataId, string title, int? year); + Album FindByTitleAndYearInexact(int artistMetadataId, string title, int? year); List GetCandidates(int artistMetadataId, string title); + List GetCandidates(int artistMetadataId, string title, int? year); void DeleteAlbum(int albumId, bool deleteFiles, bool addImportListExclusion = false); List GetAllAlbums(); Album UpdateAlbum(Album album); @@ -49,16 +52,19 @@ public class AlbumService : IAlbumService, private readonly IAlbumRepository _albumRepository; private readonly IEventAggregator _eventAggregator; private readonly IMediaFileService _mediaFileService; + private readonly IAlbumYearMatcher _yearMatcher; private readonly Logger _logger; public AlbumService(IAlbumRepository albumRepository, IEventAggregator eventAggregator, IMediaFileService mediaFileService, + IAlbumYearMatcher yearMatcher, Logger logger) { _albumRepository = albumRepository; _eventAggregator = eventAggregator; _mediaFileService = mediaFileService; + _yearMatcher = yearMatcher; _logger = logger; } @@ -97,6 +103,42 @@ public Album FindByTitle(int artistMetadataId, string title) return _albumRepository.FindByTitle(artistMetadataId, title); } + public Album FindByTitleAndYear(int artistMetadataId, string title, int? year) + { + var album = _albumRepository.FindByTitle(artistMetadataId, title); + + if (album == null) + { + return null; + } + + var matchResult = _yearMatcher.Match(album, year); + + if (!matchResult.IsMatch) + { + _logger.Debug("Album '{0}' found but year mismatch: {1}", album.Title, matchResult.RejectionReason); + return null; + } + + return album; + } + + public Album FindByTitleAndYearInexact(int artistMetadataId, string title, int? year) + { + var albums = GetAlbumsByArtistMetadataId(artistMetadataId); + + foreach (var func in AlbumScoringFunctions(title, title.CleanArtistName())) + { + var results = FindByStringAndYearInexact(albums, func.Item1, func.Item2, year); + if (results.Count == 1) + { + return results[0]; + } + } + + return null; + } + private List, string>> AlbumScoringFunctions(string title, string cleanTitle) { Func, string, Tuple, string>> tc = Tuple.Create; @@ -130,14 +172,17 @@ public Album FindByTitleInexact(int artistMetadataId, string title) return null; } - public List GetCandidates(int artistMetadataId, string title) + public List GetCandidates(int artistMetadataId, string title) => + GetCandidates(artistMetadataId, title, null); + + public List GetCandidates(int artistMetadataId, string title, int? year) { var albums = GetAlbumsByArtistMetadataId(artistMetadataId); var output = new List(); foreach (var func in AlbumScoringFunctions(title, title.CleanArtistName())) { - output.AddRange(FindByStringInexact(albums, func.Item1, func.Item2)); + output.AddRange(FindByStringAndYearInexact(albums, func.Item1, func.Item2, year)); } return output.DistinctBy(x => x.Id).ToList(); @@ -145,9 +190,6 @@ public List GetCandidates(int artistMetadataId, string title) private List FindByStringInexact(List albums, Func scoreFunction, string title) { - const double fuzzThreshold = 0.7; - const double fuzzGap = 0.4; - var sortedAlbums = albums.Select(s => new { MatchProb = scoreFunction(s, title), @@ -157,8 +199,73 @@ private List FindByStringInexact(List albums, Func s.MatchProb) .ToList(); - return sortedAlbums.TakeWhile((x, i) => i == 0 || sortedAlbums[i - 1].MatchProb - x.MatchProb < fuzzGap) - .TakeWhile((x, i) => x.MatchProb > fuzzThreshold || (i > 0 && sortedAlbums[i - 1].MatchProb > fuzzThreshold)) + return sortedAlbums + .TakeWhile((x, i) => i == 0 || sortedAlbums[i - 1].MatchProb - x.MatchProb < AlbumYearMatchingOptions.TitleFuzzGap) + .TakeWhile((x, i) => x.MatchProb > AlbumYearMatchingOptions.TitleFuzzThreshold || + (i > 0 && sortedAlbums[i - 1].MatchProb > AlbumYearMatchingOptions.TitleFuzzThreshold)) + .Select(x => x.Album) + .ToList(); + } + + private List FindByStringAndYearInexact(List albums, Func scoreFunction, string title, int? year) + { + if (!year.HasValue) + { + return FindByStringInexact(albums, scoreFunction, title); + } + + var titleMinThreshold = AlbumYearMatchingOptions.TitleMinThresholdWithYear; + + var scoredAlbums = albums.Select(album => + { + var titleScore = scoreFunction(album, title); + var yearScore = _yearMatcher.CalculateYearScore(album.ReleaseDate, year); + var combinedScore = titleScore + yearScore; + + return new + { + TitleScore = titleScore, + YearScore = yearScore, + MatchProb = combinedScore, + Album = album + }; + }) + .Where(x => x.TitleScore >= titleMinThreshold) + .ToList(); + + if (!scoredAlbums.Any()) + { + return new List(); + } + + var sortedAlbums = scoredAlbums.OrderByDescending(s => s.MatchProb).ToList(); + var topResult = sortedAlbums.First(); + + if (sortedAlbums.Count > 1) + { + var secondResult = sortedAlbums[1]; + + // Strong year match preference: if top has positive year score and second doesn't + if (topResult.YearScore > 0 && secondResult.YearScore <= 0) + { + return new List { topResult.Album }; + } + + // Better year with comparable title score + if (topResult.YearScore > secondResult.YearScore && + topResult.TitleScore >= secondResult.TitleScore * 0.9) + { + return new List { topResult.Album }; + } + } + else if (topResult.TitleScore >= AlbumYearMatchingOptions.TitleFuzzThreshold) + { + return new List { topResult.Album }; + } + + return sortedAlbums + .Where(x => x.TitleScore >= AlbumYearMatchingOptions.TitleFuzzThreshold) + .TakeWhile((x, i) => i == 0 || sortedAlbums[i - 1].MatchProb - x.MatchProb < AlbumYearMatchingOptions.TitleFuzzGap) .Select(x => x.Album) .ToList(); } diff --git a/src/NzbDrone.Core/Parser/Model/ParsedAlbumInfo.cs b/src/NzbDrone.Core/Parser/Model/ParsedAlbumInfo.cs index 1fd80a6f9..c847e71c6 100644 --- a/src/NzbDrone.Core/Parser/Model/ParsedAlbumInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ParsedAlbumInfo.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Text.Json.Serialization; +using NzbDrone.Core.Music; using NzbDrone.Core.Qualities; namespace NzbDrone.Core.Parser.Model @@ -13,6 +14,8 @@ public class ParsedAlbumInfo public ArtistTitleInfo ArtistTitleInfo { get; set; } public QualityModel Quality { get; set; } public string ReleaseDate { get; set; } + public int? ReleaseYear { get; set; } + public YearMatchConfidence YearConfidence { get; set; } public bool Discography { get; set; } public int DiscographyStart { get; set; } public int DiscographyEnd { get; set; } @@ -29,10 +32,10 @@ public override string ToString() if (AlbumTitle != null) { - albumString = string.Format("{0}", AlbumTitle); + albumString = ReleaseYear.HasValue ? $"{AlbumTitle} ({ReleaseYear.Value})" : AlbumTitle; } - return string.Format("{0} - {1} {2}", ArtistName, albumString, Quality); + return $"{ArtistName} - {albumString} {Quality}"; } } } diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 2ca2f5ac8..522ed951c 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -79,11 +79,11 @@ public static class Parser RegexOptions.IgnoreCase | RegexOptions.Compiled), // Artist - Album (Year) Strict - new Regex(@"^(?:(?.+?)(?: - )+)(?.+?)\W*(?:\(|\[).+?(?\d{4})", + new Regex(@"^(?:(?.+?)(?: - )+)(?.+?)\W*\([^\[\]]*?(?\d{4})", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Artist - Album (Year) - new Regex(@"^(?:(?.+?)(?: - )+)(?.+?)\W*(?:\(|\[)(?\d{4})", + new Regex(@"^(?:(?.+?)(?: - )+)(?.+?)\W*\((?\d{4})", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Artist - Album - Year [something] @@ -100,12 +100,12 @@ public static class Parser // Artist-Album (Year) Strict // Hyphen no space between artist and album - new Regex(@"^(?:(?.+?)(?:-)+)(?.+?)\W*(?:\(|\[).+?(?\d{4})", + new Regex(@"^(?:(?.+?)(?:-)+)(?.+?)\W*\([^\[\]]*?(?\d{4})", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Artist-Album (Year) // Hyphen no space between artist and album - new Regex(@"^(?:(?.+?)(?:-)+)(?.+?)\W*(?:\(|\[)(?\d{4})", + new Regex(@"^(?:(?.+?)(?:-)+)(?.+?)\W*\((?\d{4})", RegexOptions.IgnoreCase | RegexOptions.Compiled), // Artist-Album [something] or Artist-Album (something) @@ -126,6 +126,10 @@ public static class Parser // Hyphen with no or more spaces between artist/album/year new Regex(@"^(?:(?.+?)(?:-))(?\d{4})(?:-)(?[^-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled), + + // Hyphen with spaces between artist - year - album + new Regex(@"^(?:(?.+?)(?:\s?-\s?))(?\d{4})(?:\s?-\s?)(?[^-]+)", + RegexOptions.IgnoreCase | RegexOptions.Compiled), }; private static readonly Regex[] RejectHashedReleasesRegex = new Regex[] @@ -723,21 +727,20 @@ private static ParsedAlbumInfo ParseAlbumMatchCollection(MatchCollection matchCo albumTitle = RequestInfoRegex.Replace(albumTitle, "").Trim(' '); releaseVersion = RequestInfoRegex.Replace(releaseVersion, "").Trim(' '); - int.TryParse(matchCollection[0].Groups["releaseyear"].Value, out var releaseYear); + var (releaseYear, yearConfidence) = ExtractYear(matchCollection[0].Groups["releaseyear"]); - ParsedAlbumInfo result; - - result = new ParsedAlbumInfo + var result = new ParsedAlbumInfo { - ReleaseTitle = releaseTitle + ReleaseTitle = releaseTitle, + ArtistName = artistName, + AlbumTitle = albumTitle, + ArtistTitleInfo = GetArtistTitleInfo(artistName), + ReleaseDate = releaseYear?.ToString() ?? string.Empty, + ReleaseYear = releaseYear, + YearConfidence = yearConfidence, + ReleaseVersion = releaseVersion }; - result.ArtistName = artistName; - result.AlbumTitle = albumTitle; - result.ArtistTitleInfo = GetArtistTitleInfo(result.ArtistName); - result.ReleaseDate = releaseYear.ToString(); - result.ReleaseVersion = releaseVersion; - if (matchCollection[0].Groups["discography"].Success) { int.TryParse(matchCollection[0].Groups["startyear"].Value, out var discStart); @@ -762,6 +765,34 @@ private static ParsedAlbumInfo ParseAlbumMatchCollection(MatchCollection matchCo return result; } + private static (int? year, YearMatchConfidence confidence) ExtractYear(Group yearGroup) + { + if (!yearGroup.Success) + { + return (null, YearMatchConfidence.None); + } + + if (!int.TryParse(yearGroup.Value, out var year)) + { + return (null, YearMatchConfidence.None); + } + + var currentYear = DateTime.UtcNow.Year; + + if (year < 1900 || year > currentYear + 1) + { + return (null, YearMatchConfidence.Low); + } + + // High confidence for recent years in standard patterns + if (year >= 1950 && year <= currentYear) + { + return (year, YearMatchConfidence.High); + } + + return (year, YearMatchConfidence.Medium); + } + private static bool ValidateBeforeParsing(string title) { if (title.ToLower().Contains("password") && title.ToLower().Contains("yenc")) diff --git a/src/NzbDrone.Core/Parser/ParsingService.cs b/src/NzbDrone.Core/Parser/ParsingService.cs index e95d2ef32..59a598369 100644 --- a/src/NzbDrone.Core/Parser/ParsingService.cs +++ b/src/NzbDrone.Core/Parser/ParsingService.cs @@ -18,8 +18,6 @@ public interface IParsingService RemoteAlbum Map(ParsedAlbumInfo parsedAlbumInfo, SearchCriteriaBase searchCriteria = null); RemoteAlbum Map(ParsedAlbumInfo parsedAlbumInfo, int artistId, IEnumerable albumIds); List GetAlbums(ParsedAlbumInfo parsedAlbumInfo, Artist artist, SearchCriteriaBase searchCriteria = null); - - // Music stuff here Album GetLocalAlbum(string filename, Artist artist); } @@ -29,18 +27,21 @@ public class ParsingService : IParsingService private readonly IAlbumService _albumService; private readonly ITrackService _trackService; private readonly IMediaFileService _mediaFileService; + private readonly IAlbumYearMatcher _yearMatcher; private readonly Logger _logger; public ParsingService(ITrackService trackService, IArtistService artistService, IAlbumService albumService, IMediaFileService mediaFileService, + IAlbumYearMatcher yearMatcher, Logger logger) { _albumService = albumService; _artistService = artistService; _trackService = trackService; _mediaFileService = mediaFileService; + _yearMatcher = yearMatcher; _logger = logger; } @@ -149,21 +150,23 @@ public List GetAlbums(ParsedAlbumInfo parsedAlbumInfo, Artist artist, Sea return _albumService.GetAlbumsByArtist(artist.Id); } + var releaseYear = parsedAlbumInfo.ReleaseYear; + if (searchCriteria != null) { - albumInfo = searchCriteria.Albums.ExclusiveOrDefault(e => e.Title == albumTitle); + albumInfo = FindAlbumInSearchCriteria(searchCriteria.Albums, albumTitle, releaseYear); } if (albumInfo == null) { - // TODO: Search by Title and Year instead of just Title when matching - albumInfo = _albumService.FindByTitle(artist.ArtistMetadataId, parsedAlbumInfo.AlbumTitle); + albumInfo = _albumService.FindByTitleAndYear(artist.ArtistMetadataId, parsedAlbumInfo.AlbumTitle, releaseYear); } if (albumInfo == null) { - _logger.Debug("Trying inexact album match for {0}", parsedAlbumInfo.AlbumTitle); - albumInfo = _albumService.FindByTitleInexact(artist.ArtistMetadataId, parsedAlbumInfo.AlbumTitle); + var yearInfo = releaseYear.HasValue ? $" ({releaseYear.Value})" : string.Empty; + _logger.Debug("Trying inexact album match for {0}{1}", parsedAlbumInfo.AlbumTitle, yearInfo); + albumInfo = _albumService.FindByTitleAndYearInexact(artist.ArtistMetadataId, parsedAlbumInfo.AlbumTitle, releaseYear); } if (albumInfo != null) @@ -178,6 +181,59 @@ public List GetAlbums(ParsedAlbumInfo parsedAlbumInfo, Artist artist, Sea return result; } + private Album FindAlbumInSearchCriteria(List albums, string albumTitle, int? releaseYear) + { + var matchingAlbums = albums.Where(e => e.Title == albumTitle).ToList(); + + if (!matchingAlbums.Any()) + { + return null; + } + + if (matchingAlbums.Count == 1) + { + var album = matchingAlbums.First(); + + if (releaseYear.HasValue) + { + var matchResult = _yearMatcher.Match(album, releaseYear); + if (!matchResult.IsMatch) + { + _logger.Debug("Album '{0}' matched by title but {1}", album.Title, matchResult.RejectionReason); + return null; + } + } + + return album; + } + + // Multiple albums with same title - use year to disambiguate + if (releaseYear.HasValue) + { + var bestMatch = matchingAlbums + .Select(a => new + { + Album = a, + YearResult = _yearMatcher.Match(a, releaseYear) + }) + .Where(x => x.YearResult.IsMatch) + .OrderByDescending(x => x.YearResult.ScoreAdjustment) + .FirstOrDefault(); + + if (bestMatch != null) + { + return bestMatch.Album; + } + + _logger.Debug("Multiple albums named '{0}' found but none match year {1}", albumTitle, releaseYear); + return null; + } + + // No year to disambiguate, cannot safely choose from multiple matches + _logger.Trace("Multiple albums named '{0}' found without year to disambiguate, unable to determine correct match", albumTitle); + return null; + } + public RemoteAlbum Map(ParsedAlbumInfo parsedAlbumInfo, int artistId, IEnumerable albumIds) { return new RemoteAlbum