diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceFixture.cs new file mode 100644 index 000000000..3634498ab --- /dev/null +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceFixture.cs @@ -0,0 +1,810 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using NLog; +using NUnit.Framework; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.LlmMatching; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests.LlmMatchingTests +{ + [TestFixture] + public class LlmSeriesMatchingServiceFixture + { + private Mock _configService; + private Mock _mockLlmService; + private List _testSeries; + + [SetUp] + public void Setup() + { + _configService = new Mock(); + _mockLlmService = new Mock(); + + _testSeries = new List + { + new Series + { + Id = 1, + TvdbId = 81189, + Title = "Breaking Bad", + CleanTitle = "breakingbad", + Year = 2008, + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 2, + TvdbId = 121361, + Title = "Game of Thrones", + CleanTitle = "gameofthrones", + Year = 2011, + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 3, + TvdbId = 267440, + Title = "Attack on Titan", + CleanTitle = "attackontitan", + Year = 2013, + SeriesType = SeriesTypes.Anime + }, + new Series + { + Id = 4, + TvdbId = 153021, + Title = "The Walking Dead", + CleanTitle = "thewalkingdead", + Year = 2010, + SeriesType = SeriesTypes.Standard + } + }; + } + + [Test] + public void IsEnabled_should_return_false_when_llm_matching_disabled() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(false); + _configService.Setup(s => s.OpenAiApiKey).Returns("test-key"); + + _mockLlmService.Setup(s => s.IsEnabled).Returns(false); + + _mockLlmService.Object.IsEnabled.Should().BeFalse(); + } + + [Test] + public void IsEnabled_should_return_false_when_api_key_is_empty() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(true); + _configService.Setup(s => s.OpenAiApiKey).Returns(string.Empty); + + _mockLlmService.Setup(s => s.IsEnabled).Returns(false); + + _mockLlmService.Object.IsEnabled.Should().BeFalse(); + } + + [Test] + public void IsEnabled_should_return_false_when_api_key_is_null() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(true); + _configService.Setup(s => s.OpenAiApiKey).Returns((string)null); + + _mockLlmService.Setup(s => s.IsEnabled).Returns(false); + + _mockLlmService.Object.IsEnabled.Should().BeFalse(); + } + + [Test] + public void IsEnabled_should_return_true_when_properly_configured() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(true); + _configService.Setup(s => s.OpenAiApiKey).Returns("sk-test-key-12345"); + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + + _mockLlmService.Object.IsEnabled.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_return_null_when_disabled() + { + _mockLlmService.Setup(s => s.IsEnabled).Returns(false); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync((LlmMatchResult)null); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_return_null_when_no_series_available() + { + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync((LlmMatchResult)null); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync("Breaking.Bad.S01E01", new List()); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_return_null_when_title_is_empty() + { + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync(string.Empty, It.IsAny>())) + .ReturnsAsync((LlmMatchResult)null); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync(string.Empty, _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_return_match_with_high_confidence() + { + var expectedSeries = _testSeries.First(s => s.TvdbId == 81189); + var expectedResult = new LlmMatchResult + { + Series = expectedSeries, + Confidence = 0.95, + Reasoning = "Direct title match after removing dots and quality tags" + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync("Breaking.Bad.S01E01.720p.WEB-DL", _testSeries)) + .ReturnsAsync(expectedResult); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync("Breaking.Bad.S01E01.720p.WEB-DL", _testSeries); + + result.Should().NotBeNull(); + result.Series.Should().Be(expectedSeries); + result.Confidence.Should().Be(0.95); + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_return_unsuccessful_match_with_low_confidence() + { + var expectedSeries = _testSeries.First(s => s.TvdbId == 81189); + var expectedResult = new LlmMatchResult + { + Series = expectedSeries, + Confidence = 0.45, + Reasoning = "Possible match but title is ambiguous" + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync("Bad.Show.S01E01", _testSeries)) + .ReturnsAsync(expectedResult); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync("Bad.Show.S01E01", _testSeries); + + result.Should().NotBeNull(); + result.Confidence.Should().Be(0.45); + result.IsSuccessfulMatch.Should().BeFalse(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_handle_anime_alternate_titles() + { + var expectedSeries = _testSeries.First(s => s.TvdbId == 267440); + var expectedResult = new LlmMatchResult + { + Series = expectedSeries, + Confidence = 0.92, + Reasoning = "Shingeki no Kyojin is the Japanese title for Attack on Titan" + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync("Shingeki.no.Kyojin.S04E01.1080p", _testSeries)) + .ReturnsAsync(expectedResult); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync("Shingeki.no.Kyojin.S04E01.1080p", _testSeries); + + result.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(267440); + result.Series.Title.Should().Be("Attack on Titan"); + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_include_alternatives_when_uncertain() + { + var primarySeries = _testSeries.First(s => s.TvdbId == 81189); + var alternativeSeries = _testSeries.First(s => s.TvdbId == 153021); + + var expectedResult = new LlmMatchResult + { + Series = primarySeries, + Confidence = 0.55, + Reasoning = "Could be Breaking Bad but title is unclear", + Alternatives = new List + { + new AlternativeMatch + { + Series = alternativeSeries, + Confidence = 0.35, + Reasoning = "Walking Dead also possible" + } + } + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync("The.Bad.Dead.S01E01", _testSeries)) + .ReturnsAsync(expectedResult); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync("The.Bad.Dead.S01E01", _testSeries); + + result.Should().NotBeNull(); + result.IsSuccessfulMatch.Should().BeFalse(); + result.Alternatives.Should().HaveCount(1); + result.Alternatives.First().Series.TvdbId.Should().Be(153021); + } + + [Test] + public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_should_return_null_when_disabled() + { + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad", + ReleaseTitle = "Breaking.Bad.S01E01.720p.WEB-DL", + SeasonNumber = 1, + EpisodeNumbers = new[] { 1 } + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(false); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync((LlmMatchResult)null); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_should_use_parsed_metadata() + { + // Note: IsAbsoluteNumbering and IsDaily are computed properties, we don't set them directly + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad German", + ReleaseTitle = "Breaking.Bad.German.S01E01.720p.WEB-DL", + SeasonNumber = 1, + EpisodeNumbers = new[] { 1 } + }; + + var expectedSeries = _testSeries.First(s => s.TvdbId == 81189); + var expectedResult = new LlmMatchResult + { + Series = expectedSeries, + Confidence = 0.88, + Reasoning = "Recognized 'German' as language tag, matched to Breaking Bad" + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync(parsedInfo, _testSeries)) + .ReturnsAsync(expectedResult); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_with_anime_ParsedEpisodeInfo_should_consider_series_type() + { + // Note: IsAbsoluteNumbering is computed from AbsoluteEpisodeNumbers + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Shingeki no Kyojin", + ReleaseTitle = "[SubGroup] Shingeki no Kyojin - 01 [1080p]", + SeasonNumber = 1, + AbsoluteEpisodeNumbers = new[] { 1 } + }; + + var expectedSeries = _testSeries.First(s => s.TvdbId == 267440); + var expectedResult = new LlmMatchResult + { + Series = expectedSeries, + Confidence = 0.94, + Reasoning = "Japanese anime title matched to Attack on Titan" + }; + + _mockLlmService.Setup(s => s.IsEnabled).Returns(true); + _mockLlmService + .Setup(s => s.TryMatchSeriesAsync(parsedInfo, _testSeries)) + .ReturnsAsync(expectedResult); + + var result = await _mockLlmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result.Should().NotBeNull(); + result.Series.SeriesType.Should().Be(SeriesTypes.Anime); + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_should_return_true_when_confidence_at_threshold() + { + var result = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.7 + }; + + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_should_return_true_when_confidence_above_threshold() + { + var result = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.95 + }; + + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_should_return_false_when_confidence_below_threshold() + { + var result = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.69 + }; + + result.IsSuccessfulMatch.Should().BeFalse(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_should_return_false_when_series_is_null() + { + var result = new LlmMatchResult + { + Series = null, + Confidence = 0.95 + }; + + result.IsSuccessfulMatch.Should().BeFalse(); + } + + [Test] + public void LlmMatchResult_should_initialize_alternatives_as_empty_list() + { + var result = new LlmMatchResult(); + + result.Alternatives.Should().NotBeNull(); + result.Alternatives.Should().BeEmpty(); + } + } + + [TestFixture] + public class CachedLlmSeriesMatchingServiceFixture + { + private Mock _configService; + private Mock _httpClientFactory; + private Mock _innerService; + private CachedLlmSeriesMatchingService _subject; + private List _testSeries; + + [SetUp] + public void Setup() + { + _configService = new Mock(); + _httpClientFactory = new Mock(); + + _innerService = new Mock( + _configService.Object, + _httpClientFactory.Object, + Mock.Of()); + + _configService.Setup(s => s.LlmCacheEnabled).Returns(true); + _configService.Setup(s => s.LlmCacheDurationHours).Returns(24); + + _subject = new CachedLlmSeriesMatchingService( + _innerService.Object, + _configService.Object, + Mock.Of()); + + _testSeries = new List + { + new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" }, + new Series { Id = 2, TvdbId = 121361, Title = "Game of Thrones" } + }; + } + + [Test] + public void IsEnabled_should_delegate_to_inner_service() + { + _innerService.Setup(s => s.IsEnabled).Returns(true); + + _subject.IsEnabled.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_bypass_cache_when_caching_disabled() + { + _configService.Setup(s => s.LlmCacheEnabled).Returns(false); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // Call twice + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + // Inner service should be called twice (no caching) + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(2)); + } + + [Test] + public async Task TryMatchSeriesAsync_should_cache_results_when_enabled() + { + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9, + Reasoning = "Test match" + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // Call twice with same title + var result1 = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + var result2 = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + // Inner service should only be called once (second call uses cache) + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Once); + + result1.Should().NotBeNull(); + result2.Should().NotBeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_call_inner_service_for_different_titles() + { + var result1 = new LlmMatchResult + { + Series = _testSeries[0], + Confidence = 0.9 + }; + + var result2 = new LlmMatchResult + { + Series = _testSeries[1], + Confidence = 0.85 + }; + + _innerService + .SetupSequence(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(result1) + .ReturnsAsync(result2); + + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Game.of.Thrones.S01E01", _testSeries); + + // Inner service should be called twice (different titles) + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(2)); + } + } + + [TestFixture] + public class RateLimitedLlmSeriesMatchingServiceFixture + { + private Mock _configService; + private Mock _httpClientFactory; + private Mock _innerService; + private RateLimitedLlmSeriesMatchingService _subject; + private List _testSeries; + + [SetUp] + public void Setup() + { + _configService = new Mock(); + _httpClientFactory = new Mock(); + + var openAiService = new Mock( + _configService.Object, + _httpClientFactory.Object, + Mock.Of()); + + _innerService = new Mock( + openAiService.Object, + _configService.Object, + Mock.Of()); + + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(60); + + _subject = new RateLimitedLlmSeriesMatchingService( + _innerService.Object, + _configService.Object, + Mock.Of()); + + _testSeries = new List + { + new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" } + }; + } + + [Test] + public void IsEnabled_should_delegate_to_inner_service() + { + _innerService.Setup(s => s.IsEnabled).Returns(true); + + _subject.IsEnabled.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_should_allow_calls_within_rate_limit() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(5); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // Make 5 calls (within limit) + for (var i = 0; i < 5; i++) + { + var result = await _subject.TryMatchSeriesAsync($"Title{i}.S01E01", _testSeries); + result.Should().NotBeNull(); + } + + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(5)); + } + + [Test] + public async Task TryMatchSeriesAsync_should_return_null_when_rate_limit_exceeded() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(2); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // Make 2 calls (at limit) + await _subject.TryMatchSeriesAsync("Title1.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Title2.S01E01", _testSeries); + + // Third call should be rate limited + var result = await _subject.TryMatchSeriesAsync("Title3.S01E01", _testSeries); + + result.Should().BeNull(); + + // Inner service should only be called twice + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(2)); + } + + [Test] + public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_should_respect_rate_limit() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(1); + + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad", + ReleaseTitle = "Breaking.Bad.S01E01" + }; + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // First call should succeed + var result1 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + result1.Should().NotBeNull(); + + // Second call should be rate limited + var result2 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + result2.Should().BeNull(); + } + } + + [TestFixture] + public class ParsingServiceLlmIntegrationFixture + { + private Mock _llmService; + private List _testSeries; + + [SetUp] + public void Setup() + { + _llmService = new Mock(); + + _testSeries = new List + { + new Series + { + Id = 1, + TvdbId = 81189, + Title = "Breaking Bad", + CleanTitle = "breakingbad", + Year = 2008 + }, + new Series + { + Id = 2, + TvdbId = 267440, + Title = "Attack on Titan", + CleanTitle = "attackontitan", + Year = 2013, + SeriesType = SeriesTypes.Anime + } + }; + } + + [Test] + public void LlmService_should_not_be_called_when_traditional_matching_succeeds() + { + // This tests that LLM is only used as fallback + _llmService.Setup(s => s.IsEnabled).Returns(true); + + // LLM should never be called if traditional matching works + _llmService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Never); + + _llmService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Never); + } + + [Test] + public async Task LlmService_should_be_called_when_traditional_matching_fails() + { + _llmService.Setup(s => s.IsEnabled).Returns(true); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.88, + Reasoning = "Matched after removing language tag" + }; + + _llmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // Simulate call that would happen from ParsingService + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad German", + ReleaseTitle = "Breaking.Bad.German.S01E01.720p" + }; + + var result = await _llmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result.Should().NotBeNull(); + result.IsSuccessfulMatch.Should().BeTrue(); + result.Series.Title.Should().Be("Breaking Bad"); + } + + [Test] + public async Task LlmService_should_handle_anime_alternate_titles() + { + _llmService.Setup(s => s.IsEnabled).Returns(true); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(s => s.TvdbId == 267440), + Confidence = 0.92, + Reasoning = "Shingeki no Kyojin is the Japanese title for Attack on Titan" + }; + + _llmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + var result = await _llmService.Object.TryMatchSeriesAsync( + "[SubGroup] Shingeki no Kyojin - 01 [1080p].mkv", + _testSeries); + + result.Should().NotBeNull(); + result.Series.Title.Should().Be("Attack on Titan"); + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public async Task LlmService_should_return_null_when_disabled() + { + _llmService.Setup(s => s.IsEnabled).Returns(false); + _llmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync((LlmMatchResult)null); + + var result = await _llmService.Object.TryMatchSeriesAsync("Some.Title.S01E01", _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task LlmService_low_confidence_should_not_auto_match() + { + _llmService.Setup(s => s.IsEnabled).Returns(true); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.45, + Reasoning = "Title is ambiguous, multiple possible matches", + Alternatives = new List + { + new AlternativeMatch + { + Series = _testSeries[1], + Confidence = 0.30, + Reasoning = "Could also be this series" + } + } + }; + + _llmService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + var result = await _llmService.Object.TryMatchSeriesAsync("Ambiguous.Title.S01E01", _testSeries); + + result.Should().NotBeNull(); + result.IsSuccessfulMatch.Should().BeFalse(); + result.Alternatives.Should().HaveCount(1); + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceIntegrationFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceIntegrationFixture.cs new file mode 100644 index 000000000..646bfc669 --- /dev/null +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceIntegrationFixture.cs @@ -0,0 +1,605 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using NLog; +using NUnit.Framework; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.LlmMatching; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests.LlmMatchingTests +{ + /// + /// Integration tests for OpenAiSeriesMatchingService. + /// These tests verify service logic without making actual API calls. + /// + [TestFixture] + public class OpenAiSeriesMatchingServiceIntegrationFixture + { + private Mock _configService; + private Mock _httpClientFactory; + private OpenAiSeriesMatchingService _subject; + private List _testSeries; + + [SetUp] + public void Setup() + { + _configService = new Mock(); + _httpClientFactory = new Mock(); + + _configService.Setup(s => s.LlmMatchingEnabled).Returns(true); + _configService.Setup(s => s.OpenAiApiKey).Returns("test-api-key-12345"); + _configService.Setup(s => s.OpenAiApiEndpoint).Returns("https://api.openai.com/v1/chat/completions"); + _configService.Setup(s => s.OpenAiModel).Returns("gpt-4o-mini"); + _configService.Setup(s => s.LlmConfidenceThreshold).Returns(0.7); + + _subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + _testSeries = new List + { + new Series + { + Id = 1, + TvdbId = 81189, + Title = "Breaking Bad", + CleanTitle = "breakingbad", + Year = 2008, + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 2, + TvdbId = 121361, + Title = "Game of Thrones", + CleanTitle = "gameofthrones", + Year = 2011, + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 3, + TvdbId = 267440, + Title = "Attack on Titan", + CleanTitle = "attackontitan", + Year = 2013, + SeriesType = SeriesTypes.Anime + } + }; + } + + [Test] + public void IsEnabled_returns_true_when_properly_configured() + { + _subject.IsEnabled.Should().BeTrue(); + } + + [Test] + public void IsEnabled_returns_false_when_disabled_in_config() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(false); + + var subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + subject.IsEnabled.Should().BeFalse(); + } + + [Test] + public void IsEnabled_returns_false_when_api_key_empty() + { + _configService.Setup(s => s.OpenAiApiKey).Returns(string.Empty); + + var subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + subject.IsEnabled.Should().BeFalse(); + } + + [Test] + public void IsEnabled_returns_false_when_api_key_null() + { + _configService.Setup(s => s.OpenAiApiKey).Returns((string)null); + + var subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + subject.IsEnabled.Should().BeFalse(); + } + + [Test] + public void IsEnabled_returns_false_when_api_key_whitespace() + { + _configService.Setup(s => s.OpenAiApiKey).Returns(" "); + + var subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + subject.IsEnabled.Should().BeFalse(); + } + + [Test] + public async Task TryMatchSeriesAsync_string_returns_null_when_disabled() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(false); + + var subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + var result = await subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_string_returns_null_when_title_empty() + { + var result = await _subject.TryMatchSeriesAsync(string.Empty, _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_string_returns_null_when_title_null() + { + var result = await _subject.TryMatchSeriesAsync((string)null, _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_string_returns_null_when_series_list_empty() + { + var result = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", new List()); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_string_returns_null_when_series_list_null() + { + var result = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", null); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_ParsedEpisodeInfo_returns_null_when_disabled() + { + _configService.Setup(s => s.LlmMatchingEnabled).Returns(false); + + var subject = new OpenAiSeriesMatchingService( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad", + ReleaseTitle = "Breaking.Bad.S01E01" + }; + + var result = await subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_ParsedEpisodeInfo_returns_null_when_no_title() + { + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = null, + ReleaseTitle = null + }; + + var result = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result.Should().BeNull(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_true_at_threshold() + { + var result = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.7 + }; + + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_true_above_threshold() + { + var result = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.95 + }; + + result.IsSuccessfulMatch.Should().BeTrue(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_false_below_threshold() + { + var result = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.69 + }; + + result.IsSuccessfulMatch.Should().BeFalse(); + } + + [Test] + public void LlmMatchResult_IsSuccessfulMatch_false_when_series_null() + { + var result = new LlmMatchResult + { + Series = null, + Confidence = 0.95 + }; + + result.IsSuccessfulMatch.Should().BeFalse(); + } + + [Test] + public void LlmMatchResult_Alternatives_initialized_empty() + { + var result = new LlmMatchResult(); + + result.Alternatives.Should().NotBeNull(); + result.Alternatives.Should().BeEmpty(); + } + + [Test] + public void LlmMatchResult_can_hold_multiple_alternatives() + { + var result = new LlmMatchResult + { + Series = _testSeries[0], + Confidence = 0.6, + Alternatives = new List + { + new AlternativeMatch { Series = _testSeries[1], Confidence = 0.4 }, + new AlternativeMatch { Series = _testSeries[2], Confidence = 0.3 } + } + }; + + result.Alternatives.Should().HaveCount(2); + } + } + + /// + /// Tests for the caching decorator service. + /// + [TestFixture] + public class CachedLlmSeriesMatchingServiceIntegrationFixture + { + private Mock _configService; + private Mock _httpClientFactory; + private Mock _innerService; + private CachedLlmSeriesMatchingService _subject; + private List _testSeries; + + [SetUp] + public void Setup() + { + _configService = new Mock(); + _httpClientFactory = new Mock(); + + _configService.Setup(s => s.LlmCacheEnabled).Returns(true); + _configService.Setup(s => s.LlmCacheDurationHours).Returns(24); + + _innerService = new Mock( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + _innerService.Setup(s => s.IsEnabled).Returns(true); + + _subject = new CachedLlmSeriesMatchingService( + _innerService.Object, + _configService.Object, + LogManager.GetCurrentClassLogger()); + + _testSeries = new List + { + new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" }, + new Series { Id = 2, TvdbId = 121361, Title = "Game of Thrones" } + }; + } + + [Test] + public void IsEnabled_delegates_to_inner_service() + { + _subject.IsEnabled.Should().BeTrue(); + + _innerService.Verify(s => s.IsEnabled, Times.Once); + } + + [Test] + public async Task TryMatchSeriesAsync_calls_inner_service_on_cache_miss() + { + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9, + Reasoning = "Test" + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + var result = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + result.Should().NotBeNull(); + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Once); + } + + [Test] + public async Task TryMatchSeriesAsync_returns_cached_result_on_cache_hit() + { + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9, + Reasoning = "Test" + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // First call - cache miss + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + // Second call - cache hit + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + // Inner service should only be called once + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Once); + } + + [Test] + public async Task TryMatchSeriesAsync_bypasses_cache_when_disabled() + { + _configService.Setup(s => s.LlmCacheEnabled).Returns(false); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + // Inner service should be called twice (no caching) + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(2)); + } + + [Test] + public async Task TryMatchSeriesAsync_uses_different_cache_keys_for_different_titles() + { + var result1 = new LlmMatchResult { Series = _testSeries[0], Confidence = 0.9 }; + var result2 = new LlmMatchResult { Series = _testSeries[1], Confidence = 0.85 }; + + _innerService + .SetupSequence(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(result1) + .ReturnsAsync(result2); + + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Game.of.Thrones.S01E01", _testSeries); + + // Both should result in calls to inner service + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(2)); + } + } + + /// + /// Tests for the rate limiting decorator service. + /// + [TestFixture] + public class RateLimitedLlmSeriesMatchingServiceIntegrationFixture + { + private Mock _configService; + private Mock _httpClientFactory; + private Mock _innerService; + private RateLimitedLlmSeriesMatchingService _subject; + private List _testSeries; + + [SetUp] + public void Setup() + { + _configService = new Mock(); + _httpClientFactory = new Mock(); + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(60); + _configService.Setup(s => s.LlmCacheEnabled).Returns(true); + _configService.Setup(s => s.LlmCacheDurationHours).Returns(24); + + var openAiMock = new Mock( + _configService.Object, + _httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + _innerService = new Mock( + openAiMock.Object, + _configService.Object, + LogManager.GetCurrentClassLogger()); + + _innerService.Setup(s => s.IsEnabled).Returns(true); + + _subject = new RateLimitedLlmSeriesMatchingService( + _innerService.Object, + _configService.Object, + LogManager.GetCurrentClassLogger()); + + _testSeries = new List + { + new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" } + }; + } + + [Test] + public void IsEnabled_delegates_to_inner_service() + { + _subject.IsEnabled.Should().BeTrue(); + } + + [Test] + public async Task TryMatchSeriesAsync_allows_calls_within_limit() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(10); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + var results = new List(); + + for (var i = 0; i < 10; i++) + { + var result = await _subject.TryMatchSeriesAsync($"Title{i}.S01E01", _testSeries); + results.Add(result); + } + + results.Should().AllSatisfy(r => r.Should().NotBeNull()); + } + + [Test] + public async Task TryMatchSeriesAsync_blocks_calls_over_limit() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(3); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // Make 3 calls (at limit) + await _subject.TryMatchSeriesAsync("Title1.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Title2.S01E01", _testSeries); + await _subject.TryMatchSeriesAsync("Title3.S01E01", _testSeries); + + // 4th call should be blocked + var result = await _subject.TryMatchSeriesAsync("Title4.S01E01", _testSeries); + + result.Should().BeNull(); + + // Only 3 calls should have reached inner service + _innerService.Verify( + s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>()), + Times.Exactly(3)); + } + + [Test] + public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_respects_rate_limit() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(2); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad", + ReleaseTitle = "Breaking.Bad.S01E01" + }; + + // First two calls succeed + var result1 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + var result2 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + // Third call blocked + var result3 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + result1.Should().NotBeNull(); + result2.Should().NotBeNull(); + result3.Should().BeNull(); + } + + [Test] + public async Task TryMatchSeriesAsync_rate_limit_applies_across_both_methods() + { + _configService.Setup(s => s.LlmMaxCallsPerHour).Returns(2); + + var expectedResult = new LlmMatchResult + { + Series = _testSeries.First(), + Confidence = 0.9 + }; + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + _innerService + .Setup(s => s.TryMatchSeriesAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(expectedResult); + + // One call with string + await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries); + + // One call with ParsedEpisodeInfo + var parsedInfo = new ParsedEpisodeInfo { SeriesTitle = "Test", ReleaseTitle = "Test.S01E01" }; + await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + // Third call (either type) should be blocked + var result = await _subject.TryMatchSeriesAsync("Another.Title.S01E01", _testSeries); + + result.Should().BeNull(); + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/RealOpenAiIntegrationFixture.cs b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/RealOpenAiIntegrationFixture.cs new file mode 100644 index 000000000..17e4c9afa --- /dev/null +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/RealOpenAiIntegrationFixture.cs @@ -0,0 +1,992 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Text.Json; +using System.Threading.Tasks; +using FluentAssertions; +using Moq; +using NLog; +using NUnit.Framework; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.LlmMatching; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests.LlmMatchingTests +{ + /// + /// Real integration tests that make actual API calls to OpenAI. + /// These tests require a valid API key in appsettings.llm.local.json. + /// + /// SETUP: + /// 1. Copy appsettings.llm.template.json to appsettings.llm.local.json + /// 2. Add your OpenAI API key to the local file + /// 3. Ensure the file is set to "Copy to Output Directory: Copy if newer" in Visual Studio + /// 4. Run tests with: dotnet test --filter "FullyQualifiedName~RealOpenAiIntegrationFixture" + /// + /// NOTE: These tests are marked [Explicit] and won't run during normal test execution. + /// They make real API calls which incur costs and require network access. + /// + [TestFixture] + [Explicit("Requires real OpenAI API key and makes actual API calls")] + [Category("Integration")] + [Category("LlmMatching")] + [Category("ExternalApi")] + public class RealOpenAiIntegrationFixture + { + private OpenAiSeriesMatchingService _subject; + private List _testSeries; + private LlmTestSettings _settings; + private bool _isConfigured; + + [OneTimeSetUp] + public void OneTimeSetup() + { + _settings = LoadSettings(); + _isConfigured = !string.IsNullOrWhiteSpace(_settings?.OpenAiApiKey) && + !_settings.OpenAiApiKey.StartsWith("sk-your-"); + + if (!_isConfigured) + { + TestContext.WriteLine("========================================"); + TestContext.WriteLine("WARNING: OpenAI API key not configured."); + TestContext.WriteLine("========================================"); + TestContext.WriteLine(""); + TestContext.WriteLine("Setup steps:"); + TestContext.WriteLine("1. Copy appsettings.llm.template.json to appsettings.llm.local.json"); + TestContext.WriteLine("2. Add your OpenAI API key to appsettings.llm.local.json"); + TestContext.WriteLine("3. In Visual Studio: Right-click the file -> Properties"); + TestContext.WriteLine(" Set 'Copy to Output Directory' = 'Copy if newer'"); + TestContext.WriteLine(""); + TestContext.WriteLine("Or set environment variable: OPENAI_API_KEY=sk-..."); + TestContext.WriteLine(""); + TestContext.WriteLine("Searched locations:"); + foreach (var path in GetSearchPaths()) + { + var exists = File.Exists(path) ? "[FOUND]" : "[NOT FOUND]"; + TestContext.WriteLine($" {exists} {path}"); + } + } + else + { + TestContext.WriteLine($"OpenAI API configured. Using model: {_settings.OpenAiModel}"); + } + } + + [SetUp] + public void Setup() + { + if (!_isConfigured) + { + Assert.Ignore("OpenAI API key not configured. Skipping real API tests."); + return; + } + + var configService = new Mock(); + configService.Setup(s => s.LlmMatchingEnabled).Returns(true); + configService.Setup(s => s.OpenAiApiKey).Returns(_settings.OpenAiApiKey); + configService.Setup(s => s.OpenAiApiEndpoint).Returns(_settings.OpenAiApiEndpoint); + configService.Setup(s => s.OpenAiModel).Returns(_settings.OpenAiModel); + configService.Setup(s => s.LlmConfidenceThreshold).Returns(_settings.ConfidenceThreshold); + + var httpClientFactory = new Mock(); + httpClientFactory + .Setup(f => f.CreateClient(It.IsAny())) + .Returns(new HttpClient()); + + _subject = new OpenAiSeriesMatchingService( + configService.Object, + httpClientFactory.Object, + LogManager.GetCurrentClassLogger()); + + // Setup test series library - simulates a real user's library + _testSeries = new List + { + new Series + { + Id = 1, + TvdbId = 81189, + Title = "Breaking Bad", + CleanTitle = "breakingbad", + Year = 2008, + Network = "AMC", + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 2, + TvdbId = 121361, + Title = "Game of Thrones", + CleanTitle = "gameofthrones", + Year = 2011, + Network = "HBO", + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 3, + TvdbId = 267440, + Title = "Attack on Titan", + CleanTitle = "attackontitan", + Year = 2013, + SeriesType = SeriesTypes.Anime + }, + new Series + { + Id = 4, + TvdbId = 153021, + Title = "The Walking Dead", + CleanTitle = "thewalkingdead", + Year = 2010, + Network = "AMC", + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 5, + TvdbId = 295759, + Title = "Stranger Things", + CleanTitle = "strangerthings", + Year = 2016, + Network = "Netflix", + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 6, + TvdbId = 305288, + Title = "Westworld", + CleanTitle = "westworld", + Year = 2016, + Network = "HBO", + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 7, + TvdbId = 78804, + Title = "Doctor Who", + CleanTitle = "doctorwho", + Year = 2005, + Network = "BBC", + SeriesType = SeriesTypes.Standard + }, + new Series + { + Id = 8, + TvdbId = 73255, + Title = "Doctor Who", + CleanTitle = "doctorwho", + Year = 1963, + Network = "BBC", + SeriesType = SeriesTypes.Standard + } + }; + } + + [Test] + public async Task Should_match_standard_release_title() + { + // Arrange + var releaseTitle = "Breaking.Bad.S01E01.720p.BluRay.x264-DEMAND"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + result.Series.Title.Should().Be("Breaking Bad"); + result.Confidence.Should().BeGreaterOrEqualTo(0.7); + result.IsSuccessfulMatch.Should().BeTrue(); + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_match_release_with_dots_as_spaces() + { + // Arrange + var releaseTitle = "Game.of.Thrones.S08E06.The.Iron.Throne.1080p.AMZN.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(121361); + result.Confidence.Should().BeGreaterOrEqualTo(0.7); + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_match_anime_with_japanese_title() + { + // Arrange - Japanese title for Attack on Titan + var releaseTitle = "[SubGroup] Shingeki no Kyojin - 01 [1080p][HEVC]"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(267440); + result.Series.Title.Should().Be("Attack on Titan"); + result.Confidence.Should().BeGreaterOrEqualTo(0.7); + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_disambiguate_by_year() + { + // Arrange - Should match 2005 Doctor Who, not 1963 + var releaseTitle = "Doctor.Who.2005.S13E01.720p.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(78804); // 2005 version + result.Series.Year.Should().Be(2005); + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_release_with_language_tag() + { + // Arrange - German release + var releaseTitle = "Breaking.Bad.S01E01.GERMAN.720p.BluRay.x264"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + result.Reasoning.Should().NotBeNullOrEmpty(); + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_release_with_scene_group_tags() + { + // Arrange + var releaseTitle = "Stranger.Things.S04E09.Chapter.Nine.The.Piggyback.2160p.NF.WEB-DL.DDP5.1.Atmos.DV.HDR.H.265-FLUX"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(295759); + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_abbreviated_title() + { + // Arrange - TWD is a common abbreviation for The Walking Dead + var releaseTitle = "TWD.S11E24.Rest.in.Peace.1080p.AMZN.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + // Note: LLM might or might not recognize TWD abbreviation + // This test verifies the LLM can handle ambiguous cases + if (result.Series != null) + { + TestContext.WriteLine($"LLM recognized TWD as: {result.Series.Title}"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_miscoded_german_umlauts_utf8_as_latin1() + { + // Arrange - "Für" miscoded as "Für" (UTF-8 bytes interpreted as Latin-1) + // This happens when UTF-8 encoded text is read as ISO-8859-1 + var releaseTitle = "Breaking.Bad.S01E01.German.Für.immer.720p.BluRay.x264"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine($"LLM handled miscoded umlaut 'ü' (should be 'ü')"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_miscoded_german_umlauts_various() + { + // Arrange - Various miscoded German umlauts + // ä -> ä, ö -> ö, ü -> ü, ß -> ß + var releaseTitle = "Breaking.Bad.S02E01.Grüße.aus.Köln.GERMAN.720p.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled multiple miscoded umlauts: 'ü'='ü', 'ß'='ß', 'ö'='ö'"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_correct_german_umlauts() + { + // Arrange - Correctly encoded German umlauts for comparison + var releaseTitle = "Breaking.Bad.S01E01.Grüße.aus.Köln.GERMAN.720p.BluRay.x264"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled correct German umlauts: ü, ö, ß"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_japanese_characters_in_anime_release() + { + // Arrange - Japanese title with kanji/hiragana + // 進撃の巨人 = Shingeki no Kyojin = Attack on Titan + var releaseTitle = "[SubGroup] 進撃の巨人 - 01 [1080p][HEVC].mkv"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM recognized Japanese '進撃の巨人' as: {result.Series.Title}"); + + // Should match Attack on Titan + if (result.Series.TvdbId == 267440) + { + TestContext.WriteLine("SUCCESS: Correctly identified as Attack on Titan"); + } + } + else + { + TestContext.WriteLine("LLM could not match Japanese kanji title"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_mixed_japanese_english_title() + { + // Arrange - Mixed Japanese and English (common in anime releases) + var releaseTitle = "[Erai-raws] Shingeki no Kyojin - The Final Season - 01 [1080p][HEVC].mkv"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(267440); + + TestContext.WriteLine("LLM handled mixed Japanese/English anime title"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_chinese_characters() + { + // Arrange - Chinese title for Attack on Titan (進擊的巨人 - Traditional Chinese) + var releaseTitle = "[字幕组] 進擊的巨人 - 01 [1080p].mkv"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM recognized Chinese '進擊的巨人' as: {result.Series.Title}"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_korean_characters() + { + // Arrange - Korean title (게임 오브 스론스 = Game of Thrones) + var releaseTitle = "게임.오브.스론스.S08E06.1080p.WEB-DL.KOR"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM recognized Korean '게임 오브 스론스' as: {result.Series.Title}"); + + if (result.Series.TvdbId == 121361) + { + TestContext.WriteLine("SUCCESS: Correctly identified as Game of Thrones"); + } + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_cyrillic_characters() + { + // Arrange - Russian title (Во все тяжкие = Breaking Bad) + var releaseTitle = "Во.все.тяжкие.S01E01.720p.BluRay.RUS"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM recognized Cyrillic 'Во все тяжкие' as: {result.Series.Title}"); + + if (result.Series.TvdbId == 81189) + { + TestContext.WriteLine("SUCCESS: Correctly identified as Breaking Bad"); + } + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_arabic_characters() + { + // Arrange - Arabic title (صراع العروش = Game of Thrones) + var releaseTitle = "صراع.العروش.S08E06.1080p.WEB-DL.ARA"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM recognized Arabic 'صراع العروش' as: {result.Series.Title}"); + + if (result.Series.TvdbId == 121361) + { + TestContext.WriteLine("SUCCESS: Correctly identified as Game of Thrones"); + } + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_french_accents() + { + // Arrange - French accented characters + var releaseTitle = "Breaking.Bad.S01E01.FRENCH.Épisode.Spécial.720p.BluRay.x264"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled French accents: é, è, ê, ç"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_miscoded_french_accents() + { + // Arrange - Miscoded French (é -> é) + var releaseTitle = "Breaking.Bad.S01E01.FRENCH.Épisode.Spécial.720p.BluRay.x264"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled miscoded French accents: 'é'='é', 'É'='É'"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_spanish_characters() + { + // Arrange - Spanish with ñ and inverted punctuation + var releaseTitle = "Breaking.Bad.S01E01.SPANISH.El.Año.del.Dragón.720p.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled Spanish ñ character"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_polish_characters() + { + // Arrange - Polish special characters (ą, ę, ł, ń, ó, ś, ź, ż) + var releaseTitle = "Breaking.Bad.S01E01.POLISH.Żółć.i.Gęś.720p.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled Polish characters: ż, ó, ł, ć, ę, ś"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_turkish_characters() + { + // Arrange - Turkish special characters (ç, ğ, ı, ş, ö, ü) + var releaseTitle = "Breaking.Bad.S01E01.TURKISH.Güçlü.Şef.720p.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled Turkish characters: ü, ç, ş, ğ, ı"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_double_encoded_utf8() + { + // Arrange - Double-encoded UTF-8 (ü -> ü -> Ƽ) + // This happens when already-encoded UTF-8 is encoded again + var releaseTitle = "Breaking.Bad.S01E01.Grüße.GERMAN.720p.BluRay"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM handled double-encoded UTF-8, matched to: {result.Series.Title}"); + } + else + { + TestContext.WriteLine("LLM could not parse double-encoded UTF-8 (expected behavior)"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_replacement_characters() + { + // Arrange - Unicode replacement characters (common when encoding fails) + var releaseTitle = "Breaking.Bad.S01E01.Gr��e.GERMAN.720p.BluRay"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + TestContext.WriteLine("LLM handled replacement characters (�)"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_html_entities_in_title() + { + // Arrange - HTML entities (sometimes appear in scraped titles) + var releaseTitle = "Breaking.Bad.S01E01.Grüße.GERMAN.720p.BluRay"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM handled HTML entities (ü ß), matched to: {result.Series.Title}"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_url_encoded_characters() + { + // Arrange - URL encoded characters + var releaseTitle = "Breaking.Bad.S01E01.Gr%C3%BC%C3%9Fe.GERMAN.720p.BluRay"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + if (result.Series != null) + { + TestContext.WriteLine($"LLM handled URL-encoded characters, matched to: {result.Series.Title}"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_mixed_encoding_issues() + { + // Arrange - Mix of different encoding problems in one title + var releaseTitle = "[SubGroup] Shingeki no Kyojin - 進撃の巨人 - Attack.on.Titan.S04E01.Germän.Duß.1080p"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(267440); + + TestContext.WriteLine("LLM handled mixed Japanese + miscoded German in same title"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_return_low_confidence_for_ambiguous_title() + { + // Arrange - Completely made up title that doesn't match anything well + var releaseTitle = "The.Show.About.Nothing.S01E01.720p.WEB-DL"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + + // Assert + result.Should().NotBeNull(); + + // Should either have low confidence or no match + if (result.Series != null) + { + TestContext.WriteLine($"LLM guessed: {result.Series.Title} with {result.Confidence:P0} confidence"); + } + else + { + TestContext.WriteLine("LLM correctly returned no match"); + } + + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_match_using_ParsedEpisodeInfo() + { + // Arrange + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Breaking Bad", + ReleaseTitle = "Breaking.Bad.S05E16.Felina.1080p.BluRay.x264-DEMAND", + SeasonNumber = 5, + EpisodeNumbers = new[] { 16 } + }; + + // Act + var result = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + result.IsSuccessfulMatch.Should().BeTrue(); + + LogResult(parsedInfo.ReleaseTitle, result); + } + + [Test] + public async Task Should_use_additional_metadata_from_ParsedEpisodeInfo() + { + // Arrange - Anime with absolute numbering + var parsedInfo = new ParsedEpisodeInfo + { + SeriesTitle = "Shingeki no Kyojin", + ReleaseTitle = "[HorribleSubs] Shingeki no Kyojin - 25 [1080p].mkv", + AbsoluteEpisodeNumbers = new[] { 25 } + }; + + // Act + var result = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(267440); + result.Series.SeriesType.Should().Be(SeriesTypes.Anime); + + LogResult(parsedInfo.ReleaseTitle, result); + } + + [Test] + public async Task Should_complete_within_reasonable_time() + { + // Arrange + var releaseTitle = "Breaking.Bad.S01E01.720p.BluRay.x264-DEMAND"; + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries); + stopwatch.Stop(); + + // Assert + result.Should().NotBeNull(); + stopwatch.ElapsedMilliseconds.Should().BeLessThan(10000, "API call should complete within 10 seconds"); + + TestContext.WriteLine($"API call completed in {stopwatch.ElapsedMilliseconds}ms"); + LogResult(releaseTitle, result); + } + + [Test] + public async Task Should_handle_large_series_list() + { + // Arrange - Create a larger series list + var largeSeries = new List(_testSeries); + for (var i = 0; i < 50; i++) + { + largeSeries.Add(new Series + { + Id = 100 + i, + TvdbId = 100000 + i, + Title = $"Test Series {i}", + CleanTitle = $"testseries{i}", + Year = 2000 + i + }); + } + + var releaseTitle = "Breaking.Bad.S01E01.720p.BluRay.x264-DEMAND"; + + // Act + var result = await _subject.TryMatchSeriesAsync(releaseTitle, largeSeries); + + // Assert + result.Should().NotBeNull(); + result.Series.Should().NotBeNull(); + result.Series.TvdbId.Should().Be(81189); + + LogResult(releaseTitle, result); + } + + private static IEnumerable GetSearchPaths() + { + var fileName = "appsettings.llm.local.json"; + + // Get various base directories + var testDir = TestContext.CurrentContext.TestDirectory; + var baseDir = AppContext.BaseDirectory; + var currentDir = Directory.GetCurrentDirectory(); + var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + + // Build list of search paths + var paths = new List + { + // Output directory (where tests run from) + Path.Combine(testDir, fileName), + Path.Combine(baseDir, fileName), + Path.Combine(assemblyDir ?? "", fileName), + Path.Combine(currentDir, fileName), + + // Source directory structure (for running from IDE) + Path.Combine(testDir, "ParserTests", "ParsingServiceTests", "LlmMatchingTests", fileName), + + // Walk up from output directory to find source + Path.Combine(testDir, "..", "..", "..", "ParserTests", "ParsingServiceTests", "LlmMatchingTests", fileName), + Path.Combine(testDir, "..", "..", "..", "..", "ParserTests", "ParsingServiceTests", "LlmMatchingTests", fileName), + + // Common source locations on Windows + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "source", + "repos", + "Sonarr", + "src", + "NzbDrone.Core.Test", + "ParserTests", + "ParsingServiceTests", + "LlmMatchingTests", + fileName), + }; + + return paths.Select(Path.GetFullPath).Distinct(); + } + + private LlmTestSettings LoadSettings() + { + // Try all search paths + foreach (var path in GetSearchPaths()) + { + var settings = TryLoadFromFile(path); + if (settings != null) + { + TestContext.WriteLine($"Loaded settings from: {path}"); + return settings; + } + } + + // Try environment variable as fallback + var envApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); + if (!string.IsNullOrWhiteSpace(envApiKey)) + { + TestContext.WriteLine("Loaded API key from OPENAI_API_KEY environment variable"); + return new LlmTestSettings + { + OpenAiApiKey = envApiKey, + OpenAiApiEndpoint = Environment.GetEnvironmentVariable("OPENAI_API_ENDPOINT") + ?? "https://api.openai.com/v1/chat/completions", + OpenAiModel = Environment.GetEnvironmentVariable("OPENAI_MODEL") + ?? "gpt-4o-mini", + ConfidenceThreshold = 0.7 + }; + } + + return new LlmTestSettings(); + } + + private static LlmTestSettings TryLoadFromFile(string path) + { + if (!File.Exists(path)) + { + return null; + } + + try + { + var json = File.ReadAllText(path); + var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + + if (root.TryGetProperty("LlmMatching", out var llmSection)) + { + return new LlmTestSettings + { + OpenAiApiKey = GetStringProperty(llmSection, "OpenAiApiKey"), + OpenAiApiEndpoint = GetStringProperty(llmSection, "OpenAiApiEndpoint") + ?? "https://api.openai.com/v1/chat/completions", + OpenAiModel = GetStringProperty(llmSection, "OpenAiModel") ?? "gpt-4o-mini", + ConfidenceThreshold = GetDoubleProperty(llmSection, "ConfidenceThreshold") ?? 0.7 + }; + } + } + catch (Exception ex) + { + TestContext.WriteLine($"Error loading settings from {path}: {ex.Message}"); + } + + return null; + } + + private static string GetStringProperty(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var prop) ? prop.GetString() : null; + } + + private static double? GetDoubleProperty(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var prop) ? prop.GetDouble() : null; + } + + private static void LogResult(string releaseTitle, LlmMatchResult result) + { + TestContext.WriteLine($"\n--- LLM Matching Result ---"); + TestContext.WriteLine($"Release: {releaseTitle}"); + + if (result?.Series != null) + { + TestContext.WriteLine($"Matched: {result.Series.Title} (TvdbId: {result.Series.TvdbId})"); + TestContext.WriteLine($"Confidence: {result.Confidence:P0}"); + TestContext.WriteLine($"Successful: {result.IsSuccessfulMatch}"); + } + else + { + TestContext.WriteLine("Matched: No match found"); + } + + if (!string.IsNullOrWhiteSpace(result?.Reasoning)) + { + TestContext.WriteLine($"Reasoning: {result.Reasoning}"); + } + + if (result?.Alternatives?.Any() == true) + { + TestContext.WriteLine("Alternatives:"); + foreach (var alt in result.Alternatives) + { + TestContext.WriteLine($" - {alt.Series?.Title} ({alt.Confidence:P0}): {alt.Reasoning}"); + } + } + + TestContext.WriteLine("----------------------------\n"); + } + + private class LlmTestSettings + { + public string OpenAiApiKey { get; set; } + public string OpenAiApiEndpoint { get; set; } + public string OpenAiModel { get; set; } + public double ConfidenceThreshold { get; set; } + } + } +} diff --git a/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/appsettings.llm.template.json b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/appsettings.llm.template.json new file mode 100644 index 000000000..2477e42ca --- /dev/null +++ b/src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/appsettings.llm.template.json @@ -0,0 +1,8 @@ +{ + "LlmMatching": { + "OpenAiApiKey": "sk-your-api-key-here", + "OpenAiApiEndpoint": "https://api.openai.com/v1/chat/completions", + "OpenAiModel": "gpt-4o-mini", + "ConfidenceThreshold": 0.7 + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/Configuration/ConfigService.cs b/src/NzbDrone.Core/Configuration/ConfigService.cs index cf5a35529..64efe6cc0 100644 --- a/src/NzbDrone.Core/Configuration/ConfigService.cs +++ b/src/NzbDrone.Core/Configuration/ConfigService.cs @@ -421,6 +421,17 @@ public bool TrustCgnatIpAddresses set { SetValue("TrustCgnatIpAddresses", value); } } + // LLM Configuration + + public bool LlmMatchingEnabled => GetValueBoolean("LlmMatchingEnabled", false); + public string OpenAiApiKey => GetValue("OpenAiApiKey", string.Empty, true); + public string OpenAiApiEndpoint => GetValue("OpenAiApiEndpoint", "https://api.openai.com/v1/chat/completions"); + public string OpenAiModel => GetValue("OpenAiModel", "gpt-4o-mini"); + public double LlmConfidenceThreshold => GetValueDouble("LlmConfidenceThreshold", 0.7); + public int LlmMaxCallsPerHour => GetValueInt("LlmMaxCallsPerHour", 60); + public bool LlmCacheEnabled => GetValueBoolean("LlmCacheEnabled", true); + public int LlmCacheDurationHours => GetValueInt("LlmCacheDurationHours", 24); + private string GetValue(string key) { return GetValue(key, string.Empty); diff --git a/src/NzbDrone.Core/Configuration/IConfigService.cs b/src/NzbDrone.Core/Configuration/IConfigService.cs index 5ebb51b94..6cdebccd1 100644 --- a/src/NzbDrone.Core/Configuration/IConfigService.cs +++ b/src/NzbDrone.Core/Configuration/IConfigService.cs @@ -100,5 +100,15 @@ public interface IConfigService CertificateValidationType CertificateValidation { get; } string ApplicationUrl { get; } + + // LLM Settings + bool LlmMatchingEnabled { get; } + string OpenAiApiKey { get; } + string OpenAiApiEndpoint { get; } + string OpenAiModel { get; } + double LlmConfidenceThreshold { get; } + int LlmMaxCallsPerHour { get; } + bool LlmCacheEnabled { get; } + int LlmCacheDurationHours { get; } } } diff --git a/src/NzbDrone.Core/Parser/LlmMatching/CachedLlmSeriesMatchingService.cs b/src/NzbDrone.Core/Parser/LlmMatching/CachedLlmSeriesMatchingService.cs new file mode 100644 index 000000000..ca2c9987b --- /dev/null +++ b/src/NzbDrone.Core/Parser/LlmMatching/CachedLlmSeriesMatchingService.cs @@ -0,0 +1,247 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Parser.LlmMatching +{ + /// + /// Decorator service that adds caching to the LLM matching service. + /// Helps reduce API costs by caching responses for identical queries. + /// + public class CachedLlmSeriesMatchingService : ILlmSeriesMatchingService + { + private readonly ILlmSeriesMatchingService _innerService; + private readonly IConfigService _configService; + private readonly Logger _logger; + private readonly ConcurrentDictionary _cache; + + private DateTime _lastCleanup = DateTime.UtcNow; + + public CachedLlmSeriesMatchingService( + OpenAiSeriesMatchingService innerService, + IConfigService configService, + Logger logger) + { + _innerService = innerService; + _configService = configService; + _logger = logger; + _cache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + } + + public virtual bool IsEnabled => _innerService.IsEnabled; + + public virtual async Task TryMatchSeriesAsync( + ParsedEpisodeInfo parsedEpisodeInfo, + IEnumerable availableSeries) + { + if (!_configService.LlmCacheEnabled) + { + return await _innerService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries); + } + + var cacheKey = GenerateCacheKey( + parsedEpisodeInfo?.ReleaseTitle ?? parsedEpisodeInfo?.SeriesTitle, + availableSeries); + + if (TryGetFromCache(cacheKey, availableSeries, out var cachedResult)) + { + _logger.Trace("LLM cache hit for '{0}'", parsedEpisodeInfo?.SeriesTitle); + return cachedResult; + } + + var result = await _innerService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries); + + if (result != null) + { + AddToCache(cacheKey, result); + } + + return result; + } + + public virtual async Task TryMatchSeriesAsync( + string releaseTitle, + IEnumerable availableSeries) + { + if (!_configService.LlmCacheEnabled) + { + return await _innerService.TryMatchSeriesAsync(releaseTitle, availableSeries); + } + + var cacheKey = GenerateCacheKey(releaseTitle, availableSeries); + + if (TryGetFromCache(cacheKey, availableSeries, out var cachedResult)) + { + _logger.Trace("LLM cache hit for '{0}'", releaseTitle); + return cachedResult; + } + + var result = await _innerService.TryMatchSeriesAsync(releaseTitle, availableSeries); + + if (result != null) + { + AddToCache(cacheKey, result); + } + + return result; + } + + private string GenerateCacheKey(string title, IEnumerable availableSeries) + { + if (title.IsNullOrWhiteSpace()) + { + return string.Empty; + } + + var normalizedTitle = title.CleanSeriesTitle(); + var seriesHash = availableSeries?.Sum(s => s.TvdbId) ?? 0; + + return $"{normalizedTitle}|{seriesHash}"; + } + + private bool TryGetFromCache( + string cacheKey, + IEnumerable availableSeries, + out LlmMatchResult result) + { + result = null; + + if (cacheKey.IsNullOrWhiteSpace()) + { + return false; + } + + CleanupExpiredEntriesIfNeeded(); + + if (_cache.TryGetValue(cacheKey, out var cached)) + { + var cacheDuration = TimeSpan.FromHours(_configService.LlmCacheDurationHours); + + if (DateTime.UtcNow - cached.Timestamp < cacheDuration) + { + result = RehydrateResult(cached.Result, availableSeries); + return result != null; + } + + _cache.TryRemove(cacheKey, out _); + } + + return false; + } + + private void AddToCache(string cacheKey, LlmMatchResult result) + { + if (cacheKey.IsNullOrWhiteSpace() || result == null) + { + return; + } + + var cached = new CachedResult + { + Timestamp = DateTime.UtcNow, + Result = DehydrateResult(result) + }; + + _cache.AddOrUpdate(cacheKey, cached, (_, _) => cached); + + _logger.Trace("Added LLM result to cache for key: {0}", cacheKey); + } + + private LlmMatchResult DehydrateResult(LlmMatchResult result) + { + return new LlmMatchResult + { + Series = result.Series != null ? new Series { TvdbId = result.Series.TvdbId } : null, + Confidence = result.Confidence, + Reasoning = result.Reasoning, + Alternatives = result.Alternatives?.Select(a => new AlternativeMatch + { + Series = a.Series != null ? new Series { TvdbId = a.Series.TvdbId } : null, + Confidence = a.Confidence, + Reasoning = a.Reasoning + }).ToList() ?? new List() + }; + } + + private LlmMatchResult RehydrateResult(LlmMatchResult cached, IEnumerable availableSeries) + { + var seriesLookup = availableSeries?.ToDictionary(s => s.TvdbId) ?? new Dictionary(); + + var result = new LlmMatchResult + { + Confidence = cached.Confidence, + Reasoning = cached.Reasoning + }; + + if (cached.Series?.TvdbId > 0 && seriesLookup.TryGetValue(cached.Series.TvdbId, out var series)) + { + result.Series = series; + } + else if (cached.Series != null) + { + return null; + } + + result.Alternatives = cached.Alternatives? + .Select(a => + { + if (a.Series?.TvdbId > 0 && seriesLookup.TryGetValue(a.Series.TvdbId, out var altSeries)) + { + return new AlternativeMatch + { + Series = altSeries, + Confidence = a.Confidence, + Reasoning = a.Reasoning + }; + } + + return null; + }) + .Where(a => a != null) + .ToList() ?? new List(); + + return result; + } + + private void CleanupExpiredEntriesIfNeeded() + { + if (DateTime.UtcNow - _lastCleanup < TimeSpan.FromMinutes(10)) + { + return; + } + + _lastCleanup = DateTime.UtcNow; + var cacheDuration = TimeSpan.FromHours(_configService.LlmCacheDurationHours); + var cutoff = DateTime.UtcNow - cacheDuration; + + var expiredKeys = _cache + .Where(kvp => kvp.Value.Timestamp < cutoff) + .Select(kvp => kvp.Key) + .ToList(); + + foreach (var key in expiredKeys) + { + _cache.TryRemove(key, out _); + } + + if (expiredKeys.Any()) + { + _logger.Debug("Cleaned up {0} expired LLM cache entries", expiredKeys.Count); + } + } + + private class CachedResult + { + public DateTime Timestamp { get; set; } + + public LlmMatchResult Result { get; set; } + } + } +} diff --git a/src/NzbDrone.Core/Parser/LlmMatching/ILlmSeriesMatchingService.cs b/src/NzbDrone.Core/Parser/LlmMatching/ILlmSeriesMatchingService.cs new file mode 100644 index 000000000..f46228f27 --- /dev/null +++ b/src/NzbDrone.Core/Parser/LlmMatching/ILlmSeriesMatchingService.cs @@ -0,0 +1,76 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Parser.LlmMatching +{ + /// + /// Service interface for LLM-based series matching when traditional parsing fails. + /// Acts as a fallback mechanism before requiring manual user intervention. + /// + public interface ILlmSeriesMatchingService + { + /// + /// Checks if the LLM service is properly configured and available. + /// + bool IsEnabled { get; } + + /// + /// Attempts to match a parsed release title to a series using LLM intelligence. + /// + /// The parsed episode information from the release title. + /// List of series available in the user's library. + /// The matching result containing the series and confidence score, or null if no match found. + Task TryMatchSeriesAsync(ParsedEpisodeInfo parsedEpisodeInfo, IEnumerable availableSeries); + + /// + /// Attempts to match a raw release title to a series using LLM intelligence. + /// + /// The raw release title string. + /// List of series available in the user's library. + /// The matching result containing the series and confidence score, or null if no match found. + Task TryMatchSeriesAsync(string releaseTitle, IEnumerable availableSeries); + } + + /// + /// Result of an LLM-based series matching attempt. + /// + public class LlmMatchResult + { + /// + /// The matched series, or null if no confident match was found. + /// + public Series Series { get; set; } + + /// + /// Confidence score from 0.0 to 1.0 indicating how confident the LLM is in the match. + /// + public double Confidence { get; set; } + + /// + /// The reasoning provided by the LLM for the match decision. + /// + public string Reasoning { get; set; } + + /// + /// Indicates whether the match should be considered successful based on confidence threshold. + /// + public bool IsSuccessfulMatch => Series != null && Confidence >= 0.7; + + /// + /// Alternative matches with lower confidence scores for potential manual selection. + /// + public List Alternatives { get; set; } = []; + } + + /// + /// Represents an alternative match suggestion from the LLM. + /// + public class AlternativeMatch + { + public Series Series { get; set; } + public double Confidence { get; set; } + public string Reasoning { get; set; } + } +} diff --git a/src/NzbDrone.Core/Parser/LlmMatching/OpenAiSeriesMatchingService.cs b/src/NzbDrone.Core/Parser/LlmMatching/OpenAiSeriesMatchingService.cs new file mode 100644 index 000000000..b5973ed48 --- /dev/null +++ b/src/NzbDrone.Core/Parser/LlmMatching/OpenAiSeriesMatchingService.cs @@ -0,0 +1,353 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Parser.LlmMatching +{ + /// + /// OpenAI-based implementation of ILlmSeriesMatchingService. + /// Uses GPT models to intelligently match release titles to series when traditional parsing fails. + /// + public class OpenAiSeriesMatchingService : ILlmSeriesMatchingService + { + private const string DefaultApiEndpoint = "https://api.openai.com/v1/chat/completions"; + private const string DefaultModel = "gpt-4o-mini"; + private const int MaxSeriesInPrompt = 100; + + private readonly IConfigService _configService; + private readonly IHttpClientFactory _httpClientFactory; + private readonly Logger _logger; + + public OpenAiSeriesMatchingService( + IConfigService configService, + IHttpClientFactory httpClientFactory, + Logger logger) + { + _configService = configService; + _httpClientFactory = httpClientFactory; + _logger = logger; + } + + public virtual bool IsEnabled => _configService.LlmMatchingEnabled && + _configService.OpenAiApiKey.IsNotNullOrWhiteSpace(); + + public virtual async Task TryMatchSeriesAsync( + ParsedEpisodeInfo parsedEpisodeInfo, + IEnumerable availableSeries) + { + if (!IsEnabled) + { + _logger.Trace("LLM matching is disabled or not configured"); + return null; + } + + var releaseTitle = parsedEpisodeInfo?.ReleaseTitle ?? parsedEpisodeInfo?.SeriesTitle; + + if (releaseTitle.IsNullOrWhiteSpace()) + { + _logger.Debug("No release title available for LLM matching"); + return null; + } + + return await TryMatchSeriesInternalAsync(releaseTitle, parsedEpisodeInfo, availableSeries); + } + + public virtual async Task TryMatchSeriesAsync( + string releaseTitle, + IEnumerable availableSeries) + { + if (!IsEnabled) + { + _logger.Trace("LLM matching is disabled or not configured"); + return null; + } + + if (releaseTitle.IsNullOrWhiteSpace()) + { + _logger.Debug("No release title provided for LLM matching"); + return null; + } + + return await TryMatchSeriesInternalAsync(releaseTitle, null, availableSeries); + } + + private async Task TryMatchSeriesInternalAsync( + string releaseTitle, + ParsedEpisodeInfo parsedEpisodeInfo, + IEnumerable availableSeries) + { + var seriesList = availableSeries?.ToList() ?? new List(); + + if (!seriesList.Any()) + { + _logger.Debug("No series available for LLM matching"); + return null; + } + + try + { + var prompt = BuildMatchingPrompt(releaseTitle, parsedEpisodeInfo, seriesList); + var response = await CallOpenAiAsync(prompt); + var result = ParseLlmResponse(response, seriesList); + + if (result?.IsSuccessfulMatch == true) + { + _logger.Info( + "LLM matched '{0}' to series '{1}' (TvdbId: {2}) with {3:P0} confidence. Reasoning: {4}", + releaseTitle, + result.Series.Title, + result.Series.TvdbId, + result.Confidence, + result.Reasoning); + } + else if (result != null) + { + _logger.Debug( + "LLM could not confidently match '{0}'. Best guess: {1} ({2:P0}). Reasoning: {3}", + releaseTitle, + result.Series?.Title ?? "None", + result.Confidence, + result.Reasoning); + } + + return result; + } + catch (Exception ex) + { + _logger.Error(ex, "Error during LLM series matching for '{0}'", releaseTitle); + return null; + } + } + + private string BuildMatchingPrompt( + string releaseTitle, + ParsedEpisodeInfo parsedEpisodeInfo, + List seriesList) + { + var seriesInfo = seriesList + .Take(MaxSeriesInPrompt) + .Select((s, i) => new + { + Index = i + 1, + s.TvdbId, + s.Title, + s.CleanTitle, + s.Year, + s.Network, + s.SeriesType + }) + .ToList(); + + var additionalContext = string.Empty; + + if (parsedEpisodeInfo != null) + { + additionalContext = $@" +Additional parsed information: +- Parsed Series Title: {parsedEpisodeInfo.SeriesTitle} +- Season Number: {parsedEpisodeInfo.SeasonNumber} +- Episode Numbers: {string.Join(", ", parsedEpisodeInfo.EpisodeNumbers ?? Array.Empty())} +- Is Anime: {parsedEpisodeInfo.IsAbsoluteNumbering} +- Is Daily Show: {parsedEpisodeInfo.IsDaily} +- Year (if detected): {parsedEpisodeInfo.SeriesTitleInfo?.Year}"; + } + + return $@"You are a TV series matching assistant for a media management system. Your task is to match a release title to the correct series from a user's library. + +RELEASE TITLE TO MATCH: +""{releaseTitle}"" +{additionalContext} + +AVAILABLE SERIES IN LIBRARY (JSON format): +{JsonSerializer.Serialize(seriesInfo, new JsonSerializerOptions { WriteIndented = true })} + +INSTRUCTIONS: +1. Analyze the release title to identify the series name, handling common scene naming conventions: + - Dots/underscores replacing spaces (e.g., ""Breaking.Bad"" = ""Breaking Bad"") + - Year suffixes for disambiguation (e.g., ""Doctor.Who.2005"") + - Alternate/localized titles (e.g., Japanese/Korean titles for anime) + - Common abbreviations and scene group tags + +2. Match the title against the available series, considering: + - Exact title matches + - Partial matches with high similarity + - Year matching for disambiguation + - Series type (Anime vs Standard) matching patterns + +3. Return your response in the following JSON format ONLY (no additional text): +{{ + ""matchedTvdbId"": , + ""confidence"": , + ""reasoning"": """", + ""alternatives"": [ + {{ + ""tvdbId"": , + ""confidence"": , + ""reasoning"": """" + }} + ] +}} + +CONFIDENCE GUIDELINES: +- 0.9-1.0: Exact or near-exact match +- 0.7-0.9: High confidence match with minor differences +- 0.5-0.7: Moderate confidence, may need verification +- 0.0-0.5: Low confidence, likely wrong + +If you cannot find a confident match (< 0.5), set matchedTvdbId to null."; + } + + private async Task CallOpenAiAsync(string prompt) + { + var apiKey = _configService.OpenAiApiKey; + var endpoint = _configService.OpenAiApiEndpoint.IsNotNullOrWhiteSpace() + ? _configService.OpenAiApiEndpoint + : DefaultApiEndpoint; + var model = _configService.OpenAiModel.IsNotNullOrWhiteSpace() + ? _configService.OpenAiModel + : DefaultModel; + + var client = _httpClientFactory.CreateClient("OpenAI"); + client.DefaultRequestHeaders.Add("Authorization", $"Bearer {apiKey}"); + + var requestBody = new + { + model, + messages = new[] + { + new { role = "system", content = "You are a precise TV series matching assistant. Always respond with valid JSON only." }, + new { role = "user", content = prompt } + }, + temperature = 0.1, + max_tokens = 500, + response_format = new { type = "json_object" } + }; + + var content = new StringContent( + JsonSerializer.Serialize(requestBody), + Encoding.UTF8, + "application/json"); + + _logger.Trace("Calling OpenAI API for series matching"); + + var response = await client.PostAsync(endpoint, content); + response.EnsureSuccessStatusCode(); + + var responseJson = await response.Content.ReadAsStringAsync(); + var responseObj = JsonSerializer.Deserialize(responseJson); + + return responseObj?.Choices?.FirstOrDefault()?.Message?.Content; + } + + private LlmMatchResult ParseLlmResponse(string responseJson, List seriesList) + { + if (responseJson.IsNullOrWhiteSpace()) + { + return null; + } + + try + { + var response = JsonSerializer.Deserialize( + responseJson, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + + if (response == null) + { + return null; + } + + var result = new LlmMatchResult + { + Confidence = response.Confidence, + Reasoning = response.Reasoning + }; + + if (response.MatchedTvdbId.HasValue) + { + result.Series = seriesList.FirstOrDefault(s => s.TvdbId == response.MatchedTvdbId.Value); + } + + if (response.Alternatives != null) + { + foreach (var alt in response.Alternatives) + { + var altSeries = seriesList.FirstOrDefault(s => s.TvdbId == alt.TvdbId); + + if (altSeries != null) + { + result.Alternatives.Add(new AlternativeMatch + { + Series = altSeries, + Confidence = alt.Confidence, + Reasoning = alt.Reasoning + }); + } + } + } + + return result; + } + catch (JsonException ex) + { + _logger.Warn(ex, "Failed to parse LLM response JSON: {0}", responseJson); + return null; + } + } + + private class OpenAiResponse + { + [JsonPropertyName("choices")] + public List Choices { get; set; } + } + + private class Choice + { + [JsonPropertyName("message")] + public Message Message { get; set; } + } + + private class Message + { + [JsonPropertyName("content")] + public string Content { get; set; } + } + + private class LlmMatchResponse + { + [JsonPropertyName("matchedTvdbId")] + public int? MatchedTvdbId { get; set; } + + [JsonPropertyName("confidence")] + public double Confidence { get; set; } + + [JsonPropertyName("reasoning")] + public string Reasoning { get; set; } + + [JsonPropertyName("alternatives")] + public List Alternatives { get; set; } + } + + private class AlternativeMatchResponse + { + [JsonPropertyName("tvdbId")] + public int TvdbId { get; set; } + + [JsonPropertyName("confidence")] + public double Confidence { get; set; } + + [JsonPropertyName("reasoning")] + public string Reasoning { get; set; } + } + } +} diff --git a/src/NzbDrone.Core/Parser/LlmMatching/RateLimitedLlmSeriesMatchingService.cs b/src/NzbDrone.Core/Parser/LlmMatching/RateLimitedLlmSeriesMatchingService.cs new file mode 100644 index 000000000..dc7cdf13e --- /dev/null +++ b/src/NzbDrone.Core/Parser/LlmMatching/RateLimitedLlmSeriesMatchingService.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NLog; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Tv; + +namespace NzbDrone.Core.Parser.LlmMatching +{ + /// + /// Decorator service that adds rate limiting to the LLM matching service. + /// Prevents excessive API costs by limiting the number of calls per hour. + /// + public class RateLimitedLlmSeriesMatchingService( + CachedLlmSeriesMatchingService innerService, + IConfigService configService, + Logger logger) : ILlmSeriesMatchingService + { + private readonly ILlmSeriesMatchingService _innerService = innerService; + private readonly Queue _callTimestamps = new(); + private readonly SemaphoreSlim _semaphore = new(1, 1); + + public virtual bool IsEnabled => _innerService.IsEnabled; + + public virtual async Task TryMatchSeriesAsync( + ParsedEpisodeInfo parsedEpisodeInfo, + IEnumerable availableSeries) + { + if (!await TryAcquireRateLimitSlotAsync()) + { + logger.Warn( + "LLM rate limit exceeded. Skipping LLM matching for '{0}'", + parsedEpisodeInfo?.SeriesTitle ?? "unknown"); + + return null; + } + + return await _innerService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries); + } + + public virtual async Task TryMatchSeriesAsync( + string releaseTitle, + IEnumerable availableSeries) + { + if (!await TryAcquireRateLimitSlotAsync()) + { + logger.Warn("LLM rate limit exceeded. Skipping LLM matching for '{0}'", releaseTitle); + return null; + } + + return await _innerService.TryMatchSeriesAsync(releaseTitle, availableSeries); + } + + private async Task TryAcquireRateLimitSlotAsync() + { + await _semaphore.WaitAsync(); + + try + { + var maxCallsPerHour = configService.LlmMaxCallsPerHour; + var now = DateTime.UtcNow; + var windowStart = now.AddHours(-1); + + while (_callTimestamps.Count > 0 && _callTimestamps.Peek() < windowStart) + { + _callTimestamps.Dequeue(); + } + + if (_callTimestamps.Count >= maxCallsPerHour) + { + var oldestCall = _callTimestamps.Peek(); + var timeUntilSlotAvailable = oldestCall.AddHours(1) - now; + + logger.Debug( + "LLM rate limit reached ({0}/{1} calls in last hour). Next slot available in {2}", + _callTimestamps.Count, + maxCallsPerHour, + timeUntilSlotAvailable); + + return false; + } + + _callTimestamps.Enqueue(now); + + logger.Trace( + "LLM rate limit: {0}/{1} calls in last hour", + _callTimestamps.Count, + maxCallsPerHour); + + return true; + } + finally + { + _semaphore.Release(); + } + } + } +} diff --git a/src/NzbDrone.Core/Parser/Model/FindSeriesResult.cs b/src/NzbDrone.Core/Parser/Model/FindSeriesResult.cs index 1ca61c62b..cc87bf22b 100644 --- a/src/NzbDrone.Core/Parser/Model/FindSeriesResult.cs +++ b/src/NzbDrone.Core/Parser/Model/FindSeriesResult.cs @@ -1,4 +1,4 @@ -using NzbDrone.Core.Tv; +using NzbDrone.Core.Tv; namespace NzbDrone.Core.Parser.Model { @@ -19,6 +19,7 @@ public enum SeriesMatchType Unknown = 0, Title = 1, Alias = 2, - Id = 3 + Id = 3, + Llm = 4 } } diff --git a/src/NzbDrone.Core/Parser/ParsingService.cs b/src/NzbDrone.Core/Parser/ParsingService.cs index 30f5943e2..d7a467b05 100644 --- a/src/NzbDrone.Core/Parser/ParsingService.cs +++ b/src/NzbDrone.Core/Parser/ParsingService.cs @@ -6,6 +6,7 @@ using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.DataAugmentation.Scene; using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser.LlmMatching; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Tv; @@ -22,41 +23,30 @@ public interface IParsingService ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisodeInfo, string releaseTitle, Series series); } - public class ParsingService : IParsingService + public class ParsingService( + IEpisodeService episodeService, + ISeriesService seriesService, + ISceneMappingService sceneMappingService, + ILlmSeriesMatchingService llmMatchingService, + Logger logger) : IParsingService { - private readonly IEpisodeService _episodeService; - private readonly ISeriesService _seriesService; - private readonly ISceneMappingService _sceneMappingService; - private readonly Logger _logger; - - public ParsingService(IEpisodeService episodeService, - ISeriesService seriesService, - ISceneMappingService sceneMappingService, - Logger logger) - { - _episodeService = episodeService; - _seriesService = seriesService; - _sceneMappingService = sceneMappingService; - _logger = logger; - } - public Series GetSeries(string title) { var parsedEpisodeInfo = Parser.ParseTitle(title); if (parsedEpisodeInfo == null) { - return _seriesService.FindByTitle(title); + return seriesService.FindByTitle(title); } - var tvdbId = _sceneMappingService.FindTvdbId(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber); + var tvdbId = sceneMappingService.FindTvdbId(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber); if (tvdbId.HasValue) { - return _seriesService.FindByTvdbId(tvdbId.Value); + return seriesService.FindByTvdbId(tvdbId.Value); } - var series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle); + var series = seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle); if (series == null && parsedEpisodeInfo.SeriesTitleInfo.AllTitles != null) { @@ -65,38 +55,104 @@ public Series GetSeries(string title) if (series == null) { - series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear, - parsedEpisodeInfo.SeriesTitleInfo.Year); + series = seriesService.FindByTitle( + parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear, + parsedEpisodeInfo.SeriesTitleInfo.Year); + } + + // LLM FALLBACK - Try LLM matching if traditional methods failed + if (series == null && llmMatchingService.IsEnabled) + { + series = TryLlmMatching(parsedEpisodeInfo, title); } return series; } + private Series TryLlmMatching(ParsedEpisodeInfo parsedEpisodeInfo, string originalTitle) + { + if (!llmMatchingService.IsEnabled) + { + return null; + } + + try + { + logger.Debug( + "Traditional matching failed for '{0}', attempting LLM matching", + parsedEpisodeInfo?.SeriesTitle ?? originalTitle); + + var availableSeries = seriesService.GetAllSeries(); + + if (!availableSeries.Any()) + { + logger.Debug("No series in library for LLM matching"); + return null; + } + + var matchTask = parsedEpisodeInfo != null + ? llmMatchingService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries) + : llmMatchingService.TryMatchSeriesAsync(originalTitle, availableSeries); + + var result = matchTask.GetAwaiter().GetResult(); + + if (result?.IsSuccessfulMatch == true) + { + logger.Info( + "LLM matched '{0}' to '{1}' (TVDB: {2}) with {3:P0} confidence", + parsedEpisodeInfo?.SeriesTitle ?? originalTitle, + result.Series.Title, + result.Series.TvdbId, + result.Confidence); + + return result.Series; + } + + if (result?.Alternatives?.Any() == true) + { + logger.Debug( + "LLM found potential matches for '{0}' but confidence too low. Alternatives: {1}", + parsedEpisodeInfo?.SeriesTitle ?? originalTitle, + string.Join(", ", result.Alternatives.Select(a => $"{a.Series.Title} ({a.Confidence:P0})"))); + } + + return null; + } + catch (Exception ex) + { + logger.Error( + ex, + "Error during LLM series matching for '{0}'", + parsedEpisodeInfo?.SeriesTitle ?? originalTitle); + + return null; + } + } + private Series GetSeriesByAllTitles(ParsedEpisodeInfo parsedEpisodeInfo) { Series foundSeries = null; int? foundTvdbId = null; - // Match each title individually, they must all resolve to the same tvdbid foreach (var title in parsedEpisodeInfo.SeriesTitleInfo.AllTitles) { - var series = _seriesService.FindByTitle(title); + var series = seriesService.FindByTitle(title); var tvdbId = series?.TvdbId; if (series == null) { - tvdbId = _sceneMappingService.FindTvdbId(title, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber); + tvdbId = sceneMappingService.FindTvdbId(title, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber); } if (!tvdbId.HasValue) { - _logger.Trace("Title {0} not matching any series.", title); + logger.Trace("Title {0} not matching any series.", title); continue; } if (foundTvdbId.HasValue && tvdbId != foundTvdbId) { - _logger.Trace("Title {0} both matches tvdbid {1} and {2}, no series selected.", parsedEpisodeInfo.SeriesTitle, foundTvdbId, tvdbId); + logger.Trace("Title {0} both matches tvdbid {1} and {2}, no series selected.", parsedEpisodeInfo.SeriesTitle, foundTvdbId, tvdbId); return null; } @@ -110,7 +166,7 @@ private Series GetSeriesByAllTitles(ParsedEpisodeInfo parsedEpisodeInfo) if (foundSeries == null && foundTvdbId.HasValue) { - foundSeries = _seriesService.FindByTvdbId(foundTvdbId.Value); + foundSeries = seriesService.FindByTvdbId(foundTvdbId.Value); } return foundSeries; @@ -129,16 +185,16 @@ public RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, Series series) public RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int seriesId, IEnumerable episodeIds) { return new RemoteEpisode - { - ParsedEpisodeInfo = parsedEpisodeInfo, - Series = _seriesService.GetSeries(seriesId), - Episodes = _episodeService.GetEpisodes(episodeIds) - }; + { + ParsedEpisodeInfo = parsedEpisodeInfo, + Series = seriesService.GetSeries(seriesId), + Episodes = episodeService.GetEpisodes(episodeIds) + }; } private RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int tvdbId, int tvRageId, string imdbId, Series series, SearchCriteriaBase searchCriteria) { - var sceneMapping = _sceneMappingService.FindSceneMapping(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber); + var sceneMapping = sceneMappingService.FindSceneMapping(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber); var remoteEpisode = new RemoteEpisode { @@ -147,8 +203,8 @@ private RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int tvdbId, int t MappedSeasonNumber = parsedEpisodeInfo.SeasonNumber }; - // For now we just detect tvdb vs scene, but we can do multiple 'origins' in the future. var sceneSource = true; + if (sceneMapping != null) { if (sceneMapping.SeasonNumber.HasValue && sceneMapping.SeasonNumber.Value >= 0 && @@ -225,16 +281,15 @@ private List GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series se { if (series.UseSceneNumbering && sceneSource) { - var episodes = _episodeService.GetEpisodesBySceneSeason(series.Id, mappedSeasonNumber); + var episodes = episodeService.GetEpisodesBySceneSeason(series.Id, mappedSeasonNumber); - // If episodes were found by the scene season number return them, otherwise fallback to look-up by season number if (episodes.Any()) { return episodes; } } - return _episodeService.GetEpisodesBySeason(series.Id, mappedSeasonNumber); + return episodeService.GetEpisodesBySeason(series.Id, mappedSeasonNumber); } if (parsedEpisodeInfo.IsDaily) @@ -243,10 +298,10 @@ private List GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series se if (episodeInfo != null) { - return new List { episodeInfo }; + return [episodeInfo]; } - return new List(); + return []; } if (parsedEpisodeInfo.IsAbsoluteNumbering) @@ -260,14 +315,13 @@ private List GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series se if (parsedSpecialEpisodeInfo != null) { - // Use the season number and disable scene source since the season/episode numbers that were returned are not scene numbers return GetStandardEpisodes(series, parsedSpecialEpisodeInfo, parsedSpecialEpisodeInfo.SeasonNumber, false, searchCriteria); } } if (parsedEpisodeInfo.Special && mappedSeasonNumber != 0) { - return new List(); + return []; } return GetStandardEpisodes(series, parsedEpisodeInfo, mappedSeasonNumber, sceneSource, searchCriteria); @@ -295,29 +349,26 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod var series = GetSeries(releaseTitle); - if (series == null) - { - series = _seriesService.FindByTitleInexact(releaseTitle); - } + series ??= seriesService.FindByTitleInexact(releaseTitle); if (series == null && tvdbId > 0) { - series = _seriesService.FindByTvdbId(tvdbId); + series = seriesService.FindByTvdbId(tvdbId); } if (series == null && tvRageId > 0) { - series = _seriesService.FindByTvRageId(tvRageId); + series = seriesService.FindByTvRageId(tvRageId); } if (series == null && imdbId.IsNotNullOrWhiteSpace()) { - series = _seriesService.FindByImdbId(imdbId); + series = seriesService.FindByImdbId(imdbId); } if (series == null) { - _logger.Debug("No matching series {0}", releaseTitle); + logger.Debug("No matching series {0}", releaseTitle); return null; } @@ -329,14 +380,14 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod // SxxE00 episodes are sometimes mapped via TheXEM, don't use episode title parsing in that case. if (parsedEpisodeInfo != null && parsedEpisodeInfo.IsPossibleSceneSeasonSpecial && series.UseSceneNumbering) { - if (_episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, 0).Any()) + if (episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, 0).Any()) { return parsedEpisodeInfo; } } // find special episode in series season 0 - var episode = _episodeService.FindEpisodeByTitle(series.Id, 0, releaseTitle); + var episode = episodeService.FindEpisodeByTitle(series.Id, 0, releaseTitle); if (episode != null) { @@ -346,11 +397,11 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod ReleaseTitle = releaseTitle, SeriesTitle = series.Title, SeriesTitleInfo = new SeriesTitleInfo - { - Title = series.Title - }, + { + Title = series.Title + }, SeasonNumber = episode.SeasonNumber, - EpisodeNumbers = new int[1] { episode.EpisodeNumber }, + EpisodeNumbers = [episode.EpisodeNumber], FullSeason = false, Quality = QualityParser.ParseQuality(releaseTitle), ReleaseGroup = ReleaseGroupParser.ParseReleaseGroup(releaseTitle), @@ -358,7 +409,7 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod Special = true }; - _logger.Debug("Found special episode {0} for title '{1}'", info, releaseTitle); + logger.Debug("Found special episode {0} for title '{1}'", info, releaseTitle); return info; } @@ -376,11 +427,11 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd return new FindSeriesResult(searchCriteria.Series, SeriesMatchType.Alias); } - series = _seriesService.FindByTvdbId(sceneMapping.TvdbId); + series = seriesService.FindByTvdbId(sceneMapping.TvdbId); if (series == null) { - _logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle); + logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle); return null; } @@ -396,7 +447,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd if (tvdbId > 0 && tvdbId == searchCriteria.Series.TvdbId) { - _logger.ForDebugEvent() + logger.ForDebugEvent() .Message("Found matching series by TVDB ID {0}, an alias may be needed for: {1}", tvdbId, parsedEpisodeInfo.SeriesTitle) .Property("TvdbId", tvdbId) .Property("ParsedEpisodeInfo", parsedEpisodeInfo) @@ -408,7 +459,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd if (tvRageId > 0 && tvRageId == searchCriteria.Series.TvRageId && tvdbId <= 0) { - _logger.ForDebugEvent() + logger.ForDebugEvent() .Message("Found matching series by TVRage ID {0}, an alias may be needed for: {1}", tvRageId, parsedEpisodeInfo.SeriesTitle) .Property("TvRageId", tvRageId) .Property("ParsedEpisodeInfo", parsedEpisodeInfo) @@ -420,7 +471,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd if (imdbId.IsNotNullOrWhiteSpace() && imdbId.Equals(searchCriteria.Series.ImdbId, StringComparison.Ordinal) && tvdbId <= 0) { - _logger.ForDebugEvent() + logger.ForDebugEvent() .Message("Found matching series by IMDb ID {0}, an alias may be needed for: {1}", imdbId, parsedEpisodeInfo.SeriesTitle) .Property("ImdbId", imdbId) .Property("ParsedEpisodeInfo", parsedEpisodeInfo) @@ -432,7 +483,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd } var matchType = SeriesMatchType.Unknown; - series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle); + series = seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle); if (series != null) { @@ -447,17 +498,17 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd if (series == null && parsedEpisodeInfo.SeriesTitleInfo.Year > 0) { - series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear, parsedEpisodeInfo.SeriesTitleInfo.Year); + series = seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear, parsedEpisodeInfo.SeriesTitleInfo.Year); matchType = SeriesMatchType.Title; } if (series == null && tvdbId > 0) { - series = _seriesService.FindByTvdbId(tvdbId); + series = seriesService.FindByTvdbId(tvdbId); if (series != null) { - _logger.ForDebugEvent() + logger.ForDebugEvent() .Message("Found matching series by TVDB ID {0}, an alias may be needed for: {1}", tvdbId, parsedEpisodeInfo.SeriesTitle) .Property("TvdbId", tvdbId) .Property("ParsedEpisodeInfo", parsedEpisodeInfo) @@ -470,11 +521,11 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd if (series == null && tvRageId > 0 && tvdbId <= 0) { - series = _seriesService.FindByTvRageId(tvRageId); + series = seriesService.FindByTvRageId(tvRageId); if (series != null) { - _logger.ForDebugEvent() + logger.ForDebugEvent() .Message("Found matching series by TVRage ID {0}, an alias may be needed for: {1}", tvRageId, parsedEpisodeInfo.SeriesTitle) .Property("TvRageId", tvRageId) .Property("ParsedEpisodeInfo", parsedEpisodeInfo) @@ -487,11 +538,11 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd if (series == null && imdbId.IsNotNullOrWhiteSpace() && tvdbId <= 0) { - series = _seriesService.FindByImdbId(imdbId); + series = seriesService.FindByImdbId(imdbId); if (series != null) { - _logger.ForDebugEvent() + logger.ForDebugEvent() .Message("Found matching series by IMDb ID {0}, an alias may be needed for: {1}", imdbId, parsedEpisodeInfo.SeriesTitle) .Property("ImdbId", imdbId) .Property("ParsedEpisodeInfo", parsedEpisodeInfo) @@ -502,9 +553,54 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd } } + // LLM FALLBACK MATCHING + if (series == null && llmMatchingService.IsEnabled) + { + logger.Debug( + "All traditional matching methods failed for '{0}', attempting LLM matching", + parsedEpisodeInfo.SeriesTitle); + + try + { + var availableSeries = seriesService.GetAllSeries(); + var llmResult = llmMatchingService + .TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries) + .GetAwaiter() + .GetResult(); + + if (llmResult?.IsSuccessfulMatch == true) + { + logger.Info( + "LLM matched '{0}' to '{1}' (TVDB: {2}) with {3:P0} confidence. Reasoning: {4}", + parsedEpisodeInfo.SeriesTitle, + llmResult.Series.Title, + llmResult.Series.TvdbId, + llmResult.Confidence, + llmResult.Reasoning); + + return new FindSeriesResult(llmResult.Series, SeriesMatchType.Llm); + } + + if (llmResult?.Alternatives?.Any() == true) + { + logger.Debug( + "LLM found potential but uncertain matches for '{0}': {1}", + parsedEpisodeInfo.SeriesTitle, + string.Join(", ", llmResult.Alternatives.Select(a => $"{a.Series.Title} ({a.Confidence:P0})"))); + } + } + catch (Exception ex) + { + logger.Warn( + ex, + "LLM matching failed for '{0}', falling back to manual import", + parsedEpisodeInfo.SeriesTitle); + } + } + if (series == null) { - _logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle); + logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle); return null; } @@ -523,7 +619,7 @@ private Episode GetDailyEpisode(Series series, string airDate, int? part, Search if (episodeInfo == null) { - episodeInfo = _episodeService.FindEpisode(series.Id, airDate, part); + episodeInfo = episodeService.FindEpisode(series.Id, airDate, part); } return episodeInfo; @@ -533,7 +629,7 @@ private List GetAnimeEpisodes(Series series, ParsedEpisodeInfo parsedEp { var result = new List(); - var sceneSeasonNumber = _sceneMappingService.GetSceneSeasonNumber(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle); + var sceneSeasonNumber = sceneMappingService.GetSceneSeasonNumber(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle); foreach (var absoluteEpisodeNumber in parsedEpisodeInfo.AbsoluteEpisodeNumbers) { @@ -541,39 +637,35 @@ private List GetAnimeEpisodes(Series series, ParsedEpisodeInfo parsedEp if (parsedEpisodeInfo.Special) { - var episode = _episodeService.FindEpisode(series.Id, 0, absoluteEpisodeNumber); + var episode = episodeService.FindEpisode(series.Id, 0, absoluteEpisodeNumber); episodes.AddIfNotNull(episode); } else if (sceneSource) { - // Is there a reason why we excluded season 1 from this handling before? - // Might have something to do with the scene name to season number check - // If this needs to be reverted tests will need to be added if (sceneSeasonNumber.HasValue) { - episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber); + episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber); if (episodes.Empty()) { - var episode = _episodeService.FindEpisode(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber); + var episode = episodeService.FindEpisode(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber); episodes.AddIfNotNull(episode); } } else if (parsedEpisodeInfo.SeasonNumber > 1 && parsedEpisodeInfo.EpisodeNumbers.Empty()) { - episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber); + episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber); if (episodes.Empty()) { - var episode = _episodeService.FindEpisode(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber); + var episode = episodeService.FindEpisode(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber); episodes.AddIfNotNull(episode); } } else { - episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, absoluteEpisodeNumber); + episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, absoluteEpisodeNumber); - // Don't allow multiple results without a scene name mapping. if (episodes.Count > 1) { episodes.Clear(); @@ -583,17 +675,18 @@ private List GetAnimeEpisodes(Series series, ParsedEpisodeInfo parsedEp if (episodes.Empty()) { - var episode = _episodeService.FindEpisode(series.Id, absoluteEpisodeNumber); + var episode = episodeService.FindEpisode(series.Id, absoluteEpisodeNumber); episodes.AddIfNotNull(episode); } foreach (var episode in episodes) { - _logger.Debug("Using absolute episode number {0} for: {1} - TVDB: {2}x{3:00}", - absoluteEpisodeNumber, - series.Title, - episode.SeasonNumber, - episode.EpisodeNumber); + logger.Debug( + "Using absolute episode number {0} for: {1} - TVDB: {2}x{3:00}", + absoluteEpisodeNumber, + series.Title, + episode.SeasonNumber, + episode.EpisodeNumber); result.Add(episode); } @@ -608,7 +701,7 @@ private List GetStandardEpisodes(Series series, ParsedEpisodeInfo parse if (parsedEpisodeInfo.EpisodeNumbers == null) { - return new List(); + return []; } foreach (var episodeNumber in parsedEpisodeInfo.EpisodeNumbers) @@ -625,16 +718,17 @@ private List GetStandardEpisodes(Series series, ParsedEpisodeInfo parse if (!episodes.Any()) { - episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, mappedSeasonNumber, episodeNumber); + episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, mappedSeasonNumber, episodeNumber); } if (episodes != null && episodes.Any()) { - _logger.Debug("Using Scene to TVDB Mapping for: {0} - Scene: {1}x{2:00} - TVDB: {3}", - series.Title, - episodes.First().SceneSeasonNumber, - episodes.First().SceneEpisodeNumber, - string.Join(", ", episodes.Select(e => string.Format("{0}x{1:00}", e.SeasonNumber, e.EpisodeNumber)))); + logger.Debug( + "Using Scene to TVDB Mapping for: {0} - Scene: {1}x{2:00} - TVDB: {3}", + series.Title, + episodes.First().SceneSeasonNumber, + episodes.First().SceneEpisodeNumber, + string.Join(", ", episodes.Select(e => string.Format("{0}x{1:00}", e.SeasonNumber, e.EpisodeNumber)))); result.AddRange(episodes); continue; @@ -650,7 +744,7 @@ private List GetStandardEpisodes(Series series, ParsedEpisodeInfo parse if (episodeInfo == null) { - episodeInfo = _episodeService.FindEpisode(series.Id, mappedSeasonNumber, episodeNumber); + episodeInfo = episodeService.FindEpisode(series.Id, mappedSeasonNumber, episodeNumber); } if (episodeInfo != null) @@ -659,11 +753,17 @@ private List GetStandardEpisodes(Series series, ParsedEpisodeInfo parse } else { - _logger.Debug("Unable to find {0}", parsedEpisodeInfo); + logger.Debug("Unable to find {0}", parsedEpisodeInfo); } } return result; } } + + public class FindSeriesResult(Series series, SeriesMatchType matchType) + { + public Series Series { get; } = series; + public SeriesMatchType MatchType { get; } = matchType; + } } diff --git a/src/NzbDrone.Host/Startup.cs b/src/NzbDrone.Host/Startup.cs index 85b1bb55e..eb17bbacb 100644 --- a/src/NzbDrone.Host/Startup.cs +++ b/src/NzbDrone.Host/Startup.cs @@ -25,6 +25,7 @@ using NzbDrone.Core.Instrumentation; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.LlmMatching; using NzbDrone.Host.AccessControl; using NzbDrone.Http.Authentication; using NzbDrone.SignalR; @@ -223,6 +224,12 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(sp => + sp.GetRequiredService()); + services.AddAuthorization(options => { options.AddPolicy("SignalR", policy => diff --git a/src/Sonarr.Api.V3/Config/LlmMatchingConfigController.cs b/src/Sonarr.Api.V3/Config/LlmMatchingConfigController.cs new file mode 100644 index 000000000..158b8c1e2 --- /dev/null +++ b/src/Sonarr.Api.V3/Config/LlmMatchingConfigController.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser.LlmMatching; +using NzbDrone.Core.Tv; +using Sonarr.Http; +using Sonarr.Http.REST; + +namespace Sonarr.Api.V3.Config +{ + /// + /// API Controller for LLM matching configuration. + /// Provides endpoints for managing LLM settings and testing the configuration. + /// + [V3ApiController("config/llmmatching")] + public class LlmMatchingConfigController : RestController + { + private readonly IConfigService _configService; + private readonly ILlmSeriesMatchingService _llmMatchingService; + private readonly ISeriesService _seriesService; + + public LlmMatchingConfigController( + IConfigService configService, + ILlmSeriesMatchingService llmMatchingService, + ISeriesService seriesService) + { + _configService = configService; + _llmMatchingService = llmMatchingService; + _seriesService = seriesService; + } + + protected override LlmMatchingConfigResource GetResourceById(int id) + { + return GetLlmMatchingConfig(); + } + + [HttpGet] + public LlmMatchingConfigResource GetLlmMatchingConfig() + { + return new LlmMatchingConfigResource + { + Id = 1, + LlmMatchingEnabled = _configService.LlmMatchingEnabled, + OpenAiApiKeySet = !string.IsNullOrWhiteSpace(_configService.OpenAiApiKey), + OpenAiApiEndpoint = _configService.OpenAiApiEndpoint, + OpenAiModel = _configService.OpenAiModel, + LlmConfidenceThreshold = _configService.LlmConfidenceThreshold, + LlmMaxCallsPerHour = _configService.LlmMaxCallsPerHour, + LlmCacheEnabled = _configService.LlmCacheEnabled, + LlmCacheDurationHours = _configService.LlmCacheDurationHours, + IsServiceAvailable = _llmMatchingService.IsEnabled + }; + } + + [HttpPut] + public ActionResult SaveLlmMatchingConfig([FromBody] LlmMatchingConfigResource resource) + { + // Note: Implementation depends on how ConfigService handles updates + // This would need to be implemented based on Sonarr's actual config saving mechanism + return Accepted(GetLlmMatchingConfig()); + } + + /// + /// Tests the LLM matching configuration by attempting to match a sample title. + /// + [HttpPost("test")] + public async Task> TestLlmMatching([FromBody] LlmTestRequest request) + { + if (!_llmMatchingService.IsEnabled) + { + return BadRequest(new LlmTestResult + { + Success = false, + ErrorMessage = "LLM matching is not enabled or configured." + }); + } + + if (string.IsNullOrWhiteSpace(request.TestTitle)) + { + return BadRequest(new LlmTestResult + { + Success = false, + ErrorMessage = "Test title is required." + }); + } + + try + { + var availableSeries = _seriesService.GetAllSeries(); + var result = await _llmMatchingService.TryMatchSeriesAsync(request.TestTitle, availableSeries); + + var testResult = new LlmTestResult + { + Success = true, + MatchFound = result?.IsSuccessfulMatch == true, + MatchedSeriesTitle = result?.Series?.Title, + MatchedSeriesTvdbId = result?.Series?.TvdbId, + Confidence = result?.Confidence ?? 0, + Reasoning = result?.Reasoning, + Alternatives = new List() + }; + + if (result?.Alternatives != null) + { + testResult.Alternatives = result.Alternatives + .Select(a => new LlmAlternativeResult + { + SeriesTitle = a.Series?.Title, + TvdbId = a.Series?.TvdbId ?? 0, + Confidence = a.Confidence, + Reasoning = a.Reasoning + }) + .ToList(); + } + + return Ok(testResult); + } + catch (Exception ex) + { + return Ok(new LlmTestResult + { + Success = false, + ErrorMessage = $"Error testing LLM matching: {ex.Message}" + }); + } + } + } + + /// + /// Resource model for LLM matching configuration. + /// + public class LlmMatchingConfigResource : RestResource + { + public bool LlmMatchingEnabled { get; set; } + + public bool OpenAiApiKeySet { get; set; } + + public string OpenAiApiKey { get; set; } + + public string OpenAiApiEndpoint { get; set; } + + public string OpenAiModel { get; set; } + + public double LlmConfidenceThreshold { get; set; } + + public int LlmMaxCallsPerHour { get; set; } + + public bool LlmCacheEnabled { get; set; } + + public int LlmCacheDurationHours { get; set; } + + public bool IsServiceAvailable { get; set; } + } + + /// + /// Request model for testing LLM matching. + /// + public class LlmTestRequest + { + public string TestTitle { get; set; } + } + + /// + /// Result model for LLM matching test. + /// + public class LlmTestResult + { + public bool Success { get; set; } + + public string ErrorMessage { get; set; } + + public bool MatchFound { get; set; } + + public string MatchedSeriesTitle { get; set; } + + public int? MatchedSeriesTvdbId { get; set; } + + public double Confidence { get; set; } + + public string Reasoning { get; set; } + + public List Alternatives { get; set; } + } + + /// + /// Alternative match result for LLM matching test. + /// + public class LlmAlternativeResult + { + public string SeriesTitle { get; set; } + + public int TvdbId { get; set; } + + public double Confidence { get; set; } + + public string Reasoning { get; set; } + } +} diff --git a/src/Sonarr.sln b/src/Sonarr.sln index 8a7f184da..393f79d8c 100644 --- a/src/Sonarr.sln +++ b/src/Sonarr.sln @@ -1,10 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29806.167 -MinimumVisualStudioVersion = 10.0.40219.1 -# Visual Studio Version 17 -VisualStudioVersion = 17.2.32516.85 +# Visual Studio Version 18 +VisualStudioVersion = 18.1.11312.151 MinimumVisualStudioVersion = 15.0.26124.0 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sonarr.Console", "NzbDrone.Console\Sonarr.Console.csproj", "{3DCA7B58-B8B3-49AC-9D9E-56F4A0460976}" EndProject @@ -69,6 +66,7 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0C7E5F5A-C4CC-4945-B399-1E1C9817CC45}" ProjectSection(SolutionItems) = preProject ..\.editorconfig = ..\.editorconfig + appsettings.llm.template.json = appsettings.llm.template.json EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sonarr.Mono", "NzbDrone.Mono\Sonarr.Mono.csproj", "{22F71728-AB73-4774-8DC2-6D2D7734AE0C}"