Backup_Repos/Sonarr
1
0
Fork 0
mirror of https://github.com/Sonarr/Sonarr synced 2026-01-23 16:02:14 +01:00
Code Issues Projects Releases Wiki Activity

Add LLM-based fallback for series matching via OpenAI

Browse source 
Introduces a new LLM (GPT) fallback mechanism for series matching when traditional parsing fails. Adds `ILlmSeriesMatchingService` with OpenAI, caching, and rate-limiting implementations. Integrates LLM matching into ParsingService, adds new config options, and exposes a REST API for configuration and testing. Includes unit and integration tests, and updates DI and solution files. This improves matching for ambiguous, foreign, or scene releases in a safe and configurable way.
This commit is contained in:
Fabian Proebster 2025-12-30 15:16:31 +01:00
parent 4713615b17
commit d6958e3175
15 changed files with 3625 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

810
src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceFixture.cs Normal file
View file

@ -0,0 +1,810 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FluentAssertions;
using Moq;
using NLog;
using NUnit.Framework;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.LlmMatching;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests.LlmMatchingTests
{
[TestFixture]
public class LlmSeriesMatchingServiceFixture
{
private Mock<IConfigService> _configService;
private Mock<ILlmSeriesMatchingService> _mockLlmService;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_configService = new Mock<IConfigService>();
_mockLlmService = new Mock<ILlmSeriesMatchingService>();
_testSeries = new List<Series>
{
new Series
{
Id = 1,
TvdbId = 81189,
Title = "Breaking Bad",
CleanTitle = "breakingbad",
Year = 2008,
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 2,
TvdbId = 121361,
Title = "Game of Thrones",
CleanTitle = "gameofthrones",
Year = 2011,
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 3,
TvdbId = 267440,
Title = "Attack on Titan",
CleanTitle = "attackontitan",
Year = 2013,
SeriesType = SeriesTypes.Anime
},
new Series
{
Id = 4,
TvdbId = 153021,
Title = "The Walking Dead",
CleanTitle = "thewalkingdead",
Year = 2010,
SeriesType = SeriesTypes.Standard
}
};
}
[Test]
public void IsEnabled_should_return_false_when_llm_matching_disabled()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(false);
_configService.Setup(s => s.OpenAiApiKey).Returns("test-key");
_mockLlmService.Setup(s => s.IsEnabled).Returns(false);
_mockLlmService.Object.IsEnabled.Should().BeFalse();
}
[Test]
public void IsEnabled_should_return_false_when_api_key_is_empty()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(true);
_configService.Setup(s => s.OpenAiApiKey).Returns(string.Empty);
_mockLlmService.Setup(s => s.IsEnabled).Returns(false);
_mockLlmService.Object.IsEnabled.Should().BeFalse();
}
[Test]
public void IsEnabled_should_return_false_when_api_key_is_null()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(true);
_configService.Setup(s => s.OpenAiApiKey).Returns((string)null);
_mockLlmService.Setup(s => s.IsEnabled).Returns(false);
_mockLlmService.Object.IsEnabled.Should().BeFalse();
}
[Test]
public void IsEnabled_should_return_true_when_properly_configured()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(true);
_configService.Setup(s => s.OpenAiApiKey).Returns("sk-test-key-12345");
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService.Object.IsEnabled.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_should_return_null_when_disabled()
{
_mockLlmService.Setup(s => s.IsEnabled).Returns(false);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync((LlmMatchResult)null);
var result = await _mockLlmService.Object.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_should_return_null_when_no_series_available()
{
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync((LlmMatchResult)null);
var result = await _mockLlmService.Object.TryMatchSeriesAsync("Breaking.Bad.S01E01", new List<Series>());
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_should_return_null_when_title_is_empty()
{
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync(string.Empty, It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync((LlmMatchResult)null);
var result = await _mockLlmService.Object.TryMatchSeriesAsync(string.Empty, _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_should_return_match_with_high_confidence()
{
var expectedSeries = _testSeries.First(s => s.TvdbId == 81189);
var expectedResult = new LlmMatchResult
{
Series = expectedSeries,
Confidence = 0.95,
Reasoning = "Direct title match after removing dots and quality tags"
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync("Breaking.Bad.S01E01.720p.WEB-DL", _testSeries))
.ReturnsAsync(expectedResult);
var result = await _mockLlmService.Object.TryMatchSeriesAsync("Breaking.Bad.S01E01.720p.WEB-DL", _testSeries);
result.Should().NotBeNull();
result.Series.Should().Be(expectedSeries);
result.Confidence.Should().Be(0.95);
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_should_return_unsuccessful_match_with_low_confidence()
{
var expectedSeries = _testSeries.First(s => s.TvdbId == 81189);
var expectedResult = new LlmMatchResult
{
Series = expectedSeries,
Confidence = 0.45,
Reasoning = "Possible match but title is ambiguous"
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync("Bad.Show.S01E01", _testSeries))
.ReturnsAsync(expectedResult);
var result = await _mockLlmService.Object.TryMatchSeriesAsync("Bad.Show.S01E01", _testSeries);
result.Should().NotBeNull();
result.Confidence.Should().Be(0.45);
result.IsSuccessfulMatch.Should().BeFalse();
}
[Test]
public async Task TryMatchSeriesAsync_should_handle_anime_alternate_titles()
{
var expectedSeries = _testSeries.First(s => s.TvdbId == 267440);
var expectedResult = new LlmMatchResult
{
Series = expectedSeries,
Confidence = 0.92,
Reasoning = "Shingeki no Kyojin is the Japanese title for Attack on Titan"
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync("Shingeki.no.Kyojin.S04E01.1080p", _testSeries))
.ReturnsAsync(expectedResult);
var result = await _mockLlmService.Object.TryMatchSeriesAsync("Shingeki.no.Kyojin.S04E01.1080p", _testSeries);
result.Should().NotBeNull();
result.Series.TvdbId.Should().Be(267440);
result.Series.Title.Should().Be("Attack on Titan");
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_should_include_alternatives_when_uncertain()
{
var primarySeries = _testSeries.First(s => s.TvdbId == 81189);
var alternativeSeries = _testSeries.First(s => s.TvdbId == 153021);
var expectedResult = new LlmMatchResult
{
Series = primarySeries,
Confidence = 0.55,
Reasoning = "Could be Breaking Bad but title is unclear",
Alternatives = new List<AlternativeMatch>
{
new AlternativeMatch
{
Series = alternativeSeries,
Confidence = 0.35,
Reasoning = "Walking Dead also possible"
}
}
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync("The.Bad.Dead.S01E01", _testSeries))
.ReturnsAsync(expectedResult);
var result = await _mockLlmService.Object.TryMatchSeriesAsync("The.Bad.Dead.S01E01", _testSeries);
result.Should().NotBeNull();
result.IsSuccessfulMatch.Should().BeFalse();
result.Alternatives.Should().HaveCount(1);
result.Alternatives.First().Series.TvdbId.Should().Be(153021);
}
[Test]
public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_should_return_null_when_disabled()
{
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad",
ReleaseTitle = "Breaking.Bad.S01E01.720p.WEB-DL",
SeasonNumber = 1,
EpisodeNumbers = new[] { 1 }
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(false);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<ParsedEpisodeInfo>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync((LlmMatchResult)null);
var result = await _mockLlmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_should_use_parsed_metadata()
{
// Note: IsAbsoluteNumbering and IsDaily are computed properties, we don't set them directly
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad German",
ReleaseTitle = "Breaking.Bad.German.S01E01.720p.WEB-DL",
SeasonNumber = 1,
EpisodeNumbers = new[] { 1 }
};
var expectedSeries = _testSeries.First(s => s.TvdbId == 81189);
var expectedResult = new LlmMatchResult
{
Series = expectedSeries,
Confidence = 0.88,
Reasoning = "Recognized 'German' as language tag, matched to Breaking Bad"
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync(parsedInfo, _testSeries))
.ReturnsAsync(expectedResult);
var result = await _mockLlmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries);
result.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_with_anime_ParsedEpisodeInfo_should_consider_series_type()
{
// Note: IsAbsoluteNumbering is computed from AbsoluteEpisodeNumbers
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Shingeki no Kyojin",
ReleaseTitle = "[SubGroup] Shingeki no Kyojin - 01 [1080p]",
SeasonNumber = 1,
AbsoluteEpisodeNumbers = new[] { 1 }
};
var expectedSeries = _testSeries.First(s => s.TvdbId == 267440);
var expectedResult = new LlmMatchResult
{
Series = expectedSeries,
Confidence = 0.94,
Reasoning = "Japanese anime title matched to Attack on Titan"
};
_mockLlmService.Setup(s => s.IsEnabled).Returns(true);
_mockLlmService
.Setup(s => s.TryMatchSeriesAsync(parsedInfo, _testSeries))
.ReturnsAsync(expectedResult);
var result = await _mockLlmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries);
result.Should().NotBeNull();
result.Series.SeriesType.Should().Be(SeriesTypes.Anime);
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_should_return_true_when_confidence_at_threshold()
{
var result = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.7
};
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_should_return_true_when_confidence_above_threshold()
{
var result = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.95
};
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_should_return_false_when_confidence_below_threshold()
{
var result = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.69
};
result.IsSuccessfulMatch.Should().BeFalse();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_should_return_false_when_series_is_null()
{
var result = new LlmMatchResult
{
Series = null,
Confidence = 0.95
};
result.IsSuccessfulMatch.Should().BeFalse();
}
[Test]
public void LlmMatchResult_should_initialize_alternatives_as_empty_list()
{
var result = new LlmMatchResult();
result.Alternatives.Should().NotBeNull();
result.Alternatives.Should().BeEmpty();
}
}
[TestFixture]
public class CachedLlmSeriesMatchingServiceFixture
{
private Mock<IConfigService> _configService;
private Mock<IHttpClientFactory> _httpClientFactory;
private Mock<OpenAiSeriesMatchingService> _innerService;
private CachedLlmSeriesMatchingService _subject;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_configService = new Mock<IConfigService>();
_httpClientFactory = new Mock<IHttpClientFactory>();
_innerService = new Mock<OpenAiSeriesMatchingService>(
_configService.Object,
_httpClientFactory.Object,
Mock.Of<Logger>());
_configService.Setup(s => s.LlmCacheEnabled).Returns(true);
_configService.Setup(s => s.LlmCacheDurationHours).Returns(24);
_subject = new CachedLlmSeriesMatchingService(
_innerService.Object,
_configService.Object,
Mock.Of<Logger>());
_testSeries = new List<Series>
{
new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" },
new Series { Id = 2, TvdbId = 121361, Title = "Game of Thrones" }
};
}
[Test]
public void IsEnabled_should_delegate_to_inner_service()
{
_innerService.Setup(s => s.IsEnabled).Returns(true);
_subject.IsEnabled.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_should_bypass_cache_when_caching_disabled()
{
_configService.Setup(s => s.LlmCacheEnabled).Returns(false);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// Call twice
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
// Inner service should be called twice (no caching)
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(2));
}
[Test]
public async Task TryMatchSeriesAsync_should_cache_results_when_enabled()
{
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9,
Reasoning = "Test match"
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// Call twice with same title
var result1 = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
var result2 = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
// Inner service should only be called once (second call uses cache)
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Once);
result1.Should().NotBeNull();
result2.Should().NotBeNull();
}
[Test]
public async Task TryMatchSeriesAsync_should_call_inner_service_for_different_titles()
{
var result1 = new LlmMatchResult
{
Series = _testSeries[0],
Confidence = 0.9
};
var result2 = new LlmMatchResult
{
Series = _testSeries[1],
Confidence = 0.85
};
_innerService
.SetupSequence(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(result1)
.ReturnsAsync(result2);
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Game.of.Thrones.S01E01", _testSeries);
// Inner service should be called twice (different titles)
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(2));
}
}
[TestFixture]
public class RateLimitedLlmSeriesMatchingServiceFixture
{
private Mock<IConfigService> _configService;
private Mock<IHttpClientFactory> _httpClientFactory;
private Mock<CachedLlmSeriesMatchingService> _innerService;
private RateLimitedLlmSeriesMatchingService _subject;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_configService = new Mock<IConfigService>();
_httpClientFactory = new Mock<IHttpClientFactory>();
var openAiService = new Mock<OpenAiSeriesMatchingService>(
_configService.Object,
_httpClientFactory.Object,
Mock.Of<Logger>());
_innerService = new Mock<CachedLlmSeriesMatchingService>(
openAiService.Object,
_configService.Object,
Mock.Of<Logger>());
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(60);
_subject = new RateLimitedLlmSeriesMatchingService(
_innerService.Object,
_configService.Object,
Mock.Of<Logger>());
_testSeries = new List<Series>
{
new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" }
};
}
[Test]
public void IsEnabled_should_delegate_to_inner_service()
{
_innerService.Setup(s => s.IsEnabled).Returns(true);
_subject.IsEnabled.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_should_allow_calls_within_rate_limit()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(5);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// Make 5 calls (within limit)
for (var i = 0; i < 5; i++)
{
var result = await _subject.TryMatchSeriesAsync($"Title{i}.S01E01", _testSeries);
result.Should().NotBeNull();
}
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(5));
}
[Test]
public async Task TryMatchSeriesAsync_should_return_null_when_rate_limit_exceeded()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(2);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// Make 2 calls (at limit)
await _subject.TryMatchSeriesAsync("Title1.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Title2.S01E01", _testSeries);
// Third call should be rate limited
var result = await _subject.TryMatchSeriesAsync("Title3.S01E01", _testSeries);
result.Should().BeNull();
// Inner service should only be called twice
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(2));
}
[Test]
public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_should_respect_rate_limit()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(1);
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad",
ReleaseTitle = "Breaking.Bad.S01E01"
};
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<ParsedEpisodeInfo>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// First call should succeed
var result1 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
result1.Should().NotBeNull();
// Second call should be rate limited
var result2 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
result2.Should().BeNull();
}
}
[TestFixture]
public class ParsingServiceLlmIntegrationFixture
{
private Mock<ILlmSeriesMatchingService> _llmService;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_llmService = new Mock<ILlmSeriesMatchingService>();
_testSeries = new List<Series>
{
new Series
{
Id = 1,
TvdbId = 81189,
Title = "Breaking Bad",
CleanTitle = "breakingbad",
Year = 2008
},
new Series
{
Id = 2,
TvdbId = 267440,
Title = "Attack on Titan",
CleanTitle = "attackontitan",
Year = 2013,
SeriesType = SeriesTypes.Anime
}
};
}
[Test]
public void LlmService_should_not_be_called_when_traditional_matching_succeeds()
{
// This tests that LLM is only used as fallback
_llmService.Setup(s => s.IsEnabled).Returns(true);
// LLM should never be called if traditional matching works
_llmService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Never);
_llmService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<ParsedEpisodeInfo>(), It.IsAny<IEnumerable<Series>>()),
Times.Never);
}
[Test]
public async Task LlmService_should_be_called_when_traditional_matching_fails()
{
_llmService.Setup(s => s.IsEnabled).Returns(true);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.88,
Reasoning = "Matched after removing language tag"
};
_llmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<ParsedEpisodeInfo>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// Simulate call that would happen from ParsingService
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad German",
ReleaseTitle = "Breaking.Bad.German.S01E01.720p"
};
var result = await _llmService.Object.TryMatchSeriesAsync(parsedInfo, _testSeries);
result.Should().NotBeNull();
result.IsSuccessfulMatch.Should().BeTrue();
result.Series.Title.Should().Be("Breaking Bad");
}
[Test]
public async Task LlmService_should_handle_anime_alternate_titles()
{
_llmService.Setup(s => s.IsEnabled).Returns(true);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(s => s.TvdbId == 267440),
Confidence = 0.92,
Reasoning = "Shingeki no Kyojin is the Japanese title for Attack on Titan"
};
_llmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
var result = await _llmService.Object.TryMatchSeriesAsync(
"[SubGroup] Shingeki no Kyojin - 01 [1080p].mkv",
_testSeries);
result.Should().NotBeNull();
result.Series.Title.Should().Be("Attack on Titan");
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public async Task LlmService_should_return_null_when_disabled()
{
_llmService.Setup(s => s.IsEnabled).Returns(false);
_llmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync((LlmMatchResult)null);
var result = await _llmService.Object.TryMatchSeriesAsync("Some.Title.S01E01", _testSeries);
result.Should().BeNull();
}
[Test]
public async Task LlmService_low_confidence_should_not_auto_match()
{
_llmService.Setup(s => s.IsEnabled).Returns(true);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.45,
Reasoning = "Title is ambiguous, multiple possible matches",
Alternatives = new List<AlternativeMatch>
{
new AlternativeMatch
{
Series = _testSeries[1],
Confidence = 0.30,
Reasoning = "Could also be this series"
}
}
};
_llmService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
var result = await _llmService.Object.TryMatchSeriesAsync("Ambiguous.Title.S01E01", _testSeries);
result.Should().NotBeNull();
result.IsSuccessfulMatch.Should().BeFalse();
result.Alternatives.Should().HaveCount(1);
}
}
}

605
src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/LlmSeriesMatchingServiceIntegrationFixture.cs Normal file
View file

@ -0,0 +1,605 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using FluentAssertions;
using Moq;
using NLog;
using NUnit.Framework;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.LlmMatching;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests.LlmMatchingTests
{
/// <summary>
/// Integration tests for OpenAiSeriesMatchingService.
/// These tests verify service logic without making actual API calls.
/// </summary>
[TestFixture]
public class OpenAiSeriesMatchingServiceIntegrationFixture
{
private Mock<IConfigService> _configService;
private Mock<IHttpClientFactory> _httpClientFactory;
private OpenAiSeriesMatchingService _subject;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_configService = new Mock<IConfigService>();
_httpClientFactory = new Mock<IHttpClientFactory>();
_configService.Setup(s => s.LlmMatchingEnabled).Returns(true);
_configService.Setup(s => s.OpenAiApiKey).Returns("test-api-key-12345");
_configService.Setup(s => s.OpenAiApiEndpoint).Returns("https://api.openai.com/v1/chat/completions");
_configService.Setup(s => s.OpenAiModel).Returns("gpt-4o-mini");
_configService.Setup(s => s.LlmConfidenceThreshold).Returns(0.7);
_subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
_testSeries = new List<Series>
{
new Series
{
Id = 1,
TvdbId = 81189,
Title = "Breaking Bad",
CleanTitle = "breakingbad",
Year = 2008,
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 2,
TvdbId = 121361,
Title = "Game of Thrones",
CleanTitle = "gameofthrones",
Year = 2011,
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 3,
TvdbId = 267440,
Title = "Attack on Titan",
CleanTitle = "attackontitan",
Year = 2013,
SeriesType = SeriesTypes.Anime
}
};
}
[Test]
public void IsEnabled_returns_true_when_properly_configured()
{
_subject.IsEnabled.Should().BeTrue();
}
[Test]
public void IsEnabled_returns_false_when_disabled_in_config()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(false);
var subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
subject.IsEnabled.Should().BeFalse();
}
[Test]
public void IsEnabled_returns_false_when_api_key_empty()
{
_configService.Setup(s => s.OpenAiApiKey).Returns(string.Empty);
var subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
subject.IsEnabled.Should().BeFalse();
}
[Test]
public void IsEnabled_returns_false_when_api_key_null()
{
_configService.Setup(s => s.OpenAiApiKey).Returns((string)null);
var subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
subject.IsEnabled.Should().BeFalse();
}
[Test]
public void IsEnabled_returns_false_when_api_key_whitespace()
{
_configService.Setup(s => s.OpenAiApiKey).Returns(" ");
var subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
subject.IsEnabled.Should().BeFalse();
}
[Test]
public async Task TryMatchSeriesAsync_string_returns_null_when_disabled()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(false);
var subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
var result = await subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_string_returns_null_when_title_empty()
{
var result = await _subject.TryMatchSeriesAsync(string.Empty, _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_string_returns_null_when_title_null()
{
var result = await _subject.TryMatchSeriesAsync((string)null, _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_string_returns_null_when_series_list_empty()
{
var result = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", new List<Series>());
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_string_returns_null_when_series_list_null()
{
var result = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", null);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_ParsedEpisodeInfo_returns_null_when_disabled()
{
_configService.Setup(s => s.LlmMatchingEnabled).Returns(false);
var subject = new OpenAiSeriesMatchingService(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad",
ReleaseTitle = "Breaking.Bad.S01E01"
};
var result = await subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
result.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_ParsedEpisodeInfo_returns_null_when_no_title()
{
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = null,
ReleaseTitle = null
};
var result = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
result.Should().BeNull();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_true_at_threshold()
{
var result = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.7
};
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_true_above_threshold()
{
var result = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.95
};
result.IsSuccessfulMatch.Should().BeTrue();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_false_below_threshold()
{
var result = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.69
};
result.IsSuccessfulMatch.Should().BeFalse();
}
[Test]
public void LlmMatchResult_IsSuccessfulMatch_false_when_series_null()
{
var result = new LlmMatchResult
{
Series = null,
Confidence = 0.95
};
result.IsSuccessfulMatch.Should().BeFalse();
}
[Test]
public void LlmMatchResult_Alternatives_initialized_empty()
{
var result = new LlmMatchResult();
result.Alternatives.Should().NotBeNull();
result.Alternatives.Should().BeEmpty();
}
[Test]
public void LlmMatchResult_can_hold_multiple_alternatives()
{
var result = new LlmMatchResult
{
Series = _testSeries[0],
Confidence = 0.6,
Alternatives = new List<AlternativeMatch>
{
new AlternativeMatch { Series = _testSeries[1], Confidence = 0.4 },
new AlternativeMatch { Series = _testSeries[2], Confidence = 0.3 }
}
};
result.Alternatives.Should().HaveCount(2);
}
}
/// <summary>
/// Tests for the caching decorator service.
/// </summary>
[TestFixture]
public class CachedLlmSeriesMatchingServiceIntegrationFixture
{
private Mock<IConfigService> _configService;
private Mock<IHttpClientFactory> _httpClientFactory;
private Mock<OpenAiSeriesMatchingService> _innerService;
private CachedLlmSeriesMatchingService _subject;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_configService = new Mock<IConfigService>();
_httpClientFactory = new Mock<IHttpClientFactory>();
_configService.Setup(s => s.LlmCacheEnabled).Returns(true);
_configService.Setup(s => s.LlmCacheDurationHours).Returns(24);
_innerService = new Mock<OpenAiSeriesMatchingService>(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
_innerService.Setup(s => s.IsEnabled).Returns(true);
_subject = new CachedLlmSeriesMatchingService(
_innerService.Object,
_configService.Object,
LogManager.GetCurrentClassLogger());
_testSeries = new List<Series>
{
new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" },
new Series { Id = 2, TvdbId = 121361, Title = "Game of Thrones" }
};
}
[Test]
public void IsEnabled_delegates_to_inner_service()
{
_subject.IsEnabled.Should().BeTrue();
_innerService.Verify(s => s.IsEnabled, Times.Once);
}
[Test]
public async Task TryMatchSeriesAsync_calls_inner_service_on_cache_miss()
{
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9,
Reasoning = "Test"
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
var result = await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
result.Should().NotBeNull();
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Once);
}
[Test]
public async Task TryMatchSeriesAsync_returns_cached_result_on_cache_hit()
{
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9,
Reasoning = "Test"
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// First call - cache miss
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
// Second call - cache hit
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
// Inner service should only be called once
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Once);
}
[Test]
public async Task TryMatchSeriesAsync_bypasses_cache_when_disabled()
{
_configService.Setup(s => s.LlmCacheEnabled).Returns(false);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
// Inner service should be called twice (no caching)
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(2));
}
[Test]
public async Task TryMatchSeriesAsync_uses_different_cache_keys_for_different_titles()
{
var result1 = new LlmMatchResult { Series = _testSeries[0], Confidence = 0.9 };
var result2 = new LlmMatchResult { Series = _testSeries[1], Confidence = 0.85 };
_innerService
.SetupSequence(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(result1)
.ReturnsAsync(result2);
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Game.of.Thrones.S01E01", _testSeries);
// Both should result in calls to inner service
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(2));
}
}
/// <summary>
/// Tests for the rate limiting decorator service.
/// </summary>
[TestFixture]
public class RateLimitedLlmSeriesMatchingServiceIntegrationFixture
{
private Mock<IConfigService> _configService;
private Mock<IHttpClientFactory> _httpClientFactory;
private Mock<CachedLlmSeriesMatchingService> _innerService;
private RateLimitedLlmSeriesMatchingService _subject;
private List<Series> _testSeries;
[SetUp]
public void Setup()
{
_configService = new Mock<IConfigService>();
_httpClientFactory = new Mock<IHttpClientFactory>();
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(60);
_configService.Setup(s => s.LlmCacheEnabled).Returns(true);
_configService.Setup(s => s.LlmCacheDurationHours).Returns(24);
var openAiMock = new Mock<OpenAiSeriesMatchingService>(
_configService.Object,
_httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
_innerService = new Mock<CachedLlmSeriesMatchingService>(
openAiMock.Object,
_configService.Object,
LogManager.GetCurrentClassLogger());
_innerService.Setup(s => s.IsEnabled).Returns(true);
_subject = new RateLimitedLlmSeriesMatchingService(
_innerService.Object,
_configService.Object,
LogManager.GetCurrentClassLogger());
_testSeries = new List<Series>
{
new Series { Id = 1, TvdbId = 81189, Title = "Breaking Bad" }
};
}
[Test]
public void IsEnabled_delegates_to_inner_service()
{
_subject.IsEnabled.Should().BeTrue();
}
[Test]
public async Task TryMatchSeriesAsync_allows_calls_within_limit()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(10);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
var results = new List<LlmMatchResult>();
for (var i = 0; i < 10; i++)
{
var result = await _subject.TryMatchSeriesAsync($"Title{i}.S01E01", _testSeries);
results.Add(result);
}
results.Should().AllSatisfy(r => r.Should().NotBeNull());
}
[Test]
public async Task TryMatchSeriesAsync_blocks_calls_over_limit()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(3);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// Make 3 calls (at limit)
await _subject.TryMatchSeriesAsync("Title1.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Title2.S01E01", _testSeries);
await _subject.TryMatchSeriesAsync("Title3.S01E01", _testSeries);
// 4th call should be blocked
var result = await _subject.TryMatchSeriesAsync("Title4.S01E01", _testSeries);
result.Should().BeNull();
// Only 3 calls should have reached inner service
_innerService.Verify(
s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()),
Times.Exactly(3));
}
[Test]
public async Task TryMatchSeriesAsync_with_ParsedEpisodeInfo_respects_rate_limit()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(2);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<ParsedEpisodeInfo>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad",
ReleaseTitle = "Breaking.Bad.S01E01"
};
// First two calls succeed
var result1 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
var result2 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
// Third call blocked
var result3 = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
result1.Should().NotBeNull();
result2.Should().NotBeNull();
result3.Should().BeNull();
}
[Test]
public async Task TryMatchSeriesAsync_rate_limit_applies_across_both_methods()
{
_configService.Setup(s => s.LlmMaxCallsPerHour).Returns(2);
var expectedResult = new LlmMatchResult
{
Series = _testSeries.First(),
Confidence = 0.9
};
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<string>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
_innerService
.Setup(s => s.TryMatchSeriesAsync(It.IsAny<ParsedEpisodeInfo>(), It.IsAny<IEnumerable<Series>>()))
.ReturnsAsync(expectedResult);
// One call with string
await _subject.TryMatchSeriesAsync("Breaking.Bad.S01E01", _testSeries);
// One call with ParsedEpisodeInfo
var parsedInfo = new ParsedEpisodeInfo { SeriesTitle = "Test", ReleaseTitle = "Test.S01E01" };
await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
// Third call (either type) should be blocked
var result = await _subject.TryMatchSeriesAsync("Another.Title.S01E01", _testSeries);
result.Should().BeNull();
}
}
}

992
src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/RealOpenAiIntegrationFixture.cs Normal file
View file

@ -0,0 +1,992 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Text.Json;
using System.Threading.Tasks;
using FluentAssertions;
using Moq;
using NLog;
using NUnit.Framework;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.LlmMatching;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Test.ParserTests.ParsingServiceTests.LlmMatchingTests
{
/// <summary>
/// Real integration tests that make actual API calls to OpenAI.
/// These tests require a valid API key in appsettings.llm.local.json.
///
/// SETUP:
/// 1. Copy appsettings.llm.template.json to appsettings.llm.local.json
/// 2. Add your OpenAI API key to the local file
/// 3. Ensure the file is set to "Copy to Output Directory: Copy if newer" in Visual Studio
/// 4. Run tests with: dotnet test --filter "FullyQualifiedName~RealOpenAiIntegrationFixture"
///
/// NOTE: These tests are marked [Explicit] and won't run during normal test execution.
/// They make real API calls which incur costs and require network access.
/// </summary>
[TestFixture]
[Explicit("Requires real OpenAI API key and makes actual API calls")]
[Category("Integration")]
[Category("LlmMatching")]
[Category("ExternalApi")]
public class RealOpenAiIntegrationFixture
{
private OpenAiSeriesMatchingService _subject;
private List<Series> _testSeries;
private LlmTestSettings _settings;
private bool _isConfigured;
[OneTimeSetUp]
public void OneTimeSetup()
{
_settings = LoadSettings();
_isConfigured = !string.IsNullOrWhiteSpace(_settings?.OpenAiApiKey) &&
!_settings.OpenAiApiKey.StartsWith("sk-your-");
if (!_isConfigured)
{
TestContext.WriteLine("========================================");
TestContext.WriteLine("WARNING: OpenAI API key not configured.");
TestContext.WriteLine("========================================");
TestContext.WriteLine("");
TestContext.WriteLine("Setup steps:");
TestContext.WriteLine("1. Copy appsettings.llm.template.json to appsettings.llm.local.json");
TestContext.WriteLine("2. Add your OpenAI API key to appsettings.llm.local.json");
TestContext.WriteLine("3. In Visual Studio: Right-click the file -> Properties");
TestContext.WriteLine(" Set 'Copy to Output Directory' = 'Copy if newer'");
TestContext.WriteLine("");
TestContext.WriteLine("Or set environment variable: OPENAI_API_KEY=sk-...");
TestContext.WriteLine("");
TestContext.WriteLine("Searched locations:");
foreach (var path in GetSearchPaths())
{
var exists = File.Exists(path) ? "[FOUND]" : "[NOT FOUND]";
TestContext.WriteLine($" {exists} {path}");
}
}
else
{
TestContext.WriteLine($"OpenAI API configured. Using model: {_settings.OpenAiModel}");
}
}
[SetUp]
public void Setup()
{
if (!_isConfigured)
{
Assert.Ignore("OpenAI API key not configured. Skipping real API tests.");
return;
}
var configService = new Mock<IConfigService>();
configService.Setup(s => s.LlmMatchingEnabled).Returns(true);
configService.Setup(s => s.OpenAiApiKey).Returns(_settings.OpenAiApiKey);
configService.Setup(s => s.OpenAiApiEndpoint).Returns(_settings.OpenAiApiEndpoint);
configService.Setup(s => s.OpenAiModel).Returns(_settings.OpenAiModel);
configService.Setup(s => s.LlmConfidenceThreshold).Returns(_settings.ConfidenceThreshold);
var httpClientFactory = new Mock<IHttpClientFactory>();
httpClientFactory
.Setup(f => f.CreateClient(It.IsAny<string>()))
.Returns(new HttpClient());
_subject = new OpenAiSeriesMatchingService(
configService.Object,
httpClientFactory.Object,
LogManager.GetCurrentClassLogger());
// Setup test series library - simulates a real user's library
_testSeries = new List<Series>
{
new Series
{
Id = 1,
TvdbId = 81189,
Title = "Breaking Bad",
CleanTitle = "breakingbad",
Year = 2008,
Network = "AMC",
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 2,
TvdbId = 121361,
Title = "Game of Thrones",
CleanTitle = "gameofthrones",
Year = 2011,
Network = "HBO",
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 3,
TvdbId = 267440,
Title = "Attack on Titan",
CleanTitle = "attackontitan",
Year = 2013,
SeriesType = SeriesTypes.Anime
},
new Series
{
Id = 4,
TvdbId = 153021,
Title = "The Walking Dead",
CleanTitle = "thewalkingdead",
Year = 2010,
Network = "AMC",
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 5,
TvdbId = 295759,
Title = "Stranger Things",
CleanTitle = "strangerthings",
Year = 2016,
Network = "Netflix",
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 6,
TvdbId = 305288,
Title = "Westworld",
CleanTitle = "westworld",
Year = 2016,
Network = "HBO",
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 7,
TvdbId = 78804,
Title = "Doctor Who",
CleanTitle = "doctorwho",
Year = 2005,
Network = "BBC",
SeriesType = SeriesTypes.Standard
},
new Series
{
Id = 8,
TvdbId = 73255,
Title = "Doctor Who",
CleanTitle = "doctorwho",
Year = 1963,
Network = "BBC",
SeriesType = SeriesTypes.Standard
}
};
}
[Test]
public async Task Should_match_standard_release_title()
{
// Arrange
var releaseTitle = "Breaking.Bad.S01E01.720p.BluRay.x264-DEMAND";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
result.Series.Title.Should().Be("Breaking Bad");
result.Confidence.Should().BeGreaterOrEqualTo(0.7);
result.IsSuccessfulMatch.Should().BeTrue();
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_match_release_with_dots_as_spaces()
{
// Arrange
var releaseTitle = "Game.of.Thrones.S08E06.The.Iron.Throne.1080p.AMZN.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(121361);
result.Confidence.Should().BeGreaterOrEqualTo(0.7);
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_match_anime_with_japanese_title()
{
// Arrange - Japanese title for Attack on Titan
var releaseTitle = "[SubGroup] Shingeki no Kyojin - 01 [1080p][HEVC]";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(267440);
result.Series.Title.Should().Be("Attack on Titan");
result.Confidence.Should().BeGreaterOrEqualTo(0.7);
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_disambiguate_by_year()
{
// Arrange - Should match 2005 Doctor Who, not 1963
var releaseTitle = "Doctor.Who.2005.S13E01.720p.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(78804); // 2005 version
result.Series.Year.Should().Be(2005);
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_release_with_language_tag()
{
// Arrange - German release
var releaseTitle = "Breaking.Bad.S01E01.GERMAN.720p.BluRay.x264";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
result.Reasoning.Should().NotBeNullOrEmpty();
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_release_with_scene_group_tags()
{
// Arrange
var releaseTitle = "Stranger.Things.S04E09.Chapter.Nine.The.Piggyback.2160p.NF.WEB-DL.DDP5.1.Atmos.DV.HDR.H.265-FLUX";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(295759);
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_abbreviated_title()
{
// Arrange - TWD is a common abbreviation for The Walking Dead
var releaseTitle = "TWD.S11E24.Rest.in.Peace.1080p.AMZN.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
// Note: LLM might or might not recognize TWD abbreviation
// This test verifies the LLM can handle ambiguous cases
if (result.Series != null)
{
TestContext.WriteLine($"LLM recognized TWD as: {result.Series.Title}");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_miscoded_german_umlauts_utf8_as_latin1()
{
// Arrange - "Für" miscoded as "Für" (UTF-8 bytes interpreted as Latin-1)
// This happens when UTF-8 encoded text is read as ISO-8859-1
var releaseTitle = "Breaking.Bad.S01E01.German.Für.immer.720p.BluRay.x264";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine($"LLM handled miscoded umlaut 'ü' (should be 'ü')");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_miscoded_german_umlauts_various()
{
// Arrange - Various miscoded German umlauts
// ä -> ä, ö -> ö, ü -> ü, ß -> ß
var releaseTitle = "Breaking.Bad.S02E01.Grüße.aus.Köln.GERMAN.720p.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled multiple miscoded umlauts: 'ü'='ü', 'ß'='ß', 'ö'='ö'");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_correct_german_umlauts()
{
// Arrange - Correctly encoded German umlauts for comparison
var releaseTitle = "Breaking.Bad.S01E01.Grüße.aus.Köln.GERMAN.720p.BluRay.x264";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled correct German umlauts: ü, ö, ß");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_japanese_characters_in_anime_release()
{
// Arrange - Japanese title with kanji/hiragana
// 進撃の巨人 = Shingeki no Kyojin = Attack on Titan
var releaseTitle = "[SubGroup] 進撃の巨人 - 01 [1080p][HEVC].mkv";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM recognized Japanese '進撃の巨人' as: {result.Series.Title}");
// Should match Attack on Titan
if (result.Series.TvdbId == 267440)
{
TestContext.WriteLine("SUCCESS: Correctly identified as Attack on Titan");
}
}
else
{
TestContext.WriteLine("LLM could not match Japanese kanji title");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_mixed_japanese_english_title()
{
// Arrange - Mixed Japanese and English (common in anime releases)
var releaseTitle = "[Erai-raws] Shingeki no Kyojin - The Final Season - 01 [1080p][HEVC].mkv";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(267440);
TestContext.WriteLine("LLM handled mixed Japanese/English anime title");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_chinese_characters()
{
// Arrange - Chinese title for Attack on Titan (進擊的巨人 - Traditional Chinese)
var releaseTitle = "[字幕组] 進擊的巨人 - 01 [1080p].mkv";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM recognized Chinese '進擊的巨人' as: {result.Series.Title}");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_korean_characters()
{
// Arrange - Korean title (게임 오브 스론스 = Game of Thrones)
var releaseTitle = "게임.오브.스론스.S08E06.1080p.WEB-DL.KOR";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM recognized Korean '게임 오브 스론스' as: {result.Series.Title}");
if (result.Series.TvdbId == 121361)
{
TestContext.WriteLine("SUCCESS: Correctly identified as Game of Thrones");
}
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_cyrillic_characters()
{
// Arrange - Russian title (Во все тяжкие = Breaking Bad)
var releaseTitle = "Во.все.тяжкие.S01E01.720p.BluRay.RUS";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM recognized Cyrillic 'Во все тяжкие' as: {result.Series.Title}");
if (result.Series.TvdbId == 81189)
{
TestContext.WriteLine("SUCCESS: Correctly identified as Breaking Bad");
}
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_arabic_characters()
{
// Arrange - Arabic title (صراع العروش = Game of Thrones)
var releaseTitle = "صراع.العروش.S08E06.1080p.WEB-DL.ARA";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM recognized Arabic 'صراع العروش' as: {result.Series.Title}");
if (result.Series.TvdbId == 121361)
{
TestContext.WriteLine("SUCCESS: Correctly identified as Game of Thrones");
}
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_french_accents()
{
// Arrange - French accented characters
var releaseTitle = "Breaking.Bad.S01E01.FRENCH.Épisode.Spécial.720p.BluRay.x264";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled French accents: é, è, ê, ç");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_miscoded_french_accents()
{
// Arrange - Miscoded French (é -> é)
var releaseTitle = "Breaking.Bad.S01E01.FRENCH.Épisode.Spécial.720p.BluRay.x264";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled miscoded French accents: 'é'='é', 'É'='É'");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_spanish_characters()
{
// Arrange - Spanish with ñ and inverted punctuation
var releaseTitle = "Breaking.Bad.S01E01.SPANISH.El.Año.del.Dragón.720p.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled Spanish ñ character");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_polish_characters()
{
// Arrange - Polish special characters (ą, ę, ł, ń, ó, ś, ź, ż)
var releaseTitle = "Breaking.Bad.S01E01.POLISH.Żółć.i.Gęś.720p.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled Polish characters: ż, ó, ł, ć, ę, ś");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_turkish_characters()
{
// Arrange - Turkish special characters (ç, ğ, ı, ş, ö, ü)
var releaseTitle = "Breaking.Bad.S01E01.TURKISH.Güçlü.Şef.720p.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled Turkish characters: ü, ç, ş, ğ, ı");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_double_encoded_utf8()
{
// Arrange - Double-encoded UTF-8 (ü -> ü -> Ƽ)
// This happens when already-encoded UTF-8 is encoded again
var releaseTitle = "Breaking.Bad.S01E01.Grüße.GERMAN.720p.BluRay";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM handled double-encoded UTF-8, matched to: {result.Series.Title}");
}
else
{
TestContext.WriteLine("LLM could not parse double-encoded UTF-8 (expected behavior)");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_replacement_characters()
{
// Arrange - Unicode replacement characters (common when encoding fails)
var releaseTitle = "Breaking.Bad.S01E01.Gr<47><72>e.GERMAN.720p.BluRay";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
TestContext.WriteLine("LLM handled replacement characters (<28>)");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_html_entities_in_title()
{
// Arrange - HTML entities (sometimes appear in scraped titles)
var releaseTitle = "Breaking.Bad.S01E01.Gr&uuml;&szlig;e.GERMAN.720p.BluRay";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM handled HTML entities (&uuml; &szlig;), matched to: {result.Series.Title}");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_url_encoded_characters()
{
// Arrange - URL encoded characters
var releaseTitle = "Breaking.Bad.S01E01.Gr%C3%BC%C3%9Fe.GERMAN.720p.BluRay";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
if (result.Series != null)
{
TestContext.WriteLine($"LLM handled URL-encoded characters, matched to: {result.Series.Title}");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_mixed_encoding_issues()
{
// Arrange - Mix of different encoding problems in one title
var releaseTitle = "[SubGroup] Shingeki no Kyojin - 進撃の巨人 - Attack.on.Titan.S04E01.Germän.Duß.1080p";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(267440);
TestContext.WriteLine("LLM handled mixed Japanese + miscoded German in same title");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_return_low_confidence_for_ambiguous_title()
{
// Arrange - Completely made up title that doesn't match anything well
var releaseTitle = "The.Show.About.Nothing.S01E01.720p.WEB-DL";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
// Assert
result.Should().NotBeNull();
// Should either have low confidence or no match
if (result.Series != null)
{
TestContext.WriteLine($"LLM guessed: {result.Series.Title} with {result.Confidence:P0} confidence");
}
else
{
TestContext.WriteLine("LLM correctly returned no match");
}
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_match_using_ParsedEpisodeInfo()
{
// Arrange
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Breaking Bad",
ReleaseTitle = "Breaking.Bad.S05E16.Felina.1080p.BluRay.x264-DEMAND",
SeasonNumber = 5,
EpisodeNumbers = new[] { 16 }
};
// Act
var result = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
result.IsSuccessfulMatch.Should().BeTrue();
LogResult(parsedInfo.ReleaseTitle, result);
}
[Test]
public async Task Should_use_additional_metadata_from_ParsedEpisodeInfo()
{
// Arrange - Anime with absolute numbering
var parsedInfo = new ParsedEpisodeInfo
{
SeriesTitle = "Shingeki no Kyojin",
ReleaseTitle = "[HorribleSubs] Shingeki no Kyojin - 25 [1080p].mkv",
AbsoluteEpisodeNumbers = new[] { 25 }
};
// Act
var result = await _subject.TryMatchSeriesAsync(parsedInfo, _testSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(267440);
result.Series.SeriesType.Should().Be(SeriesTypes.Anime);
LogResult(parsedInfo.ReleaseTitle, result);
}
[Test]
public async Task Should_complete_within_reasonable_time()
{
// Arrange
var releaseTitle = "Breaking.Bad.S01E01.720p.BluRay.x264-DEMAND";
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, _testSeries);
stopwatch.Stop();
// Assert
result.Should().NotBeNull();
stopwatch.ElapsedMilliseconds.Should().BeLessThan(10000, "API call should complete within 10 seconds");
TestContext.WriteLine($"API call completed in {stopwatch.ElapsedMilliseconds}ms");
LogResult(releaseTitle, result);
}
[Test]
public async Task Should_handle_large_series_list()
{
// Arrange - Create a larger series list
var largeSeries = new List<Series>(_testSeries);
for (var i = 0; i < 50; i++)
{
largeSeries.Add(new Series
{
Id = 100 + i,
TvdbId = 100000 + i,
Title = $"Test Series {i}",
CleanTitle = $"testseries{i}",
Year = 2000 + i
});
}
var releaseTitle = "Breaking.Bad.S01E01.720p.BluRay.x264-DEMAND";
// Act
var result = await _subject.TryMatchSeriesAsync(releaseTitle, largeSeries);
// Assert
result.Should().NotBeNull();
result.Series.Should().NotBeNull();
result.Series.TvdbId.Should().Be(81189);
LogResult(releaseTitle, result);
}
private static IEnumerable<string> GetSearchPaths()
{
var fileName = "appsettings.llm.local.json";
// Get various base directories
var testDir = TestContext.CurrentContext.TestDirectory;
var baseDir = AppContext.BaseDirectory;
var currentDir = Directory.GetCurrentDirectory();
var assemblyDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
// Build list of search paths
var paths = new List<string>
{
// Output directory (where tests run from)
Path.Combine(testDir, fileName),
Path.Combine(baseDir, fileName),
Path.Combine(assemblyDir ?? "", fileName),
Path.Combine(currentDir, fileName),
// Source directory structure (for running from IDE)
Path.Combine(testDir, "ParserTests", "ParsingServiceTests", "LlmMatchingTests", fileName),
// Walk up from output directory to find source
Path.Combine(testDir, "..", "..", "..", "ParserTests", "ParsingServiceTests", "LlmMatchingTests", fileName),
Path.Combine(testDir, "..", "..", "..", "..", "ParserTests", "ParsingServiceTests", "LlmMatchingTests", fileName),
// Common source locations on Windows
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
"source",
"repos",
"Sonarr",
"src",
"NzbDrone.Core.Test",
"ParserTests",
"ParsingServiceTests",
"LlmMatchingTests",
fileName),
};
return paths.Select(Path.GetFullPath).Distinct();
}
private LlmTestSettings LoadSettings()
{
// Try all search paths
foreach (var path in GetSearchPaths())
{
var settings = TryLoadFromFile(path);
if (settings != null)
{
TestContext.WriteLine($"Loaded settings from: {path}");
return settings;
}
}
// Try environment variable as fallback
var envApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY");
if (!string.IsNullOrWhiteSpace(envApiKey))
{
TestContext.WriteLine("Loaded API key from OPENAI_API_KEY environment variable");
return new LlmTestSettings
{
OpenAiApiKey = envApiKey,
OpenAiApiEndpoint = Environment.GetEnvironmentVariable("OPENAI_API_ENDPOINT")
?? "https://api.openai.com/v1/chat/completions",
OpenAiModel = Environment.GetEnvironmentVariable("OPENAI_MODEL")
?? "gpt-4o-mini",
ConfidenceThreshold = 0.7
};
}
return new LlmTestSettings();
}
private static LlmTestSettings TryLoadFromFile(string path)
{
if (!File.Exists(path))
{
return null;
}
try
{
var json = File.ReadAllText(path);
var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
if (root.TryGetProperty("LlmMatching", out var llmSection))
{
return new LlmTestSettings
{
OpenAiApiKey = GetStringProperty(llmSection, "OpenAiApiKey"),
OpenAiApiEndpoint = GetStringProperty(llmSection, "OpenAiApiEndpoint")
?? "https://api.openai.com/v1/chat/completions",
OpenAiModel = GetStringProperty(llmSection, "OpenAiModel") ?? "gpt-4o-mini",
ConfidenceThreshold = GetDoubleProperty(llmSection, "ConfidenceThreshold") ?? 0.7
};
}
}
catch (Exception ex)
{
TestContext.WriteLine($"Error loading settings from {path}: {ex.Message}");
}
return null;
}
private static string GetStringProperty(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var prop) ? prop.GetString() : null;
}
private static double? GetDoubleProperty(JsonElement element, string propertyName)
{
return element.TryGetProperty(propertyName, out var prop) ? prop.GetDouble() : null;
}
private static void LogResult(string releaseTitle, LlmMatchResult result)
{
TestContext.WriteLine($"\n--- LLM Matching Result ---");
TestContext.WriteLine($"Release: {releaseTitle}");
if (result?.Series != null)
{
TestContext.WriteLine($"Matched: {result.Series.Title} (TvdbId: {result.Series.TvdbId})");
TestContext.WriteLine($"Confidence: {result.Confidence:P0}");
TestContext.WriteLine($"Successful: {result.IsSuccessfulMatch}");
}
else
{
TestContext.WriteLine("Matched: No match found");
}
if (!string.IsNullOrWhiteSpace(result?.Reasoning))
{
TestContext.WriteLine($"Reasoning: {result.Reasoning}");
}
if (result?.Alternatives?.Any() == true)
{
TestContext.WriteLine("Alternatives:");
foreach (var alt in result.Alternatives)
{
TestContext.WriteLine($" - {alt.Series?.Title} ({alt.Confidence:P0}): {alt.Reasoning}");
}
}
TestContext.WriteLine("----------------------------\n");
}
private class LlmTestSettings
{
public string OpenAiApiKey { get; set; }
public string OpenAiApiEndpoint { get; set; }
public string OpenAiModel { get; set; }
public double ConfidenceThreshold { get; set; }
}
}
}

8
src/NzbDrone.Core.Test/ParserTests/ParsingServiceTests/LlmMatchingTests/appsettings.llm.template.json Normal file
View file

@ -0,0 +1,8 @@
{
"LlmMatching": {
"OpenAiApiKey": "sk-your-api-key-here",
"OpenAiApiEndpoint": "https://api.openai.com/v1/chat/completions",
"OpenAiModel": "gpt-4o-mini",
"ConfidenceThreshold": 0.7
}
}

11
src/NzbDrone.Core/Configuration/ConfigService.cs
View file

@ -421,6 +421,17 @@ public bool TrustCgnatIpAddresses
set { SetValue("TrustCgnatIpAddresses", value); }
}
// LLM Configuration
public bool LlmMatchingEnabled => GetValueBoolean("LlmMatchingEnabled", false);
public string OpenAiApiKey => GetValue("OpenAiApiKey", string.Empty, true);
public string OpenAiApiEndpoint => GetValue("OpenAiApiEndpoint", "https://api.openai.com/v1/chat/completions");
public string OpenAiModel => GetValue("OpenAiModel", "gpt-4o-mini");
public double LlmConfidenceThreshold => GetValueDouble("LlmConfidenceThreshold", 0.7);
public int LlmMaxCallsPerHour => GetValueInt("LlmMaxCallsPerHour", 60);
public bool LlmCacheEnabled => GetValueBoolean("LlmCacheEnabled", true);
public int LlmCacheDurationHours => GetValueInt("LlmCacheDurationHours", 24);
private string GetValue(string key)
{
return GetValue(key, string.Empty);

10
src/NzbDrone.Core/Configuration/IConfigService.cs
View file

@ -100,5 +100,15 @@ public interface IConfigService
CertificateValidationType CertificateValidation { get; }
string ApplicationUrl { get; }
// LLM Settings
bool LlmMatchingEnabled { get; }
string OpenAiApiKey { get; }
string OpenAiApiEndpoint { get; }
string OpenAiModel { get; }
double LlmConfidenceThreshold { get; }
int LlmMaxCallsPerHour { get; }
bool LlmCacheEnabled { get; }
int LlmCacheDurationHours { get; }
}
}

247
src/NzbDrone.Core/Parser/LlmMatching/CachedLlmSeriesMatchingService.cs Normal file
View file

@ -0,0 +1,247 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Parser.LlmMatching
{
/// <summary>
/// Decorator service that adds caching to the LLM matching service.
/// Helps reduce API costs by caching responses for identical queries.
/// </summary>
public class CachedLlmSeriesMatchingService : ILlmSeriesMatchingService
{
private readonly ILlmSeriesMatchingService _innerService;
private readonly IConfigService _configService;
private readonly Logger _logger;
private readonly ConcurrentDictionary<string, CachedResult> _cache;
private DateTime _lastCleanup = DateTime.UtcNow;
public CachedLlmSeriesMatchingService(
OpenAiSeriesMatchingService innerService,
IConfigService configService,
Logger logger)
{
_innerService = innerService;
_configService = configService;
_logger = logger;
_cache = new ConcurrentDictionary<string, CachedResult>(StringComparer.OrdinalIgnoreCase);
}
public virtual bool IsEnabled => _innerService.IsEnabled;
public virtual async Task<LlmMatchResult> TryMatchSeriesAsync(
ParsedEpisodeInfo parsedEpisodeInfo,
IEnumerable<Series> availableSeries)
{
if (!_configService.LlmCacheEnabled)
{
return await _innerService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries);
}
var cacheKey = GenerateCacheKey(
parsedEpisodeInfo?.ReleaseTitle ?? parsedEpisodeInfo?.SeriesTitle,
availableSeries);
if (TryGetFromCache(cacheKey, availableSeries, out var cachedResult))
{
_logger.Trace("LLM cache hit for '{0}'", parsedEpisodeInfo?.SeriesTitle);
return cachedResult;
}
var result = await _innerService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries);
if (result != null)
{
AddToCache(cacheKey, result);
}
return result;
}
public virtual async Task<LlmMatchResult> TryMatchSeriesAsync(
string releaseTitle,
IEnumerable<Series> availableSeries)
{
if (!_configService.LlmCacheEnabled)
{
return await _innerService.TryMatchSeriesAsync(releaseTitle, availableSeries);
}
var cacheKey = GenerateCacheKey(releaseTitle, availableSeries);
if (TryGetFromCache(cacheKey, availableSeries, out var cachedResult))
{
_logger.Trace("LLM cache hit for '{0}'", releaseTitle);
return cachedResult;
}
var result = await _innerService.TryMatchSeriesAsync(releaseTitle, availableSeries);
if (result != null)
{
AddToCache(cacheKey, result);
}
return result;
}
private string GenerateCacheKey(string title, IEnumerable<Series> availableSeries)
{
if (title.IsNullOrWhiteSpace())
{
return string.Empty;
}
var normalizedTitle = title.CleanSeriesTitle();
var seriesHash = availableSeries?.Sum(s => s.TvdbId) ?? 0;
return $"{normalizedTitle}|{seriesHash}";
}
private bool TryGetFromCache(
string cacheKey,
IEnumerable<Series> availableSeries,
out LlmMatchResult result)
{
result = null;
if (cacheKey.IsNullOrWhiteSpace())
{
return false;
}
CleanupExpiredEntriesIfNeeded();
if (_cache.TryGetValue(cacheKey, out var cached))
{
var cacheDuration = TimeSpan.FromHours(_configService.LlmCacheDurationHours);
if (DateTime.UtcNow - cached.Timestamp < cacheDuration)
{
result = RehydrateResult(cached.Result, availableSeries);
return result != null;
}
_cache.TryRemove(cacheKey, out _);
}
return false;
}
private void AddToCache(string cacheKey, LlmMatchResult result)
{
if (cacheKey.IsNullOrWhiteSpace() || result == null)
{
return;
}
var cached = new CachedResult
{
Timestamp = DateTime.UtcNow,
Result = DehydrateResult(result)
};
_cache.AddOrUpdate(cacheKey, cached, (_, _) => cached);
_logger.Trace("Added LLM result to cache for key: {0}", cacheKey);
}
private LlmMatchResult DehydrateResult(LlmMatchResult result)
{
return new LlmMatchResult
{
Series = result.Series != null ? new Series { TvdbId = result.Series.TvdbId } : null,
Confidence = result.Confidence,
Reasoning = result.Reasoning,
Alternatives = result.Alternatives?.Select(a => new AlternativeMatch
{
Series = a.Series != null ? new Series { TvdbId = a.Series.TvdbId } : null,
Confidence = a.Confidence,
Reasoning = a.Reasoning
}).ToList() ?? new List<AlternativeMatch>()
};
}
private LlmMatchResult RehydrateResult(LlmMatchResult cached, IEnumerable<Series> availableSeries)
{
var seriesLookup = availableSeries?.ToDictionary(s => s.TvdbId) ?? new Dictionary<int, Series>();
var result = new LlmMatchResult
{
Confidence = cached.Confidence,
Reasoning = cached.Reasoning
};
if (cached.Series?.TvdbId > 0 && seriesLookup.TryGetValue(cached.Series.TvdbId, out var series))
{
result.Series = series;
}
else if (cached.Series != null)
{
return null;
}
result.Alternatives = cached.Alternatives?
.Select(a =>
{
if (a.Series?.TvdbId > 0 && seriesLookup.TryGetValue(a.Series.TvdbId, out var altSeries))
{
return new AlternativeMatch
{
Series = altSeries,
Confidence = a.Confidence,
Reasoning = a.Reasoning
};
}
return null;
})
.Where(a => a != null)
.ToList() ?? new List<AlternativeMatch>();
return result;
}
private void CleanupExpiredEntriesIfNeeded()
{
if (DateTime.UtcNow - _lastCleanup < TimeSpan.FromMinutes(10))
{
return;
}
_lastCleanup = DateTime.UtcNow;
var cacheDuration = TimeSpan.FromHours(_configService.LlmCacheDurationHours);
var cutoff = DateTime.UtcNow - cacheDuration;
var expiredKeys = _cache
.Where(kvp => kvp.Value.Timestamp < cutoff)
.Select(kvp => kvp.Key)
.ToList();
foreach (var key in expiredKeys)
{
_cache.TryRemove(key, out _);
}
if (expiredKeys.Any())
{
_logger.Debug("Cleaned up {0} expired LLM cache entries", expiredKeys.Count);
}
}
private class CachedResult
{
public DateTime Timestamp { get; set; }
public LlmMatchResult Result { get; set; }
}
}
}

76
src/NzbDrone.Core/Parser/LlmMatching/ILlmSeriesMatchingService.cs Normal file
View file

@ -0,0 +1,76 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Parser.LlmMatching
{
/// <summary>
/// Service interface for LLM-based series matching when traditional parsing fails.
/// Acts as a fallback mechanism before requiring manual user intervention.
/// </summary>
public interface ILlmSeriesMatchingService
{
/// <summary>
/// Checks if the LLM service is properly configured and available.
/// </summary>
bool IsEnabled { get; }
/// <summary>
/// Attempts to match a parsed release title to a series using LLM intelligence.
/// </summary>
/// <param name="parsedEpisodeInfo">The parsed episode information from the release title.</param>
/// <param name="availableSeries">List of series available in the user's library.</param>
/// <returns>The matching result containing the series and confidence score, or null if no match found.</returns>
Task<LlmMatchResult> TryMatchSeriesAsync(ParsedEpisodeInfo parsedEpisodeInfo, IEnumerable<Series> availableSeries);
/// <summary>
/// Attempts to match a raw release title to a series using LLM intelligence.
/// </summary>
/// <param name="releaseTitle">The raw release title string.</param>
/// <param name="availableSeries">List of series available in the user's library.</param>
/// <returns>The matching result containing the series and confidence score, or null if no match found.</returns>
Task<LlmMatchResult> TryMatchSeriesAsync(string releaseTitle, IEnumerable<Series> availableSeries);
}
/// <summary>
/// Result of an LLM-based series matching attempt.
/// </summary>
public class LlmMatchResult
{
/// <summary>
/// The matched series, or null if no confident match was found.
/// </summary>
public Series Series { get; set; }
/// <summary>
/// Confidence score from 0.0 to 1.0 indicating how confident the LLM is in the match.
/// </summary>
public double Confidence { get; set; }
/// <summary>
/// The reasoning provided by the LLM for the match decision.
/// </summary>
public string Reasoning { get; set; }
/// <summary>
/// Indicates whether the match should be considered successful based on confidence threshold.
/// </summary>
public bool IsSuccessfulMatch => Series != null && Confidence >= 0.7;
/// <summary>
/// Alternative matches with lower confidence scores for potential manual selection.
/// </summary>
public List<AlternativeMatch> Alternatives { get; set; } = [];
}
/// <summary>
/// Represents an alternative match suggestion from the LLM.
/// </summary>
public class AlternativeMatch
{
public Series Series { get; set; }
public double Confidence { get; set; }
public string Reasoning { get; set; }
}
}

353
src/NzbDrone.Core/Parser/LlmMatching/OpenAiSeriesMatchingService.cs Normal file
View file

@ -0,0 +1,353 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Parser.LlmMatching
{
/// <summary>
/// OpenAI-based implementation of ILlmSeriesMatchingService.
/// Uses GPT models to intelligently match release titles to series when traditional parsing fails.
/// </summary>
public class OpenAiSeriesMatchingService : ILlmSeriesMatchingService
{
private const string DefaultApiEndpoint = "https://api.openai.com/v1/chat/completions";
private const string DefaultModel = "gpt-4o-mini";
private const int MaxSeriesInPrompt = 100;
private readonly IConfigService _configService;
private readonly IHttpClientFactory _httpClientFactory;
private readonly Logger _logger;
public OpenAiSeriesMatchingService(
IConfigService configService,
IHttpClientFactory httpClientFactory,
Logger logger)
{
_configService = configService;
_httpClientFactory = httpClientFactory;
_logger = logger;
}
public virtual bool IsEnabled => _configService.LlmMatchingEnabled &&
_configService.OpenAiApiKey.IsNotNullOrWhiteSpace();
public virtual async Task<LlmMatchResult> TryMatchSeriesAsync(
ParsedEpisodeInfo parsedEpisodeInfo,
IEnumerable<Series> availableSeries)
{
if (!IsEnabled)
{
_logger.Trace("LLM matching is disabled or not configured");
return null;
}
var releaseTitle = parsedEpisodeInfo?.ReleaseTitle ?? parsedEpisodeInfo?.SeriesTitle;
if (releaseTitle.IsNullOrWhiteSpace())
{
_logger.Debug("No release title available for LLM matching");
return null;
}
return await TryMatchSeriesInternalAsync(releaseTitle, parsedEpisodeInfo, availableSeries);
}
public virtual async Task<LlmMatchResult> TryMatchSeriesAsync(
string releaseTitle,
IEnumerable<Series> availableSeries)
{
if (!IsEnabled)
{
_logger.Trace("LLM matching is disabled or not configured");
return null;
}
if (releaseTitle.IsNullOrWhiteSpace())
{
_logger.Debug("No release title provided for LLM matching");
return null;
}
return await TryMatchSeriesInternalAsync(releaseTitle, null, availableSeries);
}
private async Task<LlmMatchResult> TryMatchSeriesInternalAsync(
string releaseTitle,
ParsedEpisodeInfo parsedEpisodeInfo,
IEnumerable<Series> availableSeries)
{
var seriesList = availableSeries?.ToList() ?? new List<Series>();
if (!seriesList.Any())
{
_logger.Debug("No series available for LLM matching");
return null;
}
try
{
var prompt = BuildMatchingPrompt(releaseTitle, parsedEpisodeInfo, seriesList);
var response = await CallOpenAiAsync(prompt);
var result = ParseLlmResponse(response, seriesList);
if (result?.IsSuccessfulMatch == true)
{
_logger.Info(
"LLM matched '{0}' to series '{1}' (TvdbId: {2}) with {3:P0} confidence. Reasoning: {4}",
releaseTitle,
result.Series.Title,
result.Series.TvdbId,
result.Confidence,
result.Reasoning);
}
else if (result != null)
{
_logger.Debug(
"LLM could not confidently match '{0}'. Best guess: {1} ({2:P0}). Reasoning: {3}",
releaseTitle,
result.Series?.Title ?? "None",
result.Confidence,
result.Reasoning);
}
return result;
}
catch (Exception ex)
{
_logger.Error(ex, "Error during LLM series matching for '{0}'", releaseTitle);
return null;
}
}
private string BuildMatchingPrompt(
string releaseTitle,
ParsedEpisodeInfo parsedEpisodeInfo,
List<Series> seriesList)
{
var seriesInfo = seriesList
.Take(MaxSeriesInPrompt)
.Select((s, i) => new
{
Index = i + 1,
s.TvdbId,
s.Title,
s.CleanTitle,
s.Year,
s.Network,
s.SeriesType
})
.ToList();
var additionalContext = string.Empty;
if (parsedEpisodeInfo != null)
{
additionalContext = $@"
Additional parsed information:
- Parsed Series Title: {parsedEpisodeInfo.SeriesTitle}
- Season Number: {parsedEpisodeInfo.SeasonNumber}
- Episode Numbers: {string.Join(", ", parsedEpisodeInfo.EpisodeNumbers ?? Array.Empty<int>())}
- Is Anime: {parsedEpisodeInfo.IsAbsoluteNumbering}
- Is Daily Show: {parsedEpisodeInfo.IsDaily}
- Year (if detected): {parsedEpisodeInfo.SeriesTitleInfo?.Year}";
}
return $@"You are a TV series matching assistant for a media management system. Your task is to match a release title to the correct series from a user's library.
RELEASE TITLE TO MATCH:
""{releaseTitle}""
{additionalContext}
AVAILABLE SERIES IN LIBRARY (JSON format):
{JsonSerializer.Serialize(seriesInfo, new JsonSerializerOptions { WriteIndented = true })}
INSTRUCTIONS:
1. Analyze the release title to identify the series name, handling common scene naming conventions:
- Dots/underscores replacing spaces (e.g., ""Breaking.Bad"" = ""Breaking Bad"")
- Year suffixes for disambiguation (e.g., ""Doctor.Who.2005"")
- Alternate/localized titles (e.g., Japanese/Korean titles for anime)
- Common abbreviations and scene group tags
2. Match the title against the available series, considering:
- Exact title matches
- Partial matches with high similarity
- Year matching for disambiguation
- Series type (Anime vs Standard) matching patterns
3. Return your response in the following JSON format ONLY (no additional text):
{{
""matchedTvdbId"": <number or null if no confident match>,
""confidence"": <number from 0.0 to 1.0>,
""reasoning"": ""<brief explanation of your matching logic>"",
""alternatives"": [
{{
""tvdbId"": <number>,
""confidence"": <number>,
""reasoning"": ""<why this could also be a match>""
}}
]
}}
CONFIDENCE GUIDELINES:
- 0.9-1.0: Exact or near-exact match
- 0.7-0.9: High confidence match with minor differences
- 0.5-0.7: Moderate confidence, may need verification
- 0.0-0.5: Low confidence, likely wrong
If you cannot find a confident match (< 0.5), set matchedTvdbId to null.";
}
private async Task<string> CallOpenAiAsync(string prompt)
{
var apiKey = _configService.OpenAiApiKey;
var endpoint = _configService.OpenAiApiEndpoint.IsNotNullOrWhiteSpace()
? _configService.OpenAiApiEndpoint
: DefaultApiEndpoint;
var model = _configService.OpenAiModel.IsNotNullOrWhiteSpace()
? _configService.OpenAiModel
: DefaultModel;
var client = _httpClientFactory.CreateClient("OpenAI");
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {apiKey}");
var requestBody = new
{
model,
messages = new[]
{
new { role = "system", content = "You are a precise TV series matching assistant. Always respond with valid JSON only." },
new { role = "user", content = prompt }
},
temperature = 0.1,
max_tokens = 500,
response_format = new { type = "json_object" }
};
var content = new StringContent(
JsonSerializer.Serialize(requestBody),
Encoding.UTF8,
"application/json");
_logger.Trace("Calling OpenAI API for series matching");
var response = await client.PostAsync(endpoint, content);
response.EnsureSuccessStatusCode();
var responseJson = await response.Content.ReadAsStringAsync();
var responseObj = JsonSerializer.Deserialize<OpenAiResponse>(responseJson);
return responseObj?.Choices?.FirstOrDefault()?.Message?.Content;
}
private LlmMatchResult ParseLlmResponse(string responseJson, List<Series> seriesList)
{
if (responseJson.IsNullOrWhiteSpace())
{
return null;
}
try
{
var response = JsonSerializer.Deserialize<LlmMatchResponse>(
responseJson,
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
if (response == null)
{
return null;
}
var result = new LlmMatchResult
{
Confidence = response.Confidence,
Reasoning = response.Reasoning
};
if (response.MatchedTvdbId.HasValue)
{
result.Series = seriesList.FirstOrDefault(s => s.TvdbId == response.MatchedTvdbId.Value);
}
if (response.Alternatives != null)
{
foreach (var alt in response.Alternatives)
{
var altSeries = seriesList.FirstOrDefault(s => s.TvdbId == alt.TvdbId);
if (altSeries != null)
{
result.Alternatives.Add(new AlternativeMatch
{
Series = altSeries,
Confidence = alt.Confidence,
Reasoning = alt.Reasoning
});
}
}
}
return result;
}
catch (JsonException ex)
{
_logger.Warn(ex, "Failed to parse LLM response JSON: {0}", responseJson);
return null;
}
}
private class OpenAiResponse
{
[JsonPropertyName("choices")]
public List<Choice> Choices { get; set; }
}
private class Choice
{
[JsonPropertyName("message")]
public Message Message { get; set; }
}
private class Message
{
[JsonPropertyName("content")]
public string Content { get; set; }
}
private class LlmMatchResponse
{
[JsonPropertyName("matchedTvdbId")]
public int? MatchedTvdbId { get; set; }
[JsonPropertyName("confidence")]
public double Confidence { get; set; }
[JsonPropertyName("reasoning")]
public string Reasoning { get; set; }
[JsonPropertyName("alternatives")]
public List<AlternativeMatchResponse> Alternatives { get; set; }
}
private class AlternativeMatchResponse
{
[JsonPropertyName("tvdbId")]
public int TvdbId { get; set; }
[JsonPropertyName("confidence")]
public double Confidence { get; set; }
[JsonPropertyName("reasoning")]
public string Reasoning { get; set; }
}
}
}

100
src/NzbDrone.Core/Parser/LlmMatching/RateLimitedLlmSeriesMatchingService.cs Normal file
View file

@ -0,0 +1,100 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Parser.LlmMatching
{
/// <summary>
/// Decorator service that adds rate limiting to the LLM matching service.
/// Prevents excessive API costs by limiting the number of calls per hour.
/// </summary>
public class RateLimitedLlmSeriesMatchingService(
CachedLlmSeriesMatchingService innerService,
IConfigService configService,
Logger logger) : ILlmSeriesMatchingService
{
private readonly ILlmSeriesMatchingService _innerService = innerService;
private readonly Queue<DateTime> _callTimestamps = new();
private readonly SemaphoreSlim _semaphore = new(1, 1);
public virtual bool IsEnabled => _innerService.IsEnabled;
public virtual async Task<LlmMatchResult> TryMatchSeriesAsync(
ParsedEpisodeInfo parsedEpisodeInfo,
IEnumerable<Series> availableSeries)
{
if (!await TryAcquireRateLimitSlotAsync())
{
logger.Warn(
"LLM rate limit exceeded. Skipping LLM matching for '{0}'",
parsedEpisodeInfo?.SeriesTitle ?? "unknown");
return null;
}
return await _innerService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries);
}
public virtual async Task<LlmMatchResult> TryMatchSeriesAsync(
string releaseTitle,
IEnumerable<Series> availableSeries)
{
if (!await TryAcquireRateLimitSlotAsync())
{
logger.Warn("LLM rate limit exceeded. Skipping LLM matching for '{0}'", releaseTitle);
return null;
}
return await _innerService.TryMatchSeriesAsync(releaseTitle, availableSeries);
}
private async Task<bool> TryAcquireRateLimitSlotAsync()
{
await _semaphore.WaitAsync();
try
{
var maxCallsPerHour = configService.LlmMaxCallsPerHour;
var now = DateTime.UtcNow;
var windowStart = now.AddHours(-1);
while (_callTimestamps.Count > 0 && _callTimestamps.Peek() < windowStart)
{
_callTimestamps.Dequeue();
}
if (_callTimestamps.Count >= maxCallsPerHour)
{
var oldestCall = _callTimestamps.Peek();
var timeUntilSlotAvailable = oldestCall.AddHours(1) - now;
logger.Debug(
"LLM rate limit reached ({0}/{1} calls in last hour). Next slot available in {2}",
_callTimestamps.Count,
maxCallsPerHour,
timeUntilSlotAvailable);
return false;
}
_callTimestamps.Enqueue(now);
logger.Trace(
"LLM rate limit: {0}/{1} calls in last hour",
_callTimestamps.Count,
maxCallsPerHour);
return true;
}
finally
{
_semaphore.Release();
}
}
}
}

5
src/NzbDrone.Core/Parser/Model/FindSeriesResult.cs
View file

@ -1,4 +1,4 @@
using NzbDrone.Core.Tv;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.Parser.Model
{
@ -19,6 +19,7 @@ public enum SeriesMatchType
Unknown = 0,
Title = 1,
Alias = 2,
Id = 3
Id = 3,
Llm = 4
}
}

298
src/NzbDrone.Core/Parser/ParsingService.cs
View file

@ -6,6 +6,7 @@
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.DataAugmentation.Scene;
using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.Parser.LlmMatching;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Tv;
@ -22,41 +23,30 @@ public interface IParsingService
ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisodeInfo, string releaseTitle, Series series);
}
public class ParsingService : IParsingService
public class ParsingService(
IEpisodeService episodeService,
ISeriesService seriesService,
ISceneMappingService sceneMappingService,
ILlmSeriesMatchingService llmMatchingService,
Logger logger) : IParsingService
{
private readonly IEpisodeService _episodeService;
private readonly ISeriesService _seriesService;
private readonly ISceneMappingService _sceneMappingService;
private readonly Logger _logger;
public ParsingService(IEpisodeService episodeService,
ISeriesService seriesService,
ISceneMappingService sceneMappingService,
Logger logger)
{
_episodeService = episodeService;
_seriesService = seriesService;
_sceneMappingService = sceneMappingService;
_logger = logger;
}
public Series GetSeries(string title)
{
var parsedEpisodeInfo = Parser.ParseTitle(title);
if (parsedEpisodeInfo == null)
{
return _seriesService.FindByTitle(title);
return seriesService.FindByTitle(title);
}
var tvdbId = _sceneMappingService.FindTvdbId(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber);
var tvdbId = sceneMappingService.FindTvdbId(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber);
if (tvdbId.HasValue)
{
return _seriesService.FindByTvdbId(tvdbId.Value);
return seriesService.FindByTvdbId(tvdbId.Value);
}
var series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle);
var series = seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle);
if (series == null && parsedEpisodeInfo.SeriesTitleInfo.AllTitles != null)
{
@ -65,38 +55,104 @@ public Series GetSeries(string title)
if (series == null)
{
series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear,
parsedEpisodeInfo.SeriesTitleInfo.Year);
series = seriesService.FindByTitle(
parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear,
parsedEpisodeInfo.SeriesTitleInfo.Year);
}
// LLM FALLBACK - Try LLM matching if traditional methods failed
if (series == null && llmMatchingService.IsEnabled)
{
series = TryLlmMatching(parsedEpisodeInfo, title);
}
return series;
}
private Series TryLlmMatching(ParsedEpisodeInfo parsedEpisodeInfo, string originalTitle)
{
if (!llmMatchingService.IsEnabled)
{
return null;
}
try
{
logger.Debug(
"Traditional matching failed for '{0}', attempting LLM matching",
parsedEpisodeInfo?.SeriesTitle ?? originalTitle);
var availableSeries = seriesService.GetAllSeries();
if (!availableSeries.Any())
{
logger.Debug("No series in library for LLM matching");
return null;
}
var matchTask = parsedEpisodeInfo != null
? llmMatchingService.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries)
: llmMatchingService.TryMatchSeriesAsync(originalTitle, availableSeries);
var result = matchTask.GetAwaiter().GetResult();
if (result?.IsSuccessfulMatch == true)
{
logger.Info(
"LLM matched '{0}' to '{1}' (TVDB: {2}) with {3:P0} confidence",
parsedEpisodeInfo?.SeriesTitle ?? originalTitle,
result.Series.Title,
result.Series.TvdbId,
result.Confidence);
return result.Series;
}
if (result?.Alternatives?.Any() == true)
{
logger.Debug(
"LLM found potential matches for '{0}' but confidence too low. Alternatives: {1}",
parsedEpisodeInfo?.SeriesTitle ?? originalTitle,
string.Join(", ", result.Alternatives.Select(a => $"{a.Series.Title} ({a.Confidence:P0})")));
}
return null;
}
catch (Exception ex)
{
logger.Error(
ex,
"Error during LLM series matching for '{0}'",
parsedEpisodeInfo?.SeriesTitle ?? originalTitle);
return null;
}
}
private Series GetSeriesByAllTitles(ParsedEpisodeInfo parsedEpisodeInfo)
{
Series foundSeries = null;
int? foundTvdbId = null;
// Match each title individually, they must all resolve to the same tvdbid
foreach (var title in parsedEpisodeInfo.SeriesTitleInfo.AllTitles)
{
var series = _seriesService.FindByTitle(title);
var series = seriesService.FindByTitle(title);
var tvdbId = series?.TvdbId;
if (series == null)
{
tvdbId = _sceneMappingService.FindTvdbId(title, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber);
tvdbId = sceneMappingService.FindTvdbId(title, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber);
}
if (!tvdbId.HasValue)
{
_logger.Trace("Title {0} not matching any series.", title);
logger.Trace("Title {0} not matching any series.", title);
continue;
}
if (foundTvdbId.HasValue && tvdbId != foundTvdbId)
{
_logger.Trace("Title {0} both matches tvdbid {1} and {2}, no series selected.", parsedEpisodeInfo.SeriesTitle, foundTvdbId, tvdbId);
logger.Trace("Title {0} both matches tvdbid {1} and {2}, no series selected.", parsedEpisodeInfo.SeriesTitle, foundTvdbId, tvdbId);
return null;
}
@ -110,7 +166,7 @@ private Series GetSeriesByAllTitles(ParsedEpisodeInfo parsedEpisodeInfo)
if (foundSeries == null && foundTvdbId.HasValue)
{
foundSeries = _seriesService.FindByTvdbId(foundTvdbId.Value);
foundSeries = seriesService.FindByTvdbId(foundTvdbId.Value);
}
return foundSeries;
@ -129,16 +185,16 @@ public RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, Series series)
public RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int seriesId, IEnumerable<int> episodeIds)
{
return new RemoteEpisode
{
ParsedEpisodeInfo = parsedEpisodeInfo,
Series = _seriesService.GetSeries(seriesId),
Episodes = _episodeService.GetEpisodes(episodeIds)
};
{
ParsedEpisodeInfo = parsedEpisodeInfo,
Series = seriesService.GetSeries(seriesId),
Episodes = episodeService.GetEpisodes(episodeIds)
};
}
private RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int tvdbId, int tvRageId, string imdbId, Series series, SearchCriteriaBase searchCriteria)
{
var sceneMapping = _sceneMappingService.FindSceneMapping(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber);
var sceneMapping = sceneMappingService.FindSceneMapping(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle, parsedEpisodeInfo.SeasonNumber);
var remoteEpisode = new RemoteEpisode
{
@ -147,8 +203,8 @@ private RemoteEpisode Map(ParsedEpisodeInfo parsedEpisodeInfo, int tvdbId, int t
MappedSeasonNumber = parsedEpisodeInfo.SeasonNumber
};
// For now we just detect tvdb vs scene, but we can do multiple 'origins' in the future.
var sceneSource = true;
if (sceneMapping != null)
{
if (sceneMapping.SeasonNumber.HasValue && sceneMapping.SeasonNumber.Value >= 0 &&
@ -225,16 +281,15 @@ private List<Episode> GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series se
{
if (series.UseSceneNumbering && sceneSource)
{
var episodes = _episodeService.GetEpisodesBySceneSeason(series.Id, mappedSeasonNumber);
var episodes = episodeService.GetEpisodesBySceneSeason(series.Id, mappedSeasonNumber);
// If episodes were found by the scene season number return them, otherwise fallback to look-up by season number
if (episodes.Any())
{
return episodes;
}
}
return _episodeService.GetEpisodesBySeason(series.Id, mappedSeasonNumber);
return episodeService.GetEpisodesBySeason(series.Id, mappedSeasonNumber);
}
if (parsedEpisodeInfo.IsDaily)
@ -243,10 +298,10 @@ private List<Episode> GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series se
if (episodeInfo != null)
{
return new List<Episode> { episodeInfo };
return [episodeInfo];
}
return new List<Episode>();
return [];
}
if (parsedEpisodeInfo.IsAbsoluteNumbering)
@ -260,14 +315,13 @@ private List<Episode> GetEpisodes(ParsedEpisodeInfo parsedEpisodeInfo, Series se
if (parsedSpecialEpisodeInfo != null)
{
// Use the season number and disable scene source since the season/episode numbers that were returned are not scene numbers
return GetStandardEpisodes(series, parsedSpecialEpisodeInfo, parsedSpecialEpisodeInfo.SeasonNumber, false, searchCriteria);
}
}
if (parsedEpisodeInfo.Special && mappedSeasonNumber != 0)
{
return new List<Episode>();
return [];
}
return GetStandardEpisodes(series, parsedEpisodeInfo, mappedSeasonNumber, sceneSource, searchCriteria);
@ -295,29 +349,26 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod
var series = GetSeries(releaseTitle);
if (series == null)
{
series = _seriesService.FindByTitleInexact(releaseTitle);
}
series ??= seriesService.FindByTitleInexact(releaseTitle);
if (series == null && tvdbId > 0)
{
series = _seriesService.FindByTvdbId(tvdbId);
series = seriesService.FindByTvdbId(tvdbId);
}
if (series == null && tvRageId > 0)
{
series = _seriesService.FindByTvRageId(tvRageId);
series = seriesService.FindByTvRageId(tvRageId);
}
if (series == null && imdbId.IsNotNullOrWhiteSpace())
{
series = _seriesService.FindByImdbId(imdbId);
series = seriesService.FindByImdbId(imdbId);
}
if (series == null)
{
_logger.Debug("No matching series {0}", releaseTitle);
logger.Debug("No matching series {0}", releaseTitle);
return null;
}
@ -329,14 +380,14 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod
// SxxE00 episodes are sometimes mapped via TheXEM, don't use episode title parsing in that case.
if (parsedEpisodeInfo != null && parsedEpisodeInfo.IsPossibleSceneSeasonSpecial && series.UseSceneNumbering)
{
if (_episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, 0).Any())
if (episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, 0).Any())
{
return parsedEpisodeInfo;
}
}
// find special episode in series season 0
var episode = _episodeService.FindEpisodeByTitle(series.Id, 0, releaseTitle);
var episode = episodeService.FindEpisodeByTitle(series.Id, 0, releaseTitle);
if (episode != null)
{
@ -346,11 +397,11 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod
ReleaseTitle = releaseTitle,
SeriesTitle = series.Title,
SeriesTitleInfo = new SeriesTitleInfo
{
Title = series.Title
},
{
Title = series.Title
},
SeasonNumber = episode.SeasonNumber,
EpisodeNumbers = new int[1] { episode.EpisodeNumber },
EpisodeNumbers = [episode.EpisodeNumber],
FullSeason = false,
Quality = QualityParser.ParseQuality(releaseTitle),
ReleaseGroup = ReleaseGroupParser.ParseReleaseGroup(releaseTitle),
@ -358,7 +409,7 @@ public ParsedEpisodeInfo ParseSpecialEpisodeTitle(ParsedEpisodeInfo parsedEpisod
Special = true
};
_logger.Debug("Found special episode {0} for title '{1}'", info, releaseTitle);
logger.Debug("Found special episode {0} for title '{1}'", info, releaseTitle);
return info;
}
@ -376,11 +427,11 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
return new FindSeriesResult(searchCriteria.Series, SeriesMatchType.Alias);
}
series = _seriesService.FindByTvdbId(sceneMapping.TvdbId);
series = seriesService.FindByTvdbId(sceneMapping.TvdbId);
if (series == null)
{
_logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle);
logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle);
return null;
}
@ -396,7 +447,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
if (tvdbId > 0 && tvdbId == searchCriteria.Series.TvdbId)
{
_logger.ForDebugEvent()
logger.ForDebugEvent()
.Message("Found matching series by TVDB ID {0}, an alias may be needed for: {1}", tvdbId, parsedEpisodeInfo.SeriesTitle)
.Property("TvdbId", tvdbId)
.Property("ParsedEpisodeInfo", parsedEpisodeInfo)
@ -408,7 +459,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
if (tvRageId > 0 && tvRageId == searchCriteria.Series.TvRageId && tvdbId <= 0)
{
_logger.ForDebugEvent()
logger.ForDebugEvent()
.Message("Found matching series by TVRage ID {0}, an alias may be needed for: {1}", tvRageId, parsedEpisodeInfo.SeriesTitle)
.Property("TvRageId", tvRageId)
.Property("ParsedEpisodeInfo", parsedEpisodeInfo)
@ -420,7 +471,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
if (imdbId.IsNotNullOrWhiteSpace() && imdbId.Equals(searchCriteria.Series.ImdbId, StringComparison.Ordinal) && tvdbId <= 0)
{
_logger.ForDebugEvent()
logger.ForDebugEvent()
.Message("Found matching series by IMDb ID {0}, an alias may be needed for: {1}", imdbId, parsedEpisodeInfo.SeriesTitle)
.Property("ImdbId", imdbId)
.Property("ParsedEpisodeInfo", parsedEpisodeInfo)
@ -432,7 +483,7 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
}
var matchType = SeriesMatchType.Unknown;
series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle);
series = seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitle);
if (series != null)
{
@ -447,17 +498,17 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
if (series == null && parsedEpisodeInfo.SeriesTitleInfo.Year > 0)
{
series = _seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear, parsedEpisodeInfo.SeriesTitleInfo.Year);
series = seriesService.FindByTitle(parsedEpisodeInfo.SeriesTitleInfo.TitleWithoutYear, parsedEpisodeInfo.SeriesTitleInfo.Year);
matchType = SeriesMatchType.Title;
}
if (series == null && tvdbId > 0)
{
series = _seriesService.FindByTvdbId(tvdbId);
series = seriesService.FindByTvdbId(tvdbId);
if (series != null)
{
_logger.ForDebugEvent()
logger.ForDebugEvent()
.Message("Found matching series by TVDB ID {0}, an alias may be needed for: {1}", tvdbId, parsedEpisodeInfo.SeriesTitle)
.Property("TvdbId", tvdbId)
.Property("ParsedEpisodeInfo", parsedEpisodeInfo)
@ -470,11 +521,11 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
if (series == null && tvRageId > 0 && tvdbId <= 0)
{
series = _seriesService.FindByTvRageId(tvRageId);
series = seriesService.FindByTvRageId(tvRageId);
if (series != null)
{
_logger.ForDebugEvent()
logger.ForDebugEvent()
.Message("Found matching series by TVRage ID {0}, an alias may be needed for: {1}", tvRageId, parsedEpisodeInfo.SeriesTitle)
.Property("TvRageId", tvRageId)
.Property("ParsedEpisodeInfo", parsedEpisodeInfo)
@ -487,11 +538,11 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
if (series == null && imdbId.IsNotNullOrWhiteSpace() && tvdbId <= 0)
{
series = _seriesService.FindByImdbId(imdbId);
series = seriesService.FindByImdbId(imdbId);
if (series != null)
{
_logger.ForDebugEvent()
logger.ForDebugEvent()
.Message("Found matching series by IMDb ID {0}, an alias may be needed for: {1}", imdbId, parsedEpisodeInfo.SeriesTitle)
.Property("ImdbId", imdbId)
.Property("ParsedEpisodeInfo", parsedEpisodeInfo)
@ -502,9 +553,54 @@ private FindSeriesResult FindSeries(ParsedEpisodeInfo parsedEpisodeInfo, int tvd
}
}
// LLM FALLBACK MATCHING
if (series == null && llmMatchingService.IsEnabled)
{
logger.Debug(
"All traditional matching methods failed for '{0}', attempting LLM matching",
parsedEpisodeInfo.SeriesTitle);
try
{
var availableSeries = seriesService.GetAllSeries();
var llmResult = llmMatchingService
.TryMatchSeriesAsync(parsedEpisodeInfo, availableSeries)
.GetAwaiter()
.GetResult();
if (llmResult?.IsSuccessfulMatch == true)
{
logger.Info(
"LLM matched '{0}' to '{1}' (TVDB: {2}) with {3:P0} confidence. Reasoning: {4}",
parsedEpisodeInfo.SeriesTitle,
llmResult.Series.Title,
llmResult.Series.TvdbId,
llmResult.Confidence,
llmResult.Reasoning);
return new FindSeriesResult(llmResult.Series, SeriesMatchType.Llm);
}
if (llmResult?.Alternatives?.Any() == true)
{
logger.Debug(
"LLM found potential but uncertain matches for '{0}': {1}",
parsedEpisodeInfo.SeriesTitle,
string.Join(", ", llmResult.Alternatives.Select(a => $"{a.Series.Title} ({a.Confidence:P0})")));
}
}
catch (Exception ex)
{
logger.Warn(
ex,
"LLM matching failed for '{0}', falling back to manual import",
parsedEpisodeInfo.SeriesTitle);
}
}
if (series == null)
{
_logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle);
logger.Debug("No matching series {0}", parsedEpisodeInfo.SeriesTitle);
return null;
}
@ -523,7 +619,7 @@ private Episode GetDailyEpisode(Series series, string airDate, int? part, Search
if (episodeInfo == null)
{
episodeInfo = _episodeService.FindEpisode(series.Id, airDate, part);
episodeInfo = episodeService.FindEpisode(series.Id, airDate, part);
}
return episodeInfo;
@ -533,7 +629,7 @@ private List<Episode> GetAnimeEpisodes(Series series, ParsedEpisodeInfo parsedEp
{
var result = new List<Episode>();
var sceneSeasonNumber = _sceneMappingService.GetSceneSeasonNumber(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle);
var sceneSeasonNumber = sceneMappingService.GetSceneSeasonNumber(parsedEpisodeInfo.SeriesTitle, parsedEpisodeInfo.ReleaseTitle);
foreach (var absoluteEpisodeNumber in parsedEpisodeInfo.AbsoluteEpisodeNumbers)
{
@ -541,39 +637,35 @@ private List<Episode> GetAnimeEpisodes(Series series, ParsedEpisodeInfo parsedEp
if (parsedEpisodeInfo.Special)
{
var episode = _episodeService.FindEpisode(series.Id, 0, absoluteEpisodeNumber);
var episode = episodeService.FindEpisode(series.Id, 0, absoluteEpisodeNumber);
episodes.AddIfNotNull(episode);
}
else if (sceneSource)
{
// Is there a reason why we excluded season 1 from this handling before?
// Might have something to do with the scene name to season number check
// If this needs to be reverted tests will need to be added
if (sceneSeasonNumber.HasValue)
{
episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber);
episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber);
if (episodes.Empty())
{
var episode = _episodeService.FindEpisode(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber);
var episode = episodeService.FindEpisode(series.Id, sceneSeasonNumber.Value, absoluteEpisodeNumber);
episodes.AddIfNotNull(episode);
}
}
else if (parsedEpisodeInfo.SeasonNumber > 1 && parsedEpisodeInfo.EpisodeNumbers.Empty())
{
episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber);
episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber);
if (episodes.Empty())
{
var episode = _episodeService.FindEpisode(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber);
var episode = episodeService.FindEpisode(series.Id, parsedEpisodeInfo.SeasonNumber, absoluteEpisodeNumber);
episodes.AddIfNotNull(episode);
}
}
else
{
episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, absoluteEpisodeNumber);
episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, absoluteEpisodeNumber);
// Don't allow multiple results without a scene name mapping.
if (episodes.Count > 1)
{
episodes.Clear();
@ -583,17 +675,18 @@ private List<Episode> GetAnimeEpisodes(Series series, ParsedEpisodeInfo parsedEp
if (episodes.Empty())
{
var episode = _episodeService.FindEpisode(series.Id, absoluteEpisodeNumber);
var episode = episodeService.FindEpisode(series.Id, absoluteEpisodeNumber);
episodes.AddIfNotNull(episode);
}
foreach (var episode in episodes)
{
_logger.Debug("Using absolute episode number {0} for: {1} - TVDB: {2}x{3:00}",
absoluteEpisodeNumber,
series.Title,
episode.SeasonNumber,
episode.EpisodeNumber);
logger.Debug(
"Using absolute episode number {0} for: {1} - TVDB: {2}x{3:00}",
absoluteEpisodeNumber,
series.Title,
episode.SeasonNumber,
episode.EpisodeNumber);
result.Add(episode);
}
@ -608,7 +701,7 @@ private List<Episode> GetStandardEpisodes(Series series, ParsedEpisodeInfo parse
if (parsedEpisodeInfo.EpisodeNumbers == null)
{
return new List<Episode>();
return [];
}
foreach (var episodeNumber in parsedEpisodeInfo.EpisodeNumbers)
@ -625,16 +718,17 @@ private List<Episode> GetStandardEpisodes(Series series, ParsedEpisodeInfo parse
if (!episodes.Any())
{
episodes = _episodeService.FindEpisodesBySceneNumbering(series.Id, mappedSeasonNumber, episodeNumber);
episodes = episodeService.FindEpisodesBySceneNumbering(series.Id, mappedSeasonNumber, episodeNumber);
}
if (episodes != null && episodes.Any())
{
_logger.Debug("Using Scene to TVDB Mapping for: {0} - Scene: {1}x{2:00} - TVDB: {3}",
series.Title,
episodes.First().SceneSeasonNumber,
episodes.First().SceneEpisodeNumber,
string.Join(", ", episodes.Select(e => string.Format("{0}x{1:00}", e.SeasonNumber, e.EpisodeNumber))));
logger.Debug(
"Using Scene to TVDB Mapping for: {0} - Scene: {1}x{2:00} - TVDB: {3}",
series.Title,
episodes.First().SceneSeasonNumber,
episodes.First().SceneEpisodeNumber,
string.Join(", ", episodes.Select(e => string.Format("{0}x{1:00}", e.SeasonNumber, e.EpisodeNumber))));
result.AddRange(episodes);
continue;
@ -650,7 +744,7 @@ private List<Episode> GetStandardEpisodes(Series series, ParsedEpisodeInfo parse
if (episodeInfo == null)
{
episodeInfo = _episodeService.FindEpisode(series.Id, mappedSeasonNumber, episodeNumber);
episodeInfo = episodeService.FindEpisode(series.Id, mappedSeasonNumber, episodeNumber);
}
if (episodeInfo != null)
@ -659,11 +753,17 @@ private List<Episode> GetStandardEpisodes(Series series, ParsedEpisodeInfo parse
}
else
{
_logger.Debug("Unable to find {0}", parsedEpisodeInfo);
logger.Debug("Unable to find {0}", parsedEpisodeInfo);
}
}
return result;
}
}
public class FindSeriesResult(Series series, SeriesMatchType matchType)
{
public Series Series { get; } = series;
public SeriesMatchType MatchType { get; } = matchType;
}
}

7
src/NzbDrone.Host/Startup.cs
View file

@ -25,6 +25,7 @@
using NzbDrone.Core.Instrumentation;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser.LlmMatching;
using NzbDrone.Host.AccessControl;
using NzbDrone.Http.Authentication;
using NzbDrone.SignalR;
@ -223,6 +224,12 @@ public void ConfigureServices(IServiceCollection services)
services.AddSingleton<IAuthorizationPolicyProvider, UiAuthorizationPolicyProvider>();
services.AddSingleton<IAuthorizationHandler, UiAuthorizationHandler>();
services.AddSingleton<OpenAiSeriesMatchingService>();
services.AddSingleton<CachedLlmSeriesMatchingService>();
services.AddSingleton<RateLimitedLlmSeriesMatchingService>();
services.AddSingleton<ILlmSeriesMatchingService>(sp =>
sp.GetRequiredService<RateLimitedLlmSeriesMatchingService>());
services.AddAuthorization(options =>
{
options.AddPolicy("SignalR", policy =>

201
src/Sonarr.Api.V3/Config/LlmMatchingConfigController.cs Normal file
View file

@ -0,0 +1,201 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser.LlmMatching;
using NzbDrone.Core.Tv;
using Sonarr.Http;
using Sonarr.Http.REST;
namespace Sonarr.Api.V3.Config
{
/// <summary>
/// API Controller for LLM matching configuration.
/// Provides endpoints for managing LLM settings and testing the configuration.
/// </summary>
[V3ApiController("config/llmmatching")]
public class LlmMatchingConfigController : RestController<LlmMatchingConfigResource>
{
private readonly IConfigService _configService;
private readonly ILlmSeriesMatchingService _llmMatchingService;
private readonly ISeriesService _seriesService;
public LlmMatchingConfigController(
IConfigService configService,
ILlmSeriesMatchingService llmMatchingService,
ISeriesService seriesService)
{
_configService = configService;
_llmMatchingService = llmMatchingService;
_seriesService = seriesService;
}
protected override LlmMatchingConfigResource GetResourceById(int id)
{
return GetLlmMatchingConfig();
}
[HttpGet]
public LlmMatchingConfigResource GetLlmMatchingConfig()
{
return new LlmMatchingConfigResource
{
Id = 1,
LlmMatchingEnabled = _configService.LlmMatchingEnabled,
OpenAiApiKeySet = !string.IsNullOrWhiteSpace(_configService.OpenAiApiKey),
OpenAiApiEndpoint = _configService.OpenAiApiEndpoint,
OpenAiModel = _configService.OpenAiModel,
LlmConfidenceThreshold = _configService.LlmConfidenceThreshold,
LlmMaxCallsPerHour = _configService.LlmMaxCallsPerHour,
LlmCacheEnabled = _configService.LlmCacheEnabled,
LlmCacheDurationHours = _configService.LlmCacheDurationHours,
IsServiceAvailable = _llmMatchingService.IsEnabled
};
}
[HttpPut]
public ActionResult<LlmMatchingConfigResource> SaveLlmMatchingConfig([FromBody] LlmMatchingConfigResource resource)
{
// Note: Implementation depends on how ConfigService handles updates
// This would need to be implemented based on Sonarr's actual config saving mechanism
return Accepted(GetLlmMatchingConfig());
}
/// <summary>
/// Tests the LLM matching configuration by attempting to match a sample title.
/// </summary>
[HttpPost("test")]
public async Task<ActionResult<LlmTestResult>> TestLlmMatching([FromBody] LlmTestRequest request)
{
if (!_llmMatchingService.IsEnabled)
{
return BadRequest(new LlmTestResult
{
Success = false,
ErrorMessage = "LLM matching is not enabled or configured."
});
}
if (string.IsNullOrWhiteSpace(request.TestTitle))
{
return BadRequest(new LlmTestResult
{
Success = false,
ErrorMessage = "Test title is required."
});
}
try
{
var availableSeries = _seriesService.GetAllSeries();
var result = await _llmMatchingService.TryMatchSeriesAsync(request.TestTitle, availableSeries);
var testResult = new LlmTestResult
{
Success = true,
MatchFound = result?.IsSuccessfulMatch == true,
MatchedSeriesTitle = result?.Series?.Title,
MatchedSeriesTvdbId = result?.Series?.TvdbId,
Confidence = result?.Confidence ?? 0,
Reasoning = result?.Reasoning,
Alternatives = new List<LlmAlternativeResult>()
};
if (result?.Alternatives != null)
{
testResult.Alternatives = result.Alternatives
.Select(a => new LlmAlternativeResult
{
SeriesTitle = a.Series?.Title,
TvdbId = a.Series?.TvdbId ?? 0,
Confidence = a.Confidence,
Reasoning = a.Reasoning
})
.ToList();
}
return Ok(testResult);
}
catch (Exception ex)
{
return Ok(new LlmTestResult
{
Success = false,
ErrorMessage = $"Error testing LLM matching: {ex.Message}"
});
}
}
}
/// <summary>
/// Resource model for LLM matching configuration.
/// </summary>
public class LlmMatchingConfigResource : RestResource
{
public bool LlmMatchingEnabled { get; set; }
public bool OpenAiApiKeySet { get; set; }
public string OpenAiApiKey { get; set; }
public string OpenAiApiEndpoint { get; set; }
public string OpenAiModel { get; set; }
public double LlmConfidenceThreshold { get; set; }
public int LlmMaxCallsPerHour { get; set; }
public bool LlmCacheEnabled { get; set; }
public int LlmCacheDurationHours { get; set; }
public bool IsServiceAvailable { get; set; }
}
/// <summary>
/// Request model for testing LLM matching.
/// </summary>
public class LlmTestRequest
{
public string TestTitle { get; set; }
}
/// <summary>
/// Result model for LLM matching test.
/// </summary>
public class LlmTestResult
{
public bool Success { get; set; }
public string ErrorMessage { get; set; }
public bool MatchFound { get; set; }
public string MatchedSeriesTitle { get; set; }
public int? MatchedSeriesTvdbId { get; set; }
public double Confidence { get; set; }
public string Reasoning { get; set; }
public List<LlmAlternativeResult> Alternatives { get; set; }
}
/// <summary>
/// Alternative match result for LLM matching test.
/// </summary>
public class LlmAlternativeResult
{
public string SeriesTitle { get; set; }
public int TvdbId { get; set; }
public double Confidence { get; set; }
public string Reasoning { get; set; }
}
}

8
src/Sonarr.sln
View file

@ -1,10 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29806.167
MinimumVisualStudioVersion = 10.0.40219.1
# Visual Studio Version 17
VisualStudioVersion = 17.2.32516.85
# Visual Studio Version 18
VisualStudioVersion = 18.1.11312.151
MinimumVisualStudioVersion = 15.0.26124.0
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sonarr.Console", "NzbDrone.Console\Sonarr.Console.csproj", "{3DCA7B58-B8B3-49AC-9D9E-56F4A0460976}"
EndProject
@ -69,6 +66,7 @@ EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0C7E5F5A-C4CC-4945-B399-1E1C9817CC45}"
ProjectSection(SolutionItems) = preProject
..\.editorconfig = ..\.editorconfig
appsettings.llm.template.json = appsettings.llm.template.json
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sonarr.Mono", "NzbDrone.Mono\Sonarr.Mono.csproj", "{22F71728-AB73-4774-8DC2-6D2D7734AE0C}"