From 690de685c7e0a40cad1e75a2d157bd4a9c0844f9 Mon Sep 17 00:00:00 2001 From: Kyle Butler Date: Wed, 19 Feb 2025 13:12:04 -0500 Subject: [PATCH] New: Limit filenames to a maximum of 255 characters --- .../TruncatedBookTitlesFixture.cs | 214 ++++++++++++++++++ src/NzbDrone.Core/Fluent.cs | 5 + .../Organizer/FileNameBuilder.cs | 132 +++++++++-- 3 files changed, 326 insertions(+), 25 deletions(-) create mode 100644 src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/TruncatedBookTitlesFixture.cs diff --git a/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/TruncatedBookTitlesFixture.cs b/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/TruncatedBookTitlesFixture.cs new file mode 100644 index 000000000..8b2c963d1 --- /dev/null +++ b/src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/TruncatedBookTitlesFixture.cs @@ -0,0 +1,214 @@ +using System.Linq; +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Books; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Organizer; +using NzbDrone.Core.Qualities; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.OrganizerTests.FileNameBuilderTests +{ + [TestFixture] + public class TruncatedBookTitlesFixture : CoreTest + { + private BookFile _bookFile; + private NamingConfig _namingConfig; + + [SetUp] + public void Setup() + { + _namingConfig = NamingConfig.Default; + _namingConfig.RenameBooks = true; + + Mocker.GetMock() + .Setup(c => c.GetConfig()).Returns(_namingConfig); + + _bookFile = new BookFile { Quality = new QualityModel(Quality.EPUB), ReleaseGroup = "ReadarrTest" }; + + Mocker.GetMock() + .Setup(v => v.Get(Moq.It.IsAny())) + .Returns(v => Quality.DefaultQualityDefinitions.First(c => c.Quality == v)); + } + + private (Author, Edition) BuildTestInputs(string authorName, string bookTitle, string seriesName, string seriesNumber) + { + var author = Builder + .CreateNew() + .With(s => s.Name = authorName) + .Build(); + + var series = Builder + .CreateNew() + .With(x => x.Title = seriesName) + .Build(); + + var seriesLink = Builder + .CreateListOfSize(1) + .All() + .With(s => s.Position = seriesNumber) + .With(s => s.Series = series) + .BuildListOfNew(); + + var book = Builder + .CreateNew() + .With(s => s.Title = bookTitle) + .With(s => s.AuthorMetadata = author.Metadata.Value) + .With(s => s.SeriesLinks = seriesLink) + .Build(); + + var edition = Builder + .CreateNew() + .With(s => s.Title = book.Title) + .With(s => s.Book = book) + .Build(); + + return (author, edition); + } + + [Test] + public void should_not_truncate_filename_if_length_is_less_than_max_length_limit_long_book_name() + { + var authorName = "Brandon Sanderson"; + var bookTitle = "Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var seriesName = "The Stormlight Archive"; + var seriesNumber = "5"; + var expected = "Brandon Sanderson - The Stormlight Archive #5 - Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().BeLessThan(255); + } + + [Test] + public void should_not_truncate_filename_if_length_is_less_than_max_length_limit_long_author_name() + { + var authorName = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var bookTitle = "Knights of Wind and Truth"; + var seriesName = "The Stormlight Archive"; + var seriesNumber = "5"; + var expected = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum - The Stormlight Archive #5 - Knights of Wind and Truth"; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().BeLessThan(255); + } + + [Test] + public void should_not_truncate_filename_if_length_is_less_than_max_length_limit_long_series_name() + { + var authorName = "Brandon Sanderson"; + var bookTitle = "Knights of Wind and Truth"; + var seriesName = "The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var seriesNumber = "5"; + var expected = "Brandon Sanderson - The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum #5 - Knights of Wind and Truth"; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().BeLessThan(255); + } + + [Test] + public void should_not_truncate_filename_if_length_is_equal_to_than_max_length_limit_long_book_name() + { + var authorName = "Brandon Sanderson"; + var bookTitle = "Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var seriesName = "The Stormlight Archive"; + var seriesNumber = "5-1"; + var expected = "Brandon Sanderson - The Stormlight Archive #5-1 - Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().Be(255); + } + + [Test] + public void should_not_truncate_filename_if_length_is_equal_to_than_max_length_limit_long_author_name() + { + var authorName = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var bookTitle = "Knights of Wind and Truth"; + var seriesName = "The Stormlight Archive"; + var seriesNumber = "5-1"; + var expected = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum - The Stormlight Archive #5-1 - Knights of Wind and Truth"; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().Be(255); + } + + [Test] + public void should_not_truncate_filename_if_length_is_equal_to_than_max_length_limit_long_series_name() + { + var authorName = "Brandon Sanderson"; + var bookTitle = "Knights of Wind and Truth"; + var seriesName = "The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var seriesNumber = "5-1"; + var expected = "Brandon Sanderson - The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum #5-1 - Knights of Wind and Truth"; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().Be(255); + } + + [Test] + public void should_truncate_filename_if_length_is_greater_than_max_length_limit_long_book_name() + { + var authorName = "Brandon Sanderson and Janci Patterson"; + var bookTitle = "Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var seriesName = "The Stormlight Archive"; + var seriesNumber = "5"; + var expected = "Brandon Sanderson and Janci Patterson - The Stormlight Archive #5 - Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Belo..."; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().Be(255); + } + + [Test] + public void should_truncate_filename_if_length_is_greater_than_than_max_length_limit_long_author_name() + { + var authorName = "Brandon Sanderson and Janci Patterson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var bookTitle = "Knights of Wind and Truth"; + var seriesName = "The Stormlight Archive"; + var seriesNumber = "5"; + var expected = "Brandon Sanderson and Janci Patterson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum - The Stormlight Archive #5 - Knig..."; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().Be(255); + } + + [Test] + public void should_truncate_filename_if_length_is_greater_than_than_max_length_limit_long_series_name() + { + var authorName = "Brandon Sanderson and Janci Patterson"; + var bookTitle = "Knights of Wind and Truth"; + var seriesName = "The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum"; + var seriesNumber = "5"; + var expected = "Brandon Sanderson and Janci Patterson - The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum #5 - Knig..."; + _namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}"; + var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber); + + var result = Subject.BuildBookFileName(author, edition, _bookFile); + result.Should().Be(expected); + result.Length.Should().Be(255); + } + } +} diff --git a/src/NzbDrone.Core/Fluent.cs b/src/NzbDrone.Core/Fluent.cs index 541d843f7..1e04fc227 100644 --- a/src/NzbDrone.Core/Fluent.cs +++ b/src/NzbDrone.Core/Fluent.cs @@ -97,6 +97,11 @@ public static int MaxOrDefault(this IEnumerable ints) return intList.Max(); } + public static int GetByteCount(this string input) + { + return Encoding.UTF8.GetByteCount(input); + } + public static string Truncate(this string s, int maxLength) { if (Encoding.UTF8.GetByteCount(s) <= maxLength) diff --git a/src/NzbDrone.Core/Organizer/FileNameBuilder.cs b/src/NzbDrone.Core/Organizer/FileNameBuilder.cs index 3dfcc44dc..1008e0097 100644 --- a/src/NzbDrone.Core/Organizer/FileNameBuilder.cs +++ b/src/NzbDrone.Core/Organizer/FileNameBuilder.cs @@ -6,6 +6,7 @@ using System.Text.RegularExpressions; using NLog; using NzbDrone.Common.Cache; +using NzbDrone.Common.Disk; using NzbDrone.Common.EnsureThat; using NzbDrone.Common.Extensions; using NzbDrone.Core.Books; @@ -30,7 +31,7 @@ public class FileNameBuilder : IBuildFileNames private readonly INamingConfigService _namingConfigService; private readonly IQualityDefinitionService _qualityDefinitionService; private readonly ICustomFormatCalculationService _formatCalculator; - private readonly ICached _trackFormatCache; + private readonly ICached _bookFormatCache; private readonly Logger _logger; private static readonly Regex TitleRegex = new Regex(@"\{(?[- ._\[(]*)(?(?:[a-z0-9]+)(?:(?[- ._]+)(?:[a-z0-9]+))?)(?::(?[a-z0-9]+))?(?[- ._)\]]*)\}", @@ -56,6 +57,8 @@ public class FileNameBuilder : IBuildFileNames private static readonly Regex TitlePrefixRegex = new Regex(@"^(The|An|A) (.*?)((?: *\([^)]+\))*)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly char[] BookTitleTrimCharacters = new[] { ' ', '.', '?' }; + public FileNameBuilder(INamingConfigService namingConfigService, IQualityDefinitionService qualityDefinitionService, ICacheManager cacheManager, @@ -65,11 +68,11 @@ public FileNameBuilder(INamingConfigService namingConfigService, _namingConfigService = namingConfigService; _qualityDefinitionService = qualityDefinitionService; _formatCalculator = formatCalculator; - _trackFormatCache = cacheManager.GetCache(GetType(), "bookFormat"); + _bookFormatCache = cacheManager.GetCache(GetType(), "bookFormat"); _logger = logger; } - public string BuildBookFileName(Author author, Edition edition, BookFile bookFile, NamingConfig namingConfig = null, List customFormats = null) + private string BuildBookFileName(Author author, Edition edition, BookFile bookFile, int maxPath, NamingConfig namingConfig = null, List customFormats = null) { if (namingConfig == null) { @@ -88,15 +91,6 @@ public string BuildBookFileName(Author author, Edition edition, BookFile bookFil var pattern = namingConfig.StandardBookFormat; - var tokenHandlers = new Dictionary>(FileNameBuilderTokenEqualityComparer.Instance); - - AddAuthorTokens(tokenHandlers, author); - AddBookTokens(tokenHandlers, edition); - AddBookFileTokens(tokenHandlers, bookFile); - AddQualityTokens(tokenHandlers, author, bookFile); - AddMediaInfoTokens(tokenHandlers, bookFile); - AddCustomFormats(tokenHandlers, author, bookFile, customFormats); - var splitPatterns = pattern.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries); var components = new List(); @@ -104,11 +98,29 @@ public string BuildBookFileName(Author author, Edition edition, BookFile bookFil { var splitPattern = s; + var tokenHandlers = new Dictionary>(FileNameBuilderTokenEqualityComparer.Instance); + + /* Replace all tokens excluding the book title */ + AddAuthorTokens(tokenHandlers, author); + AddBookTokens(tokenHandlers, edition); + AddBookTitlePlaceholderTokens(tokenHandlers); + AddBookFileTokens(tokenHandlers, bookFile); + AddQualityTokens(tokenHandlers, author, bookFile); + AddMediaInfoTokens(tokenHandlers, bookFile); + AddCustomFormats(tokenHandlers, author, bookFile, customFormats); var component = ReplacePartTokens(splitPattern, tokenHandlers, namingConfig).Trim(); component = ReplaceTokens(component, tokenHandlers, namingConfig).Trim(); + /* Determine how long the name is and compute the max book title length based on what is left below the max */ + var maxPathSegmentLength = Math.Min(LongPathSupport.MaxFileNameLength, maxPath); + var maxBookTitleLength = maxPathSegmentLength - GetLengthWithoutBookTitle(component, namingConfig); + + /* Add the book title, truncating the length as necessary */ + AddBookTitleTokens(tokenHandlers, edition, maxBookTitleLength); + component = ReplaceTokens(component, tokenHandlers, namingConfig).Trim(); component = FileNameCleanupRegex.Replace(component, match => match.Captures[0].Value[0].ToString()); component = TrimSeparatorsRegex.Replace(component, string.Empty); + component = component.Replace("{ellipsis}", "..."); if (component.IsNotNullOrWhiteSpace()) { @@ -119,6 +131,11 @@ public string BuildBookFileName(Author author, Edition edition, BookFile bookFil return Path.Combine(components.ToArray()); } + public string BuildBookFileName(Author author, Edition edition, BookFile bookFile, NamingConfig namingConfig = null, List customFormats = null) + { + return BuildBookFileName(author, edition, bookFile, LongPathSupport.MaxFilePathLength, namingConfig, customFormats); + } + public string BuildBookFilePath(Author author, Edition edition, string fileName, string extension) { Ensure.That(extension, () => extension).IsNotNullOrWhiteSpace(); @@ -135,16 +152,16 @@ public string BuildBookPath(Author author) public BasicNamingConfig GetBasicNamingConfig(NamingConfig nameSpec) { - var trackFormat = GetTrackFormat(nameSpec.StandardBookFormat).LastOrDefault(); + var bookFormat = GetBookFormat(nameSpec.StandardBookFormat).LastOrDefault(); - if (trackFormat == null) + if (bookFormat == null) { return new BasicNamingConfig(); } var basicNamingConfig = new BasicNamingConfig { - Separator = trackFormat.Separator + Separator = bookFormat.Separator }; var titleTokens = TitleRegex.Matches(nameSpec.StandardBookFormat); @@ -251,10 +268,6 @@ private void AddAuthorTokens(Dictionary> tokenH private void AddBookTokens(Dictionary> tokenHandlers, Edition edition) { - tokenHandlers["{Book Title}"] = m => edition.Title; - tokenHandlers["{Book CleanTitle}"] = m => CleanTitle(edition.Title); - tokenHandlers["{Book TitleThe}"] = m => TitleThe(edition.Title); - var (titleNoSub, subtitle) = edition.Title.SplitBookTitle(edition.Book.Value.AuthorMetadata.Value.Name); tokenHandlers["{Book TitleNoSub}"] = m => titleNoSub; @@ -313,6 +326,20 @@ private void AddBookTokens(Dictionary> tokenHan } } + private void AddBookTitlePlaceholderTokens(Dictionary> tokenHandlers) + { + tokenHandlers["{Book Title}"] = m => null; + tokenHandlers["{Book CleanTitle}"] = m => null; + tokenHandlers["{Book TitleThe}"] = m => null; + } + + private void AddBookTitleTokens(Dictionary> tokenHandlers, Edition edition, int maxLength) + { + tokenHandlers["{Book Title}"] = m => GetBookTitle(edition.Title, maxLength); + tokenHandlers["{Book CleanTitle}"] = m => GetBookTitle(CleanTitle(edition.Title), maxLength); + tokenHandlers["{Book TitleThe}"] = m => GetBookTitle(TitleThe(edition.Title), maxLength); + } + private void AddBookFileTokens(Dictionary> tokenHandlers, BookFile bookFile) { tokenHandlers["{Original Title}"] = m => GetOriginalTitle(bookFile); @@ -372,13 +399,29 @@ private void AddCustomFormats(Dictionary> token tokenHandlers["{Custom Formats}"] = m => string.Join(" ", customFormats.Where(x => x.IncludeCustomFormatWhenRenaming)); } - private string ReplaceTokens(string pattern, Dictionary> tokenHandlers, NamingConfig namingConfig) + private string ReplaceTokens(string pattern, Dictionary> tokenHandlers, NamingConfig namingConfig, bool escape = false) { - return TitleRegex.Replace(pattern, match => ReplaceToken(match, tokenHandlers, namingConfig)); + return TitleRegex.Replace(pattern, match => ReplaceToken(match, tokenHandlers, namingConfig, escape)); } - private string ReplaceToken(Match match, Dictionary> tokenHandlers, NamingConfig namingConfig) + private string ReplaceToken(Match match, Dictionary> tokenHandlers, NamingConfig namingConfig, bool escape = false) { + if (match.Groups["escaped"].Success) + { + if (escape) + { + return match.Value; + } + else if (match.Value == "{{") + { + return "{"; + } + else if (match.Value == "}}") + { + return "}"; + } + } + var tokenMatch = new TokenMatch { RegexMatch = match, @@ -396,7 +439,14 @@ private string ReplaceToken(Match match, Dictionary string.Empty); - var replacementText = tokenHandler(tokenMatch).Trim(); + var replacementText = tokenHandler(tokenMatch); + if (replacementText == null) + { + // Preserve original token if handler returned null + return match.Value; + } + + replacementText = replacementText.Trim(); if (tokenMatch.Token.All(t => !char.IsLetter(t) || char.IsLower(t))) { @@ -419,6 +469,11 @@ private string ReplaceToken(Match match, Dictionary SeasonEpisodePatternRegex.Matches(pattern).OfType() + return _bookFormatCache.Get(pattern, () => SeasonEpisodePatternRegex.Matches(pattern).OfType() .Select(match => new BookFormat { BookSeparator = match.Groups["episodeSeparator"].Value, @@ -467,6 +522,23 @@ private BookFormat[] GetTrackFormat(string pattern) }).ToArray()); } + private string GetBookTitle(string title, int maxLength) + { + if (title.GetByteCount() <= maxLength) + { + return title; + } + + var titleLength = title.GetByteCount(); + + if (titleLength + 3 <= maxLength) + { + return $"{title.TrimEnd(' ', '.')}{{ellipsis}}"; + } + + return $"{title.Truncate(maxLength - 3).TrimEnd(' ', '.')}{{ellipsis}}"; + } + private string GetQualityProper(QualityModel quality) { if (quality.Revision.Version > 1) @@ -497,6 +569,16 @@ private string GetOriginalFileName(BookFile bookFile) return Path.GetFileNameWithoutExtension(bookFile.Path); } + private int GetLengthWithoutBookTitle(string pattern, NamingConfig namingConfig) + { + var tokenHandlers = new Dictionary>(FileNameBuilderTokenEqualityComparer.Instance); + tokenHandlers["{Book Title}"] = m => string.Empty; + tokenHandlers["{Book CleanTitle}"] = m => string.Empty; + tokenHandlers["{Book TitleThe}"] = m => string.Empty; + var result = ReplaceTokens(pattern, tokenHandlers, namingConfig); + return result.GetByteCount(); + } + private static string CleanFileName(string name, NamingConfig namingConfig) { var result = name;