Backup_Repos/Readarr
1
0
Fork 0
mirror of https://github.com/Readarr/Readarr synced 2025-12-30 20:25:18 +01:00
Code Issues Projects Releases Wiki Activity

Fixed: Contain memory usage during library import

Browse source
This commit is contained in:
ta264 2022-02-06 18:58:14 +00:00
parent 8e37aa2e78
commit 3a360d7a1e
8 changed files with 100 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/Directory.Packages.props
View file

@ -13,8 +13,10 @@
<PackageVersion Include="FluentValidation" Version="8.6.2" />
<PackageVersion Include="Ical.Net" Version="4.2.0" />
<PackageVersion Include="ImpromptuInterface" Version="7.0.1" />
<PackageVersion Include="LazyCache" Version="2.4.0" />
<PackageVersion Include="Mailkit" Version="3.1.0" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.1" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="6.0.0" />
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="6.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />

2
src/NzbDrone.Core/Books/Model/Author.cs
View file

@ -77,7 +77,9 @@ public override void UseDbFieldsFrom(Author other)
RootFolderPath = other.RootFolderPath;
Added = other.Added;
QualityProfileId = other.QualityProfileId;
QualityProfile = other.QualityProfile;
MetadataProfileId = other.MetadataProfileId;
MetadataProfile = other.MetadataProfile;
Tags = other.Tags;
AddOptions = other.AddOptions;
}

4
src/NzbDrone.Core/MediaFiles/BookImport/Identification/IdentificationService.cs
View file

@ -74,7 +74,7 @@ public List<LocalEdition> Identify(List<LocalBook> localTracks, IdentificationOv
// 3 find best candidate
var watch = System.Diagnostics.Stopwatch.StartNew();
_logger.Debug("Starting track identification");
_logger.Debug("Starting book identification");
var releases = GetLocalBookReleases(localTracks, config.SingleRelease);
@ -183,7 +183,7 @@ private void IdentifyRelease(LocalEdition localBookRelease, IdentificationOverri
_logger.Debug($"Best release found in {watch.ElapsedMilliseconds}ms");
localBookRelease.PopulateMatch();
localBookRelease.PopulateMatch(config.KeepAllEditions);
_logger.Debug($"IdentifyRelease done in {watch.ElapsedMilliseconds}ms");
}

1
src/NzbDrone.Core/MediaFiles/BookImport/ImportDecisionMaker.cs
View file

@ -42,6 +42,7 @@ public class ImportDecisionMakerConfig
public bool SingleRelease { get; set; }
public bool IncludeExisting { get; set; }
public bool AddNewAuthors { get; set; }
public bool KeepAllEditions { get; set; }
}
public class ImportDecisionMaker : IMakeImportDecision

6
src/NzbDrone.Core/MediaFiles/BookImport/Manual/ManualImportService.cs
View file

@ -117,7 +117,8 @@ public List<ManualImportItem> GetMediaFiles(string path, string downloadId, Auth
NewDownload = true,
SingleRelease = false,
IncludeExisting = !replaceExistingFiles,
AddNewAuthors = false
AddNewAuthors = false,
KeepAllEditions = true
};
var decision = _importDecisionMaker.GetImportDecisions(files, null, null, config);
@ -162,7 +163,8 @@ private List<ManualImportItem> ProcessFolder(string folder, string downloadId, A
NewDownload = true,
SingleRelease = false,
IncludeExisting = !replaceExistingFiles,
AddNewAuthors = false
AddNewAuthors = false,
KeepAllEditions = true
};
var decisions = _importDecisionMaker.GetImportDecisions(authorFiles, idOverrides, itemInfo, config);

23
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoProxy.cs
View file

@ -5,6 +5,9 @@
using System.Net;
using System.Text.Json;
using System.Threading;
using LazyCache;
using LazyCache.Providers;
using Microsoft.Extensions.Caching.Memory;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
@ -35,7 +38,7 @@ public class BookInfoProxy : IProvideAuthorInfo, IProvideBookInfo, ISearchForNew
private readonly Logger _logger;
private readonly IMetadataRequestBuilder _requestBuilder;
private readonly ICached<HashSet<string>> _cache;
private readonly ICached<Author> _authorCache;
private readonly CachingService _authorCache;
public BookInfoProxy(IHttpClient httpClient,
ICachedHttpResponseService cachedHttpClient,
@ -55,8 +58,13 @@ public BookInfoProxy(IHttpClient httpClient,
_editionService = editionService;
_requestBuilder = requestBuilder;
_cache = cacheManager.GetCache<HashSet<string>>(GetType());
_authorCache = cacheManager.GetRollingCache<Author>(GetType(), "authorCache", TimeSpan.FromMinutes(5));
_logger = logger;
_authorCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions { SizeLimit = 10 })));
_authorCache.DefaultCachePolicy = new CacheDefaults
{
DefaultCacheDurationSeconds = 60
};
}
public HashSet<string> GetChangedAuthors(DateTime startTime)
@ -532,7 +540,16 @@ private void AddDbIds(string authorId, Book book, Dictionary<string, AuthorMetad
private Author PollAuthor(string foreignAuthorId)
{
return _authorCache.Get(foreignAuthorId, () => PollAuthorUncached(foreignAuthorId));
return _authorCache.GetOrAdd(foreignAuthorId,
() => PollAuthorUncached(foreignAuthorId),
new LazyCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10),
ImmediateAbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10),
Size = 1,
SlidingExpiration = TimeSpan.FromMinutes(1),
ExpirationMode = ExpirationMode.ImmediateEviction
}.RegisterPostEvictionCallback((key, value, reason, state) => _logger.Debug($"Clearing cache for {key} due to {reason}")));
}
private Author PollAuthorUncached(string foreignAuthorId)

73
src/NzbDrone.Core/Parser/Model/LocalEdition.cs
View file

@ -4,6 +4,7 @@
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Books;
using NzbDrone.Core.MediaFiles.BookImport.Identification;
using SixLabors.ImageSharp.Processing;
namespace NzbDrone.Core.Parser.Model
{
@ -35,17 +36,77 @@ public LocalEdition(List<LocalBook> tracks)
public List<LocalBook> ExistingTracks { get; set; }
public bool NewDownload { get; set; }
public void PopulateMatch()
public void PopulateMatch(bool keepAllEditions)
{
if (Edition != null)
{
LocalBooks = LocalBooks.Concat(ExistingTracks).DistinctBy(x => x.Path).ToList();
foreach (var localTrack in LocalBooks)
if (!keepAllEditions)
{
localTrack.Edition = Edition;
localTrack.Book = Edition.Book.Value;
localTrack.Author = Edition.Book.Value.Author.Value;
localTrack.PartCount = LocalBooks.Count;
// Manually clone the edition / book to avoid holding references to *every* edition we have
// seen during the matching process
var edition = new Edition();
edition.UseMetadataFrom(Edition);
edition.UseDbFieldsFrom(Edition);
var fullBook = Edition.Book.Value;
var book = new Book();
book.UseMetadataFrom(fullBook);
book.UseDbFieldsFrom(fullBook);
book.Author.Value.UseMetadataFrom(fullBook.Author.Value);
book.Author.Value.UseDbFieldsFrom(fullBook.Author.Value);
book.Author.Value.Metadata = fullBook.AuthorMetadata.Value;
book.AuthorMetadata = fullBook.AuthorMetadata.Value;
book.BookFiles = fullBook.BookFiles;
book.Editions = new List<Edition> { edition };
if (fullBook.SeriesLinks.IsLoaded)
{
book.SeriesLinks = fullBook.SeriesLinks.Value.Select(l => new SeriesBookLink
{
Book = book,
Series = new Series
{
ForeignSeriesId = l.Series.Value.ForeignSeriesId,
Title = l.Series.Value.Title,
Description = l.Series.Value.Description,
Numbered = l.Series.Value.Numbered,
WorkCount = l.Series.Value.WorkCount,
PrimaryWorkCount = l.Series.Value.PrimaryWorkCount
},
IsPrimary = l.IsPrimary,
Position = l.Position,
SeriesPosition = l.SeriesPosition
}).ToList();
}
else
{
book.SeriesLinks = fullBook.SeriesLinks;
}
edition.Book = book;
Edition = edition;
foreach (var localTrack in LocalBooks)
{
localTrack.Edition = edition;
localTrack.Book = book;
localTrack.Author = book.Author.Value;
localTrack.PartCount = LocalBooks.Count;
}
}
else
{
foreach (var localTrack in LocalBooks)
{
localTrack.Edition = Edition;
localTrack.Book = Edition.Book.Value;
localTrack.Author = Edition.Book.Value.Author.Value;
localTrack.PartCount = LocalBooks.Count;
}
}
}
}

2
src/NzbDrone.Core/Readarr.Core.csproj
View file

@ -4,9 +4,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Dapper" />
<PackageReference Include="LazyCache" />
<PackageReference Include="System.Text.Json" />
<PackageReference Include="System.Text.Encoding.CodePages" />
<PackageReference Include="System.Memory" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
<PackageReference Include="Microsoft.Extensions.Logging" />
<PackageReference Include="Microsoft.Extensions.Configuration" />