Fixed: Track when calibre changes files to prevent unnecessary re-scans

src/NzbDrone.Core/Books/Calibre/CalibreProxy.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Net;
+using System.Threading;
 using System.Threading.Tasks;
 using FluentValidation;
 using FluentValidation.Results;
@@ -12,6 +14,7 @@
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Configuration;
 using NzbDrone.Core.MediaCover;
 using NzbDrone.Core.MediaFiles;
 using NzbDrone.Core.RemotePathMappings;
@@ -21,14 +24,11 @@ namespace NzbDrone.Core.Books.Calibre
 {
     public interface ICalibreProxy
     {
-        CalibreImportJob AddBook(BookFile book, CalibreSettings settings);
+        BookFile AddAndConvert(BookFile file, CalibreSettings settings);
         void DeleteBook(BookFile book, CalibreSettings settings);
         void DeleteBooks(List<BookFile> books, CalibreSettings settings);
-        void AddFormat(BookFile file, CalibreSettings settings);
         void RemoveFormats(int calibreId, IEnumerable<string> formats, CalibreSettings settings);
         void SetFields(BookFile file, CalibreSettings settings, bool updateCover = true, bool embed = false);
-        CalibreBookData GetBookData(int calibreId, CalibreSettings settings);
-        long ConvertBook(int calibreId, CalibreConversionOptions options, CalibreSettings settings);
         List<string> GetAllBookFilePaths(CalibreSettings settings);
         CalibreBook GetBook(int calibreId, CalibreSettings settings);
         List<CalibreBook> GetBooks(List<int> calibreId, CalibreSettings settings);
@@ -43,6 +43,8 @@ public class CalibreProxy : ICalibreProxy
         private readonly IMapCoversToLocal _mediaCoverService;
         private readonly IRemotePathMappingService _pathMapper;
         private readonly IRootFolderWatchingService _rootFolderWatchingService;
+        private readonly IMediaFileService _mediaFileService;
+        private readonly IConfigService _configService;
         private readonly Logger _logger;
         private readonly ICached<CalibreBook> _bookCache;
 
@@ -50,6 +52,8 @@ public CalibreProxy(IHttpClient httpClient,
                             IMapCoversToLocal mediaCoverService,
                             IRemotePathMappingService pathMapper,
                             IRootFolderWatchingService rootFolderWatchingService,
+                            IMediaFileService mediaFileService,
+                            IConfigService configService,
                             ICacheManager cacheManager,
                             Logger logger)
         {
@@ -57,6 +61,8 @@ public CalibreProxy(IHttpClient httpClient,
             _mediaCoverService = mediaCoverService;
             _pathMapper = pathMapper;
             _rootFolderWatchingService = rootFolderWatchingService;
+            _mediaFileService = mediaFileService;
+            _configService = configService;
             _bookCache = cacheManager.GetCache<CalibreBook>(GetType());
             _logger = logger;
         }
@@ -69,7 +75,59 @@ public static string GetOriginalFormat(Dictionary<string, CalibreBookFormat> for
                 .FirstOrDefault().Value?.Path;
         }
 
-        public CalibreImportJob AddBook(BookFile book, CalibreSettings settings)
+        public BookFile AddAndConvert(BookFile file, CalibreSettings settings)
+        {
+            _logger.Trace($"Importing to calibre: {file.Path} calibre id: {file.CalibreId}");
+
+            if (file.CalibreId == 0)
+            {
+                var import = AddBook(file, settings);
+                file.CalibreId = import.Id;
+            }
+            else
+            {
+                AddFormat(file, settings);
+            }
+
+            _rootFolderWatchingService.ReportFileSystemChangeBeginning(file.Path);
+
+            SetFields(file, settings, true, _configService.EmbedMetadata);
+
+            if (settings.OutputFormat.IsNotNullOrWhiteSpace())
+            {
+                _logger.Trace($"Getting book data for {file.CalibreId}");
+                var options = GetBookData(file.CalibreId, settings);
+                var inputFormat = file.Quality.Quality.Name.ToUpper();
+
+                options.Conversion_options.Input_fmt = inputFormat;
+
+                var formats = settings.OutputFormat.Split(',').Select(x => x.Trim());
+                foreach (var format in formats)
+                {
+                    if (format.ToLower() == inputFormat ||
+                        options.Input_formats.Contains(format, StringComparer.OrdinalIgnoreCase))
+                    {
+                        continue;
+                    }
+
+                    options.Conversion_options.Output_fmt = format;
+
+                    if (settings.OutputProfile != (int)CalibreProfile.@default)
+                    {
+                        options.Conversion_options.Options.Output_profile = ((CalibreProfile)settings.OutputProfile).ToString();
+                    }
+
+                    _logger.Trace($"Starting conversion to {format}");
+
+                    _rootFolderWatchingService.ReportFileSystemChangeBeginning(Path.ChangeExtension(file.Path, format));
+                    ConvertBook(file.CalibreId, options.Conversion_options, settings);
+                }
+            }
+
+            return file;
+        }
+
+        private CalibreImportJob AddBook(BookFile book, CalibreSettings settings)
         {
             var jobid = (int)(DateTime.UtcNow.Ticks % 1000000000);
             var addDuplicates = 1;
@@ -114,7 +172,7 @@ public void DeleteBooks(List<BookFile> books, CalibreSettings settings)
             _httpClient.Post(request);
         }
 
-        public void AddFormat(BookFile file, CalibreSettings settings)
+        private void AddFormat(BookFile file, CalibreSettings settings)
         {
             var format = Path.GetExtension(file.Path);
             var bookData = Convert.ToBase64String(File.ReadAllBytes(file.Path));
@@ -214,10 +272,29 @@ public void SetFields(BookFile file, CalibreSettings settings, bool updateCover
 
             ExecuteSetFields(file.CalibreId, payload, settings);
 
+            // updating the calibre metadata may have renamed the file, so track that
+            var updated = GetBook(file.CalibreId, settings);
+
+            var updatedPath = GetOriginalFormat(updated.Formats);
+
+            if (updatedPath != file.Path)
+            {
+                _rootFolderWatchingService.ReportFileSystemChangeBeginning(updatedPath);
+                file.Path = updatedPath;
+            }
+
+            var fileInfo = new FileInfo(file.Path);
+            file.Size = fileInfo.Length;
+            file.Modified = fileInfo.LastWriteTimeUtc;
+
+            if (file.Id > 0)
+            {
+                _mediaFileService.Update(file);
+            }
+
             if (embed)
             {
-                _rootFolderWatchingService.ReportFileSystemChangeBeginning(file.Path);
-                EmbedMetadata(file.CalibreId, settings);
+                EmbedMetadata(file, settings);
             }
         }
 
@@ -233,19 +310,54 @@ private void ExecuteSetFields(int id, CalibreChangesPayload payload, CalibreSett
             _httpClient.Execute(request);
         }
 
-        private void EmbedMetadata(int id, CalibreSettings settings)
+        private void EmbedMetadata(BookFile file, CalibreSettings settings)
         {
+            _rootFolderWatchingService.ReportFileSystemChangeBeginning(file.Path);
+
             var request = GetBuilder($"cdb/cmd/embed_metadata", settings)
                 .AddQueryParam("library_id", settings.Library)
                 .Post()
                 .SetHeader("Content-Type", "application/json")
                 .Build();
 
-            request.SetContent($"[{id}, null]");
+            request.SetContent($"[{file.CalibreId}, null]");
             _httpClient.Execute(request);
+
+            PollEmbedStatus(file, settings);
         }
 
-        public CalibreBookData GetBookData(int calibreId, CalibreSettings settings)
+        private void PollEmbedStatus(BookFile file, CalibreSettings settings)
+        {
+            var previous = new FileInfo(file.Path);
+            Thread.Sleep(100);
+
+            FileInfo current = null;
+
+            var i = 0;
+            while (i++ < 20)
+            {
+                current = new FileInfo(file.Path);
+
+                if (current.LastWriteTimeUtc == previous.LastWriteTimeUtc &&
+                    current.LastWriteTimeUtc != file.Modified)
+                {
+                    break;
+                }
+
+                previous = current;
+                Thread.Sleep(1000);
+            }
+
+            file.Size = current.Length;
+            file.Modified = current.LastWriteTimeUtc;
+
+            if (file.Id > 0)
+            {
+                _mediaFileService.Update(file);
+            }
+        }
+
+        private CalibreBookData GetBookData(int calibreId, CalibreSettings settings)
         {
             try
             {
@@ -261,7 +373,7 @@ public CalibreBookData GetBookData(int calibreId, CalibreSettings settings)
             }
         }
 
-        public long ConvertBook(int calibreId, CalibreConversionOptions options, CalibreSettings settings)
+        private long ConvertBook(int calibreId, CalibreConversionOptions options, CalibreSettings settings)
         {
             try
             {

src/NzbDrone.Core/MediaFiles/EbookTagService.cs

@@ -176,21 +176,6 @@ private void WriteTagsInternal(BookFile file, bool updateCover, bool embedMetada
 
             var rootFolder = _rootFolderService.GetBestRootFolder(file.Path);
             _calibre.SetFields(file, rootFolder.CalibreSettings, updateCover, embedMetadata);
-
-            // updating the calibre metadata may have renamed the file, so track that
-            var updated = _calibre.GetBook(file.CalibreId, rootFolder.CalibreSettings);
-
-            var updatedPath = CalibreProxy.GetOriginalFormat(updated.Formats);
-
-            if (updatedPath != file.Path)
-            {
-                file.Path = updatedPath;
-
-                if (file.Id > 0)
-                {
-                    _mediaFileService.Update(file);
-                }
-            }
         }
 
         private IEnumerable<RetagBookFilePreview> GetPreviews(List<BookFile> files)

src/NzbDrone.Core/MediaFiles/UpgradeMediaFileService.cs

@@ -19,36 +19,30 @@ public interface IUpgradeMediaFiles
 
     public class UpgradeMediaFileService : IUpgradeMediaFiles
     {
-        private readonly IConfigService _configService;
         private readonly IRecycleBinProvider _recycleBinProvider;
         private readonly IMediaFileService _mediaFileService;
         private readonly IMetadataTagService _metadataTagService;
         private readonly IMoveBookFiles _bookFileMover;
         private readonly IDiskProvider _diskProvider;
         private readonly IRootFolderService _rootFolderService;
-        private readonly IRootFolderWatchingService _rootFolderWatchingService;
         private readonly ICalibreProxy _calibre;
         private readonly Logger _logger;
 
-        public UpgradeMediaFileService(IConfigService configService,
-                                       IRecycleBinProvider recycleBinProvider,
+        public UpgradeMediaFileService(IRecycleBinProvider recycleBinProvider,
                                        IMediaFileService mediaFileService,
                                        IMetadataTagService metadataTagService,
                                        IMoveBookFiles bookFileMover,
                                        IDiskProvider diskProvider,
                                        IRootFolderService rootFolderService,
-                                       IRootFolderWatchingService rootFolderWatchingService,
                                        ICalibreProxy calibre,
                                        Logger logger)
         {
-            _configService = configService;
             _recycleBinProvider = recycleBinProvider;
             _mediaFileService = mediaFileService;
             _metadataTagService = metadataTagService;
             _bookFileMover = bookFileMover;
             _diskProvider = diskProvider;
             _rootFolderService = rootFolderService;
-            _rootFolderWatchingService = rootFolderWatchingService;
             _calibre = calibre;
             _logger = logger;
         }
@@ -115,7 +109,7 @@ public BookFileMoveResult UpgradeBookFile(BookFile bookFile, LocalBook localBook
             {
                 var source = bookFile.Path;
 
-                moveFileResult.BookFile = CalibreAddAndConvert(bookFile, settings);
+                moveFileResult.BookFile = _calibre.AddAndConvert(bookFile, settings);
 
                 if (!copyOnly)
                 {
@@ -125,60 +119,5 @@ public BookFileMoveResult UpgradeBookFile(BookFile bookFile, LocalBook localBook
 
             return moveFileResult;
         }
-
-        public BookFile CalibreAddAndConvert(BookFile file, CalibreSettings settings)
-        {
-            _logger.Trace($"Importing to calibre: {file.Path} calibre id: {file.CalibreId}");
-
-            if (file.CalibreId == 0)
-            {
-                var import = _calibre.AddBook(file, settings);
-                file.CalibreId = import.Id;
-            }
-            else
-            {
-                _calibre.AddFormat(file, settings);
-            }
-
-            _rootFolderWatchingService.ReportFileSystemChangeBeginning(file.Path);
-
-            _calibre.SetFields(file, settings, true, _configService.EmbedMetadata);
-
-            var updated = _calibre.GetBook(file.CalibreId, settings);
-            var path = updated.Formats.Values.OrderByDescending(x => x.LastModified).First().Path;
-
-            file.Path = path;
-
-            if (settings.OutputFormat.IsNotNullOrWhiteSpace())
-            {
-                _logger.Trace($"Getting book data for {file.CalibreId}");
-                var options = _calibre.GetBookData(file.CalibreId, settings);
-                var inputFormat = file.Quality.Quality.Name.ToUpper();
-
-                options.Conversion_options.Input_fmt = inputFormat;
-
-                var formats = settings.OutputFormat.Split(',').Select(x => x.Trim());
-                foreach (var format in formats)
-                {
-                    if (format.ToLower() == inputFormat ||
-                        options.Input_formats.Contains(format, StringComparer.OrdinalIgnoreCase))
-                    {
-                        continue;
-                    }
-
-                    options.Conversion_options.Output_fmt = format;
-
-                    if (settings.OutputProfile != (int)CalibreProfile.@default)
-                    {
-                        options.Conversion_options.Options.Output_profile = ((CalibreProfile)settings.OutputProfile).ToString();
-                    }
-
-                    _logger.Trace($"Starting conversion to {format}");
-                    _calibre.ConvertBook(file.CalibreId, options.Conversion_options, settings);
-                }
-            }
-
-            return file;
-        }
     }
 }