Merge pull request #17 from cheir-mneme/feature/unpackerr

feat(download): add automatic archive extraction (Unpackerr)

src/NzbDrone.Common/ArchiveService.cs

@@ -6,6 +6,8 @@
 using ICSharpCode.SharpZipLib.Tar;
 using ICSharpCode.SharpZipLib.Zip;
 using NLog;
+using SharpCompress.Archives;
+using SharpCompress.Common;
 
 namespace NzbDrone.Common
 {
@@ -13,33 +15,106 @@ public interface IArchiveService
     {
         void Extract(string compressedFile, string destination);
         void CreateZip(string path, IEnumerable<string> files);
+        bool IsArchive(string path);
+        bool CanExtract(string path);
     }
 
     public class ArchiveService : IArchiveService
     {
         private readonly Logger _logger;
 
+        private static readonly HashSet<string> SupportedExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+        {
+            ".zip", ".rar", ".7z", ".tar", ".gz", ".tgz", ".tar.gz", ".bz2", ".tar.bz2"
+        };
+
         public ArchiveService(Logger logger)
         {
             _logger = logger;
         }
 
+        public bool IsArchive(string path)
+        {
+            var extension = Path.GetExtension(path);
+            return SupportedExtensions.Contains(extension) ||
+                   path.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase) ||
+                   path.EndsWith(".tar.bz2", StringComparison.OrdinalIgnoreCase);
+        }
+
+        public bool CanExtract(string path)
+        {
+            if (!File.Exists(path))
+            {
+                return false;
+            }
+
+            return IsArchive(path);
+        }
+
         public void Extract(string compressedFile, string destination)
         {
             _logger.Debug("Extracting archive [{0}] to [{1}]", compressedFile, destination);
 
-            if (compressedFile.EndsWith(".zip", StringComparison.InvariantCultureIgnoreCase))
+            var extension = Path.GetExtension(compressedFile).ToLowerInvariant();
+
+            if (extension == ".zip")
             {
                 ExtractZip(compressedFile, destination);
             }
-            else
+            else if (extension == ".rar" || extension == ".7z" || extension == ".r00")
+            {
+                ExtractWithSharpCompress(compressedFile, destination);
+            }
+            else if (compressedFile.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase) ||
+                     compressedFile.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase) ||
+                     extension == ".tar" || extension == ".gz")
             {
                 ExtractTgz(compressedFile, destination);
             }
+            else
+            {
+                ExtractWithSharpCompress(compressedFile, destination);
+            }
 
             _logger.Debug("Extraction complete.");
         }
 
+        private void ExtractWithSharpCompress(string compressedFile, string destination)
+        {
+            var destinationFullPath = Path.GetFullPath(destination);
+            Directory.CreateDirectory(destination);
+
+            using var archive = ArchiveFactory.Open(compressedFile);
+            foreach (var entry in archive.Entries)
+            {
+                if (entry.IsDirectory)
+                {
+                    continue;
+                }
+
+                var fullPath = Path.GetFullPath(Path.Combine(destination, entry.Key));
+
+                if (!fullPath.StartsWith(destinationFullPath + Path.DirectorySeparatorChar) &&
+                    !fullPath.Equals(destinationFullPath, StringComparison.Ordinal))
+                {
+                    _logger.Warn("Skipping archive entry with path traversal attempt: {0}", entry.Key);
+                    continue;
+                }
+
+                var directoryName = Path.GetDirectoryName(fullPath);
+                if (!string.IsNullOrEmpty(directoryName))
+                {
+                    Directory.CreateDirectory(directoryName);
+                }
+
+                entry.WriteToFile(fullPath, new ExtractionOptions
+                {
+                    ExtractFullPath = false,
+                    Overwrite = true
+                });
+            }
+        }
+
         public void CreateZip(string path, IEnumerable<string> files)
         {
             _logger.Debug("Creating archive {0}", path);

src/NzbDrone.Common/Radarr.Common.csproj

@@ -15,6 +15,7 @@
     <PackageReference Include="Npgsql" Version="9.0.3" />
     <PackageReference Include="Sentry" Version="4.0.2" />
     <PackageReference Include="NLog.Targets.Syslog" Version="7.0.0" />
+    <PackageReference Include="SharpCompress" Version="0.38.0" />
     <PackageReference Include="SharpZipLib" Version="1.4.2" />
     <PackageReference Include="SourceGear.sqlite3" Version="3.50.4.2" />
     <PackageReference Include="System.Data.SQLite" Version="2.0.2" />

src/NzbDrone.Core/Configuration/ConfigService.cs

@@ -162,6 +162,20 @@ public bool EnableCompletedDownloadHandling
             set { SetValue("EnableCompletedDownloadHandling", value); }
         }
 
+        public bool AutoExtractArchives
+        {
+            get { return GetValueBoolean("AutoExtractArchives", false); }
+
+            set { SetValue("AutoExtractArchives", value); }
+        }
+
+        public bool DeleteArchiveAfterExtraction
+        {
+            get { return GetValueBoolean("DeleteArchiveAfterExtraction", true); }
+
+            set { SetValue("DeleteArchiveAfterExtraction", value); }
+        }
+
         public bool PreferIndexerFlags
         {
             get { return GetValueBoolean("PreferIndexerFlags", false); }

src/NzbDrone.Core/Configuration/IConfigService.cs

@@ -24,6 +24,10 @@ public interface IConfigService
         bool AutoRedownloadFailed { get; set; }
         bool AutoRedownloadFailedFromInteractiveSearch { get; set; }
 
+        // Archive Extraction
+        bool AutoExtractArchives { get; set; }
+        bool DeleteArchiveAfterExtraction { get; set; }
+
         // Media Management
         bool AutoUnmonitorPreviouslyDownloadedMovies { get; set; }
         string RecycleBin { get; set; }

src/NzbDrone.Core/Download/CompletedDownloadService.cs

@@ -30,6 +30,7 @@ public class CompletedDownloadService : ICompletedDownloadService
         private readonly IHistoryService _historyService;
         private readonly IProvideImportItemService _provideImportItemService;
         private readonly IDownloadedMovieImportService _downloadedMovieImportService;
+        private readonly IDownloadExtractionService _downloadExtractionService;
         private readonly IParsingService _parsingService;
         private readonly IMovieService _movieService;
         private readonly ITrackedDownloadAlreadyImported _trackedDownloadAlreadyImported;
@@ -40,6 +41,7 @@ public CompletedDownloadService(IEventAggregator eventAggregator,
                                         IHistoryService historyService,
                                         IProvideImportItemService provideImportItemService,
                                         IDownloadedMovieImportService downloadedMovieImportService,
+                                        IDownloadExtractionService downloadExtractionService,
                                         IParsingService parsingService,
                                         IMovieService movieService,
                                         ITrackedDownloadAlreadyImported trackedDownloadAlreadyImported,
@@ -50,6 +52,7 @@ public CompletedDownloadService(IEventAggregator eventAggregator,
             _historyService = historyService;
             _provideImportItemService = provideImportItemService;
             _downloadedMovieImportService = downloadedMovieImportService;
+            _downloadExtractionService = downloadExtractionService;
             _parsingService = parsingService;
             _movieService = movieService;
             _trackedDownloadAlreadyImported = trackedDownloadAlreadyImported;
@@ -66,6 +69,17 @@ public void Check(TrackedDownload trackedDownload)
 
             SetImportItem(trackedDownload);
 
+            // Extract archives if needed before import
+            var outputPath = trackedDownload.DownloadItem.OutputPath.FullPath;
+            if (_downloadExtractionService.ShouldExtract(outputPath))
+            {
+                var extractedPath = _downloadExtractionService.ExtractIfNeeded(outputPath);
+                if (extractedPath != outputPath)
+                {
+                    _logger.Info("Archives extracted, updating import path to: {0}", extractedPath);
+                }
+            }
+
             // Only process tracked downloads that are still downloading or have been blocked for importing due to an issue with matching
             if (trackedDownload.State != TrackedDownloadState.Downloading && trackedDownload.State != TrackedDownloadState.ImportBlocked)
             {

src/NzbDrone.Core/Download/DownloadExtractionService.cs

@@ -0,0 +1,194 @@
+using System;
+using System.IO;
+using System.Linq;
+using NLog;
+using NzbDrone.Common;
+using NzbDrone.Common.Disk;
+using NzbDrone.Core.Configuration;
+
+namespace NzbDrone.Core.Download
+{
+    public interface IDownloadExtractionService
+    {
+        string ExtractIfNeeded(string downloadPath);
+        bool ShouldExtract(string downloadPath);
+    }
+
+    public class DownloadExtractionService : IDownloadExtractionService
+    {
+        private readonly IArchiveService _archiveService;
+        private readonly IDiskProvider _diskProvider;
+        private readonly IConfigService _configService;
+        private readonly Logger _logger;
+
+        public DownloadExtractionService(
+            IArchiveService archiveService,
+            IDiskProvider diskProvider,
+            IConfigService configService,
+            Logger logger)
+        {
+            _archiveService = archiveService;
+            _diskProvider = diskProvider;
+            _configService = configService;
+            _logger = logger;
+        }
+
+        public bool ShouldExtract(string downloadPath)
+        {
+            if (!_configService.AutoExtractArchives)
+            {
+                return false;
+            }
+
+            if (!_diskProvider.FolderExists(downloadPath) && !_diskProvider.FileExists(downloadPath))
+            {
+                return false;
+            }
+
+            if (_diskProvider.FileExists(downloadPath))
+            {
+                return _archiveService.IsArchive(downloadPath);
+            }
+
+            var files = _diskProvider.GetFiles(downloadPath, true);
+            return files.Any(f => _archiveService.IsArchive(f));
+        }
+
+        public string ExtractIfNeeded(string downloadPath)
+        {
+            if (!ShouldExtract(downloadPath))
+            {
+                return downloadPath;
+            }
+
+            try
+            {
+                if (_diskProvider.FileExists(downloadPath))
+                {
+                    return ExtractSingleArchive(downloadPath);
+                }
+
+                return ExtractArchivesInFolder(downloadPath);
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, "Failed to extract archives in {0}", downloadPath);
+                return downloadPath;
+            }
+        }
+
+        private string ExtractSingleArchive(string archivePath)
+        {
+            var extractionPath = Path.Combine(
+                Path.GetDirectoryName(archivePath),
+                Path.GetFileNameWithoutExtension(archivePath));
+
+            _logger.Info("Extracting {0} to {1}", archivePath, extractionPath);
+
+            _archiveService.Extract(archivePath, extractionPath);
+
+            if (_configService.DeleteArchiveAfterExtraction)
+            {
+                _logger.Debug("Deleting archive after extraction: {0}", archivePath);
+                _diskProvider.DeleteFile(archivePath);
+            }
+
+            return extractionPath;
+        }
+
+        private string ExtractArchivesInFolder(string folderPath)
+        {
+            var files = _diskProvider.GetFiles(folderPath, true);
+            var archiveFiles = files
+                .Where(f => _archiveService.IsArchive(f))
+                .Where(f => !IsPartOfMultiVolumeArchive(f))
+                .ToList();
+
+            if (!archiveFiles.Any())
+            {
+                return folderPath;
+            }
+
+            foreach (var archiveFile in archiveFiles)
+            {
+                var extractionPath = Path.GetDirectoryName(archiveFile);
+                _logger.Info("Extracting {0}", archiveFile);
+
+                try
+                {
+                    _archiveService.Extract(archiveFile, extractionPath);
+
+                    if (_configService.DeleteArchiveAfterExtraction)
+                    {
+                        DeleteArchiveWithParts(archiveFile);
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.Warn(ex, "Failed to extract {0}", archiveFile);
+                }
+            }
+
+            return folderPath;
+        }
+
+        private bool IsPartOfMultiVolumeArchive(string path)
+        {
+            var extension = Path.GetExtension(path);
+
+            if (extension.StartsWith(".r", StringComparison.OrdinalIgnoreCase) &&
+                extension.Length == 4 &&
+                int.TryParse(extension.AsSpan(2), out _))
+            {
+                return true;
+            }
+
+            if (path.Contains(".part") && extension.Equals(".rar", StringComparison.OrdinalIgnoreCase))
+            {
+                var filename = Path.GetFileNameWithoutExtension(path);
+                return filename.EndsWith(".part1", StringComparison.OrdinalIgnoreCase) == false;
+            }
+
+            return false;
+        }
+
+        private void DeleteArchiveWithParts(string archivePath)
+        {
+            var directory = Path.GetDirectoryName(archivePath);
+            var baseName = Path.GetFileNameWithoutExtension(archivePath);
+
+            var partFiles = _diskProvider.GetFiles(directory, false)
+                .Where(f => IsArchivePartFile(f, baseName))
+                .ToList();
+
+            foreach (var partFile in partFiles)
+            {
+                _logger.Debug("Deleting archive part: {0}", partFile);
+                _diskProvider.DeleteFile(partFile);
+            }
+
+            _diskProvider.DeleteFile(archivePath);
+        }
+
+        private bool IsArchivePartFile(string filePath, string baseName)
+        {
+            var fileName = Path.GetFileNameWithoutExtension(filePath);
+            var extension = Path.GetExtension(filePath);
+
+            if (fileName.Equals(baseName, StringComparison.OrdinalIgnoreCase) &&
+                extension.StartsWith(".r", StringComparison.OrdinalIgnoreCase) &&
+                extension.Length == 4)
+            {
+                return true;
+            }
+
+            if (fileName.StartsWith(baseName + ".part", StringComparison.OrdinalIgnoreCase) &&
+                extension.Equals(".rar", StringComparison.OrdinalIgnoreCase))
+            {
+                return true;
+            }
+
+            return false;
+        }
+    }
+}