From a98d331b20caba71557ba9a52fa7dc8502635b08 Mon Sep 17 00:00:00 2001 From: Sean Parsons Date: Wed, 25 Mar 2026 22:51:49 +0000 Subject: [PATCH] Cap scan/import parallelism via LIDARR_MEDIA_IO_PARALLELISM Default max degree 2 to avoid IO storms on NFS/remote storage. Applies to folder scan, tag reads, and candidate scoring PLINQ. Document env var and Docker usage in MULTITHREAD_README.md. Made-with: Cursor --- MULTITHREAD_README.md | 57 +++++++++++++++++++ src/NzbDrone.Common/MediaImportParallelism.cs | 40 +++++++++++++ .../MediaFiles/DiskScanService.cs | 2 +- .../Identification/IdentificationService.cs | 2 +- .../TrackImport/ImportDecisionMaker.cs | 3 +- 5 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 MULTITHREAD_README.md create mode 100644 src/NzbDrone.Common/MediaImportParallelism.cs diff --git a/MULTITHREAD_README.md b/MULTITHREAD_README.md new file mode 100644 index 000000000..5be263da0 --- /dev/null +++ b/MULTITHREAD_README.md @@ -0,0 +1,57 @@ +# Multithreaded library scan / import (this fork) + +This branch adds a faster, **parallel** disk scan and import path. Upstream Lidarr does much of this work sequentially; this fork parallelizes folder scanning, tag reads, and release-candidate scoring. + +A **Dockerfile.multithread** in this repository builds a self-contained binary and overlays it on `ghcr.io/linuxserver/lidarr:nightly` (see CI or build from repo root per that file’s comments). A wrapper layout that keeps this tree in a `lidarr-src/` subdirectory can use the parent `Dockerfile` instead. + +## `LIDARR_MEDIA_IO_PARALLELISM` + +Parallel import work is **not** limited by Lidarr’s download bandwidth or rate settings (those apply to indexers/clients only). On **slow or remote storage** (especially NFS), too much concurrency can saturate IOPS and make the host feel stuck. This variable caps how many workers run at once for the fork’s parallel paths. + +| | | +| --- | --- | +| **Name** | `LIDARR_MEDIA_IO_PARALLELISM` | +| **Default** | `2` (used when the variable is unset, empty, or not a valid integer) | +| **Allowed** | Integers **1**–**64**; values below **1** fall back to the default | +| **Scope** | Process environment (read once at first use) | + +It applies to: + +- parallel **folder scans** when collecting audio files; +- parallel **tag / metadata reads** when building import decisions; +- parallel **candidate release scoring** during identification. + +**Docker:** fully supported. Set the variable on the container like any other env; the .NET process reads the container environment. + +### Docker Compose + +```yaml +services: + lidarr: + image: your-registry/lidarr-nightly-multithread:latest + environment: + - PUID=1000 + - PGID=1000 + - TZ=Etc/UTC + # Gentle on NFS / network mounts; omit for default (2) + - LIDARR_MEDIA_IO_PARALLELISM=1 +``` + +### `docker run` + +```bash +docker run -e LIDARR_MEDIA_IO_PARALLELISM=4 … your-image +``` + +### When to change it + +- **NFS, SMB, or sluggish disks:** try `1` or leave default `2`. +- **Library and app on fast local storage (e.g. same NAS app dataset, local SSD):** try `4`–`8` or higher (up to 64) and watch CPU, I/O, and responsiveness. + +### Implementation reference + +Logic and constant name: `src/NzbDrone.Common/MediaImportParallelism.cs`. + +## Relationship to upstream + +Behavior outside scan/import parallelism matches your chosen base (e.g. nightly image + overlaid build). For upstream docs and support channels, see [Lidarr](https://github.com/Lidarr/Lidarr) and the [Servarr wiki](https://wiki.servarr.com/lidarr). diff --git a/src/NzbDrone.Common/MediaImportParallelism.cs b/src/NzbDrone.Common/MediaImportParallelism.cs new file mode 100644 index 000000000..1100b32c1 --- /dev/null +++ b/src/NzbDrone.Common/MediaImportParallelism.cs @@ -0,0 +1,40 @@ +using System; + +namespace NzbDrone.Common +{ + /// + /// Limits parallel disk work during library scan/import (tag reads, folder scans, candidate scoring). + /// Unrelated to download bandwidth limits in Lidarr settings. + /// + public static class MediaImportParallelism + { + public const string EnvironmentVariableName = "LIDARR_MEDIA_IO_PARALLELISM"; + + private const int DefaultMaxDegree = 2; + private const int MinDegree = 1; + private const int MaxDegreeCap = 64; + + private static readonly Lazy MaxDegreeLazy = new Lazy(ReadMaxDegree); + + /// + /// Maximum concurrent workers for scan/import parallelism. Default 2; override with LIDARR_MEDIA_IO_PARALLELISM (1–64). + /// + public static int MaxDegreeOfParallelism => MaxDegreeLazy.Value; + + private static int ReadMaxDegree() + { + var raw = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(raw) || !int.TryParse(raw.Trim(), out var parsed)) + { + return DefaultMaxDegree; + } + + if (parsed < MinDegree) + { + return DefaultMaxDegree; + } + + return Math.Min(parsed, MaxDegreeCap); + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index 3904b6a80..b1c0f5b71 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -126,7 +126,7 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil var musicFilesStopwatch = Stopwatch.StartNew(); - Parallel.ForEach(foldersToScan, folder => + Parallel.ForEach(foldersToScan, new ParallelOptions { MaxDegreeOfParallelism = MediaImportParallelism.MaxDegreeOfParallelism }, folder => { _logger.ProgressInfo("Scanning {0}", folder); diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index 417eda90c..e752fad1a 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -305,7 +305,7 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List x.Path))); - var maxParallelism = Math.Max(1, Environment.ProcessorCount); + var maxParallelism = MediaImportParallelism.MaxDegreeOfParallelism; var scoredCandidates = candidateReleases .Select((candidateRelease, index) => new { candidateRelease, index }) .AsParallel() diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs index 5b55626b7..3b34824b3 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using NLog; +using NzbDrone.Common; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.DecisionEngine; @@ -106,7 +107,7 @@ public Tuple, List>> GetLocalTracks( var processedTracks = new ConcurrentBag<(int Index, LocalTrack Track)>(); var processedDecisions = new ConcurrentBag<(int Index, ImportDecision Decision)>(); var progress = 0; - var maxParallelism = Math.Max(1, Environment.ProcessorCount); + var maxParallelism = MediaImportParallelism.MaxDegreeOfParallelism; var filesWithIndex = files.Select((file, index) => new { file, index }).ToList(); Parallel.ForEach(filesWithIndex, new ParallelOptions { MaxDegreeOfParallelism = maxParallelism }, item =>