From bffe49f2be96906f114c598562994ef422e8e877 Mon Sep 17 00:00:00 2001 From: Sean Parsons Date: Thu, 12 Mar 2026 01:41:00 +0000 Subject: [PATCH 1/6] New: Multithreaded file scan and artist/album matching for faster import - ImportDecisionMaker: parallelize tag reading and file augmentation (Parallel.ForEach) with degree of parallelism based on processor count. - IdentificationService: parallelize candidate release scoring in GetBestRelease (AsParallel) so multiple album candidates are matched concurrently. Significantly faster on large libraries (60k+ tracks). Made-with: Cursor --- .../Identification/IdentificationService.cs | 51 +++++++++++-------- .../TrackImport/ImportDecisionMaker.cs | 34 +++++++++---- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index 19f81f152..417eda90c 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -305,15 +305,18 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List x.Path))); - var bestDistance = 1.0; - - foreach (var candidateRelease in candidateReleases) + var maxParallelism = Math.Max(1, Environment.ProcessorCount); + var scoredCandidates = candidateReleases + .Select((candidateRelease, index) => new { candidateRelease, index }) + .AsParallel() + .WithDegreeOfParallelism(maxParallelism) + .Select(item => { - var release = candidateRelease.AlbumRelease; - _logger.Debug("Trying Release {0} [{1}, {2} tracks, {3} existing]", release, release.Title, release.TrackCount, candidateRelease.ExistingTracks.Count); + var release = item.candidateRelease.AlbumRelease; + _logger.Debug("Trying Release {0} [{1}, {2} tracks, {3} existing]", release, release.Title, release.TrackCount, item.candidateRelease.ExistingTracks.Count); var rwatch = System.Diagnostics.Stopwatch.StartNew(); - var extraTrackPaths = candidateRelease.ExistingTracks.Select(x => x.Path).ToList(); + var extraTrackPaths = new HashSet(item.candidateRelease.ExistingTracks.Select(x => x.Path), PathEqualityComparer.Instance); var extraTracks = extraTracksOnDisk.Where(x => extraTrackPaths.Contains(x.Path)).ToList(); var allLocalTracks = localAlbumRelease.LocalTracks.Concat(extraTracks).DistinctBy(x => x.Path).ToList(); @@ -322,25 +325,33 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List x.currDistance) + .ThenBy(x => x.index) + .First(); + + localAlbumRelease.Distance = best.distance; + localAlbumRelease.AlbumRelease = best.release; + localAlbumRelease.ExistingTracks = best.extraTracks; + localAlbumRelease.TrackMapping = best.mapping; watch.Stop(); _logger.Debug($"Best release: {localAlbumRelease.AlbumRelease} Distance {localAlbumRelease.Distance.NormalizedDistance()} found in {watch.ElapsedMilliseconds}ms"); diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs index aae1a9e6f..5b55626b7 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs @@ -1,7 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO.Abstractions; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; @@ -100,19 +103,25 @@ public Tuple, List>> GetLocalTracks( downloadClientItemInfo = Parser.Parser.ParseAlbumTitle(downloadClientItem.Title); } - var i = 1; - foreach (var file in files) + var processedTracks = new ConcurrentBag<(int Index, LocalTrack Track)>(); + var processedDecisions = new ConcurrentBag<(int Index, ImportDecision Decision)>(); + var progress = 0; + var maxParallelism = Math.Max(1, Environment.ProcessorCount); + var filesWithIndex = files.Select((file, index) => new { file, index }).ToList(); + + Parallel.ForEach(filesWithIndex, new ParallelOptions { MaxDegreeOfParallelism = maxParallelism }, item => { - _logger.ProgressInfo($"Reading file {i++}/{files.Count}"); + var current = Interlocked.Increment(ref progress); + _logger.ProgressInfo($"Reading file {current}/{files.Count}"); var localTrack = new LocalTrack { DownloadClientAlbumInfo = downloadClientItemInfo, FolderAlbumInfo = folderInfo, - Path = file.FullName, - Size = file.Length, - Modified = file.LastWriteTimeUtc, - FileTrackInfo = _audioTagService.ReadTags(file.FullName), + Path = item.file.FullName, + Size = item.file.Length, + Modified = item.file.LastWriteTimeUtc, + FileTrackInfo = _audioTagService.ReadTags(item.file.FullName), AdditionalFile = false }; @@ -120,19 +129,22 @@ public Tuple, List>> GetLocalTracks( { // TODO fix otherfiles? _augmentingService.Augment(localTrack, true); - localTracks.Add(localTrack); + processedTracks.Add((item.index, localTrack)); } catch (AugmentingFailedException) { - decisions.Add(new ImportDecision(localTrack, new Rejection("Unable to parse file"))); + processedDecisions.Add((item.index, new ImportDecision(localTrack, new Rejection("Unable to parse file")))); } catch (Exception e) { _logger.Error(e, "Couldn't import file. {0}", localTrack.Path); - decisions.Add(new ImportDecision(localTrack, new Rejection("Unexpected error processing file"))); + processedDecisions.Add((item.index, new ImportDecision(localTrack, new Rejection("Unexpected error processing file")))); } - } + }); + + localTracks.AddRange(processedTracks.OrderBy(x => x.Index).Select(x => x.Track)); + decisions.AddRange(processedDecisions.OrderBy(x => x.Index).Select(x => x.Decision)); _logger.Debug($"Tags parsed for {files.Count} files in {watch.ElapsedMilliseconds}ms"); From 6f97d739fe0f1d8351dc75dde189e7161cec2873 Mon Sep 17 00:00:00 2001 From: redeyeredline Date: Wed, 25 Mar 2026 14:38:27 -0700 Subject: [PATCH 2/6] Create docker-publish.yml --- .github/workflows/docker-publish.yml | 98 ++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..e96f58bc6 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,98 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + schedule: + - cron: '19 21 * * *' + push: + branches: [ "develop" ] + # Publish semver tags as releases. + tags: [ 'v*.*.*' ] + pull_request: + branches: [ "develop" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0 + with: + cosign-release: 'v2.2.4' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} From 01d26cf02df6d56e0bc740b151d7a522e9d7218a Mon Sep 17 00:00:00 2001 From: Sean Parsons Date: Wed, 25 Mar 2026 21:42:47 +0000 Subject: [PATCH 3/6] Add GHCR Docker workflow for new-multithreaded-import; multithread scan changes - Dockerfile.multithread: build from repo root for CI overlay on linuxserver/lidarr:nightly - docker-ghcr-multithread.yml: push to ghcr.io on branch push + workflow_dispatch - MediaFiles / RefreshArtist: fork behavior for multithreaded import Made-with: Cursor --- .github/workflows/docker-ghcr-multithread.yml | 43 +++++++++++++++++++ Dockerfile.multithread | 19 ++++++++ .../MediaFiles/DiskScanService.cs | 31 +++++++------ .../Music/Services/RefreshArtistService.cs | 7 +++ 4 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/docker-ghcr-multithread.yml create mode 100644 Dockerfile.multithread diff --git a/.github/workflows/docker-ghcr-multithread.yml b/.github/workflows/docker-ghcr-multithread.yml new file mode 100644 index 000000000..a7b6dd3fd --- /dev/null +++ b/.github/workflows/docker-ghcr-multithread.yml @@ -0,0 +1,43 @@ +# Build custom Lidarr overlay image and push to GHCR when this branch updates. +name: Docker (multithread) → GHCR + +on: + push: + branches: + - new-multithreaded-import + workflow_dispatch: + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/lidarr + +jobs: + build-push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.multithread + push: true + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:new-multithreaded-import + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:git-${{ github.sha }} diff --git a/Dockerfile.multithread b/Dockerfile.multithread new file mode 100644 index 000000000..375a5aeaf --- /dev/null +++ b/Dockerfile.multithread @@ -0,0 +1,19 @@ +# CI / context = repo root (lidarr-src). Local builds from parent folder use ../Dockerfile instead. +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS builder +WORKDIR /src + +COPY . ./ + +RUN dotnet publish src/NzbDrone.Console/Lidarr.Console.csproj \ + -c Release \ + -f net8.0 \ + -r linux-musl-x64 \ + --self-contained true \ + -p:RunAnalyzers=false \ + -p:EnforceCodeStyleInBuild=false \ + -p:TreatWarningsAsErrors=false \ + -o /out + +FROM ghcr.io/linuxserver/lidarr:nightly + +COPY --from=builder /out/ /app/lidarr/bin/ diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index a77432497..3904b6a80 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using System.Linq; using System.Text.RegularExpressions; +using System.Threading.Tasks; using NLog; using NzbDrone.Common; using NzbDrone.Common.Disk; @@ -84,23 +85,19 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil } var mediaFileList = new List(); + var mediaFileListLock = new object(); - var musicFilesStopwatch = Stopwatch.StartNew(); - + // Validate folders first (early exit on error like original behaviour) + var foldersToScan = new List(); foreach (var folder in folders) { - // We could be scanning a root folder or a subset of a root folder. If it's a subset, - // check if the root folder exists before cleaning. var rootFolder = _rootFolderService.GetBestRootFolder(folder); - if (rootFolder == null) { _logger.Error("Not scanning {0}, it's not a subdirectory of a defined root folder", folder); return; } - var folderExists = _diskProvider.FolderExists(folder); - if (!folderExists) { if (!_diskProvider.FolderExists(rootFolder.Path)) @@ -110,7 +107,6 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil skippedArtists.ForEach(x => _eventAggregator.PublishEvent(new ArtistScanSkippedEvent(x, ArtistScanSkippedReason.RootFolderDoesNotExist))); return; } - if (_diskProvider.FolderEmpty(rootFolder.Path)) { _logger.Warn("Artists' root folder ({0}) is empty.", rootFolder.Path); @@ -119,28 +115,35 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil return; } } - if (!folderExists) { _logger.Debug("Specified scan folder ({0}) doesn't exist.", folder); - CleanMediaFiles(folder, new List()); continue; } + foldersToScan.Add(folder); + } + var musicFilesStopwatch = Stopwatch.StartNew(); + + Parallel.ForEach(foldersToScan, folder => + { _logger.ProgressInfo("Scanning {0}", folder); - var files = FilterFiles(folder, GetAudioFiles(folder)); + var files = FilterFiles(folder, GetAudioFiles(folder)).ToList(); if (!files.Any()) { _logger.Warn("Scan folder {0} is empty.", folder); - continue; + return; } CleanMediaFiles(folder, files.Select(x => x.FullName).ToList()); - mediaFileList.AddRange(files); - } + lock (mediaFileListLock) + { + mediaFileList.AddRange(files); + } + }); var artists = _artistService.GetArtists(artistIds); diff --git a/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs b/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs index 4d4efdf6e..fe1734da2 100644 --- a/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs +++ b/src/NzbDrone.Core/Music/Services/RefreshArtistService.cs @@ -305,6 +305,13 @@ private void RescanArtists(List artists, bool isNew, CommandTrigger trig // badly organized / partly matched libraries folders = artists.Select(x => x.Path).ToList(); } + else if (trigger == CommandTrigger.Manual && artists.Any()) + { + // Manual refresh of specific artist(s): only scan those artists' folders, + // never the entire library (avoids 60k+ file scan when refreshing e.g. Various Artists). + folders = artists.Select(x => x.Path).ToList(); + _logger.Trace("Manual refresh: rescanning only {0} artist folder(s)", folders.Count); + } else if (rescanAfterRefresh == RescanAfterRefreshType.Never) { _logger.Trace("Skipping rescan. Reason: never rescan after refresh"); From a98d331b20caba71557ba9a52fa7dc8502635b08 Mon Sep 17 00:00:00 2001 From: Sean Parsons Date: Wed, 25 Mar 2026 22:51:49 +0000 Subject: [PATCH 4/6] Cap scan/import parallelism via LIDARR_MEDIA_IO_PARALLELISM Default max degree 2 to avoid IO storms on NFS/remote storage. Applies to folder scan, tag reads, and candidate scoring PLINQ. Document env var and Docker usage in MULTITHREAD_README.md. Made-with: Cursor --- MULTITHREAD_README.md | 57 +++++++++++++++++++ src/NzbDrone.Common/MediaImportParallelism.cs | 40 +++++++++++++ .../MediaFiles/DiskScanService.cs | 2 +- .../Identification/IdentificationService.cs | 2 +- .../TrackImport/ImportDecisionMaker.cs | 3 +- 5 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 MULTITHREAD_README.md create mode 100644 src/NzbDrone.Common/MediaImportParallelism.cs diff --git a/MULTITHREAD_README.md b/MULTITHREAD_README.md new file mode 100644 index 000000000..5be263da0 --- /dev/null +++ b/MULTITHREAD_README.md @@ -0,0 +1,57 @@ +# Multithreaded library scan / import (this fork) + +This branch adds a faster, **parallel** disk scan and import path. Upstream Lidarr does much of this work sequentially; this fork parallelizes folder scanning, tag reads, and release-candidate scoring. + +A **Dockerfile.multithread** in this repository builds a self-contained binary and overlays it on `ghcr.io/linuxserver/lidarr:nightly` (see CI or build from repo root per that file’s comments). A wrapper layout that keeps this tree in a `lidarr-src/` subdirectory can use the parent `Dockerfile` instead. + +## `LIDARR_MEDIA_IO_PARALLELISM` + +Parallel import work is **not** limited by Lidarr’s download bandwidth or rate settings (those apply to indexers/clients only). On **slow or remote storage** (especially NFS), too much concurrency can saturate IOPS and make the host feel stuck. This variable caps how many workers run at once for the fork’s parallel paths. + +| | | +| --- | --- | +| **Name** | `LIDARR_MEDIA_IO_PARALLELISM` | +| **Default** | `2` (used when the variable is unset, empty, or not a valid integer) | +| **Allowed** | Integers **1**–**64**; values below **1** fall back to the default | +| **Scope** | Process environment (read once at first use) | + +It applies to: + +- parallel **folder scans** when collecting audio files; +- parallel **tag / metadata reads** when building import decisions; +- parallel **candidate release scoring** during identification. + +**Docker:** fully supported. Set the variable on the container like any other env; the .NET process reads the container environment. + +### Docker Compose + +```yaml +services: + lidarr: + image: your-registry/lidarr-nightly-multithread:latest + environment: + - PUID=1000 + - PGID=1000 + - TZ=Etc/UTC + # Gentle on NFS / network mounts; omit for default (2) + - LIDARR_MEDIA_IO_PARALLELISM=1 +``` + +### `docker run` + +```bash +docker run -e LIDARR_MEDIA_IO_PARALLELISM=4 … your-image +``` + +### When to change it + +- **NFS, SMB, or sluggish disks:** try `1` or leave default `2`. +- **Library and app on fast local storage (e.g. same NAS app dataset, local SSD):** try `4`–`8` or higher (up to 64) and watch CPU, I/O, and responsiveness. + +### Implementation reference + +Logic and constant name: `src/NzbDrone.Common/MediaImportParallelism.cs`. + +## Relationship to upstream + +Behavior outside scan/import parallelism matches your chosen base (e.g. nightly image + overlaid build). For upstream docs and support channels, see [Lidarr](https://github.com/Lidarr/Lidarr) and the [Servarr wiki](https://wiki.servarr.com/lidarr). diff --git a/src/NzbDrone.Common/MediaImportParallelism.cs b/src/NzbDrone.Common/MediaImportParallelism.cs new file mode 100644 index 000000000..1100b32c1 --- /dev/null +++ b/src/NzbDrone.Common/MediaImportParallelism.cs @@ -0,0 +1,40 @@ +using System; + +namespace NzbDrone.Common +{ + /// + /// Limits parallel disk work during library scan/import (tag reads, folder scans, candidate scoring). + /// Unrelated to download bandwidth limits in Lidarr settings. + /// + public static class MediaImportParallelism + { + public const string EnvironmentVariableName = "LIDARR_MEDIA_IO_PARALLELISM"; + + private const int DefaultMaxDegree = 2; + private const int MinDegree = 1; + private const int MaxDegreeCap = 64; + + private static readonly Lazy MaxDegreeLazy = new Lazy(ReadMaxDegree); + + /// + /// Maximum concurrent workers for scan/import parallelism. Default 2; override with LIDARR_MEDIA_IO_PARALLELISM (1–64). + /// + public static int MaxDegreeOfParallelism => MaxDegreeLazy.Value; + + private static int ReadMaxDegree() + { + var raw = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrWhiteSpace(raw) || !int.TryParse(raw.Trim(), out var parsed)) + { + return DefaultMaxDegree; + } + + if (parsed < MinDegree) + { + return DefaultMaxDegree; + } + + return Math.Min(parsed, MaxDegreeCap); + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index 3904b6a80..b1c0f5b71 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -126,7 +126,7 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil var musicFilesStopwatch = Stopwatch.StartNew(); - Parallel.ForEach(foldersToScan, folder => + Parallel.ForEach(foldersToScan, new ParallelOptions { MaxDegreeOfParallelism = MediaImportParallelism.MaxDegreeOfParallelism }, folder => { _logger.ProgressInfo("Scanning {0}", folder); diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index 417eda90c..e752fad1a 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -305,7 +305,7 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List x.Path))); - var maxParallelism = Math.Max(1, Environment.ProcessorCount); + var maxParallelism = MediaImportParallelism.MaxDegreeOfParallelism; var scoredCandidates = candidateReleases .Select((candidateRelease, index) => new { candidateRelease, index }) .AsParallel() diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs index 5b55626b7..3b34824b3 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using NLog; +using NzbDrone.Common; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.DecisionEngine; @@ -106,7 +107,7 @@ public Tuple, List>> GetLocalTracks( var processedTracks = new ConcurrentBag<(int Index, LocalTrack Track)>(); var processedDecisions = new ConcurrentBag<(int Index, ImportDecision Decision)>(); var progress = 0; - var maxParallelism = Math.Max(1, Environment.ProcessorCount); + var maxParallelism = MediaImportParallelism.MaxDegreeOfParallelism; var filesWithIndex = files.Select((file, index) => new { file, index }).ToList(); Parallel.ForEach(filesWithIndex, new ParallelOptions { MaxDegreeOfParallelism = maxParallelism }, item => From c0aec91d504c0a20c08fc9f2e29f91a9c743bf2b Mon Sep 17 00:00:00 2001 From: Sean Parsons Date: Fri, 27 Mar 2026 21:39:40 +0000 Subject: [PATCH 5/6] Fix media parallelism defaults and observability - Unset/invalid/0: use Environment.ProcessorCount (restore pre-cap behavior) - Drop Lazy cache; re-read env when parallel work runs - Log effective MaxDegree once per process on first disk scan - Update MULTITHREAD_README for NFS opt-in vs default Made-with: Cursor --- MULTITHREAD_README.md | 10 ++++++---- src/NzbDrone.Common/MediaImportParallelism.cs | 19 +++++++++---------- .../MediaFiles/DiskScanService.cs | 13 +++++++++++++ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/MULTITHREAD_README.md b/MULTITHREAD_README.md index 5be263da0..a88da9fc1 100644 --- a/MULTITHREAD_README.md +++ b/MULTITHREAD_README.md @@ -11,9 +11,9 @@ Parallel import work is **not** limited by Lidarr’s download bandwidth or rate | | | | --- | --- | | **Name** | `LIDARR_MEDIA_IO_PARALLELISM` | -| **Default** | `2` (used when the variable is unset, empty, or not a valid integer) | -| **Allowed** | Integers **1**–**64**; values below **1** fall back to the default | -| **Scope** | Process environment (read once at first use) | +| **Default** | **Unset / empty / invalid / `0`:** same as **`Environment.ProcessorCount`** (original fork behavior). For **NFS or slow storage**, set **`1`** or **`2`** explicitly. | +| **Allowed** | **`1`–`64`:** use that cap. **`0`:** treat as unset (processor count). | +| **Scope** | Process environment (re-read on each scan/import parallel section) | It applies to: @@ -23,6 +23,8 @@ It applies to: **Docker:** fully supported. Set the variable on the container like any other env; the .NET process reads the container environment. +On the first disk scan, Lidarr logs a line like `Media import parallelism: MaxDegreeOfParallelism=…` so you can confirm the value it sees (useful if compose/env typos leave the variable unset). + ### Docker Compose ```yaml @@ -33,7 +35,7 @@ services: - PUID=1000 - PGID=1000 - TZ=Etc/UTC - # Gentle on NFS / network mounts; omit for default (2) + # Gentle on NFS / network mounts (omit var for processor-count default) - LIDARR_MEDIA_IO_PARALLELISM=1 ``` diff --git a/src/NzbDrone.Common/MediaImportParallelism.cs b/src/NzbDrone.Common/MediaImportParallelism.cs index 1100b32c1..057d0f164 100644 --- a/src/NzbDrone.Common/MediaImportParallelism.cs +++ b/src/NzbDrone.Common/MediaImportParallelism.cs @@ -3,35 +3,34 @@ namespace NzbDrone.Common { /// - /// Limits parallel disk work during library scan/import (tag reads, folder scans, candidate scoring). + /// Caps parallel disk work during library scan/import (tag reads, folder scans, candidate scoring), optional via env. /// Unrelated to download bandwidth limits in Lidarr settings. /// public static class MediaImportParallelism { public const string EnvironmentVariableName = "LIDARR_MEDIA_IO_PARALLELISM"; - private const int DefaultMaxDegree = 2; - private const int MinDegree = 1; private const int MaxDegreeCap = 64; - private static readonly Lazy MaxDegreeLazy = new Lazy(ReadMaxDegree); - /// - /// Maximum concurrent workers for scan/import parallelism. Default 2; override with LIDARR_MEDIA_IO_PARALLELISM (1–64). + /// Maximum concurrent workers for scan/import parallelism. + /// If LIDARR_MEDIA_IO_PARALLELISM is unset, empty, invalid, 0, or negative: uses (matches pre-cap fork behavior). + /// Otherwise uses the set value clamped to 1–64. + /// Re-reads the environment each call so container/env changes are visible without restart (same process still needs a new read on next scan). /// - public static int MaxDegreeOfParallelism => MaxDegreeLazy.Value; + public static int MaxDegreeOfParallelism => ReadMaxDegree(); private static int ReadMaxDegree() { var raw = Environment.GetEnvironmentVariable(EnvironmentVariableName); if (string.IsNullOrWhiteSpace(raw) || !int.TryParse(raw.Trim(), out var parsed)) { - return DefaultMaxDegree; + return Math.Max(1, Environment.ProcessorCount); } - if (parsed < MinDegree) + if (parsed <= 0) { - return DefaultMaxDegree; + return Math.Max(1, Environment.ProcessorCount); } return Math.Min(parsed, MaxDegreeCap); diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index b1c0f5b71..928518757 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -5,6 +5,7 @@ using System.IO.Abstractions; using System.Linq; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using NLog; using NzbDrone.Common; @@ -48,6 +49,7 @@ public class DiskScanService : private readonly IRootFolderService _rootFolderService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; + private static int _mediaParallelismLogged; public DiskScanService(IConfigService configService, IDiskProvider diskProvider, @@ -126,6 +128,17 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil var musicFilesStopwatch = Stopwatch.StartNew(); + if (Interlocked.CompareExchange(ref _mediaParallelismLogged, 1, 0) == 0) + { + var envRaw = Environment.GetEnvironmentVariable(MediaImportParallelism.EnvironmentVariableName); + _logger.Info( + "Media import parallelism: MaxDegreeOfParallelism={0} ({1}={2}). Set 1–64 to cap IO; unset or 0 uses processor count ({3}).", + MediaImportParallelism.MaxDegreeOfParallelism, + MediaImportParallelism.EnvironmentVariableName, + string.IsNullOrEmpty(envRaw) ? "(unset)" : envRaw, + Environment.ProcessorCount); + } + Parallel.ForEach(foldersToScan, new ParallelOptions { MaxDegreeOfParallelism = MediaImportParallelism.MaxDegreeOfParallelism }, folder => { _logger.ProgressInfo("Scanning {0}", folder); From 83298dabf39029a115f00f5831c93780101b2776 Mon Sep 17 00:00:00 2001 From: Sean Parsons Date: Fri, 27 Mar 2026 21:56:55 +0000 Subject: [PATCH 6/6] Restore uncapped Parallel.ForEach when env cap unset (TPL -1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Omit/invalid/<=0 LIDARR_MEDIA_IO_PARALLELISM: default ParallelOptions for folder scan + tag reads (matches original multithread fork). - PLINQ candidate scoring: ProcessorCount when uncapped; same 1–64 cap when set. - Fixes throughput loss from equating 16 cores with TPL default concurrency. - Log line shows TPL default vs cap and PLINQ degree; update MULTITHREAD_README. Made-with: Cursor --- MULTITHREAD_README.md | 38 +++++-------- src/NzbDrone.Common/MediaImportParallelism.cs | 56 +++++++++++++++---- .../MediaFiles/DiskScanService.cs | 11 ++-- .../Identification/IdentificationService.cs | 3 +- .../TrackImport/ImportDecisionMaker.cs | 3 +- 5 files changed, 67 insertions(+), 44 deletions(-) diff --git a/MULTITHREAD_README.md b/MULTITHREAD_README.md index a88da9fc1..7796b69f8 100644 --- a/MULTITHREAD_README.md +++ b/MULTITHREAD_README.md @@ -4,26 +4,20 @@ This branch adds a faster, **parallel** disk scan and import path. Upstream Lida A **Dockerfile.multithread** in this repository builds a self-contained binary and overlays it on `ghcr.io/linuxserver/lidarr:nightly` (see CI or build from repo root per that file’s comments). A wrapper layout that keeps this tree in a `lidarr-src/` subdirectory can use the parent `Dockerfile` instead. -## `LIDARR_MEDIA_IO_PARALLELISM` +## `LIDARR_MEDIA_IO_PARALLELISM` (optional IO cap) -Parallel import work is **not** limited by Lidarr’s download bandwidth or rate settings (those apply to indexers/clients only). On **slow or remote storage** (especially NFS), too much concurrency can saturate IOPS and make the host feel stuck. This variable caps how many workers run at once for the fork’s parallel paths. +Parallel import work is **not** limited by Lidarr’s download bandwidth or rate settings (those apply to indexers/clients only). On **slow or remote storage** (especially NFS), the default **uncapped** parallelism can saturate IOPS. Set this variable only when you need to **limit** concurrency. | | | | --- | --- | | **Name** | `LIDARR_MEDIA_IO_PARALLELISM` | -| **Default** | **Unset / empty / invalid / `0`:** same as **`Environment.ProcessorCount`** (original fork behavior). For **NFS or slow storage**, set **`1`** or **`2`** explicitly. | -| **Allowed** | **`1`–`64`:** use that cap. **`0`:** treat as unset (processor count). | -| **Scope** | Process environment (re-read on each scan/import parallel section) | +| **Omit / empty / invalid / ≤0** | **Original fork behavior:** `Parallel.ForEach` uses the **TPL default** (`MaxDegreeOfParallelism = -1`), which can use **more** concurrent workers than `ProcessorCount` on I/O-heavy work (this is why setting `16` on a 16-core box could feel *slower* than before). **PLINQ** still uses **`ProcessorCount`** (TagLib / candidate scoring cannot use `-1`). | +| **1–64** | Hard cap on **both** `Parallel.ForEach` loops **and** PLINQ degree (same number). Use **`1`–`2`** on NFS if the host stalls. | +| **Scope** | Environment is read when each parallel section runs | -It applies to: +**Docker:** set on the container like any other env variable. -- parallel **folder scans** when collecting audio files; -- parallel **tag / metadata reads** when building import decisions; -- parallel **candidate release scoring** during identification. - -**Docker:** fully supported. Set the variable on the container like any other env; the .NET process reads the container environment. - -On the first disk scan, Lidarr logs a line like `Media import parallelism: MaxDegreeOfParallelism=…` so you can confirm the value it sees (useful if compose/env typos leave the variable unset). +On the first disk scan, Lidarr logs `Media import parallelism:` with **TPL default (-1, uncapped)** or your numeric cap, plus PLINQ degree and host `ProcessorCount`. ### Docker Compose @@ -35,24 +29,18 @@ services: - PUID=1000 - PGID=1000 - TZ=Etc/UTC - # Gentle on NFS / network mounts (omit var for processor-count default) - - LIDARR_MEDIA_IO_PARALLELISM=1 + # Omit LIDARR_MEDIA_IO_PARALLELISM on fast local storage (max throughput). + # - LIDARR_MEDIA_IO_PARALLELISM=2 # NFS / slow disk — cap concurrent work ``` -### `docker run` +### When to set it -```bash -docker run -e LIDARR_MEDIA_IO_PARALLELISM=4 … your-image -``` - -### When to change it - -- **NFS, SMB, or sluggish disks:** try `1` or leave default `2`. -- **Library and app on fast local storage (e.g. same NAS app dataset, local SSD):** try `4`–`8` or higher (up to 64) and watch CPU, I/O, and responsiveness. +- **Fast local RAID / SSD:** **omit** the variable (matches the first multithread fork). +- **NFS or network filesystem:** start with **`2`** (or **`1`**) if scans overwhelm the host. ### Implementation reference -Logic and constant name: `src/NzbDrone.Common/MediaImportParallelism.cs`. +`src/NzbDrone.Common/MediaImportParallelism.cs`. ## Relationship to upstream diff --git a/src/NzbDrone.Common/MediaImportParallelism.cs b/src/NzbDrone.Common/MediaImportParallelism.cs index 057d0f164..4efb33174 100644 --- a/src/NzbDrone.Common/MediaImportParallelism.cs +++ b/src/NzbDrone.Common/MediaImportParallelism.cs @@ -1,9 +1,10 @@ using System; +using System.Threading.Tasks; namespace NzbDrone.Common { /// - /// Caps parallel disk work during library scan/import (tag reads, folder scans, candidate scoring), optional via env. + /// Optional cap on parallel scan/import work via LIDARR_MEDIA_IO_PARALLELISM. /// Unrelated to download bandwidth limits in Lidarr settings. /// public static class MediaImportParallelism @@ -13,27 +14,60 @@ public static class MediaImportParallelism private const int MaxDegreeCap = 64; /// - /// Maximum concurrent workers for scan/import parallelism. - /// If LIDARR_MEDIA_IO_PARALLELISM is unset, empty, invalid, 0, or negative: uses (matches pre-cap fork behavior). - /// Otherwise uses the set value clamped to 1–64. - /// Re-reads the environment each call so container/env changes are visible without restart (same process still needs a new read on next scan). + /// Unset / empty / invalid / ≤0: Original fork behavior — no explicit cap on + /// (TPL default -1, scheduler chooses concurrency; often higher than core count for I/O). + /// 1–64: Cap loops to that many concurrent workers (use on NFS / slow storage). /// - public static int MaxDegreeOfParallelism => ReadMaxDegree(); - - private static int ReadMaxDegree() + public static ParallelOptions GetParallelForEachOptions() { + if (!TryParseUserCap(out var cap)) + { + return new ParallelOptions(); + } + + return new ParallelOptions { MaxDegreeOfParallelism = cap }; + } + + /// + /// PLINQ WithDegreeOfParallelism must be ≥ 1. + /// Uncapped: (same as pre-env ImportDecisionMaker / IdentificationService). + /// Capped: user value (1–64). + /// + public static int PlinqMaxDegreeOfParallelism + { + get + { + if (!TryParseUserCap(out var cap)) + { + return Math.Max(1, Environment.ProcessorCount); + } + + return cap; + } + } + + /// + /// For logging: -1 means TPL default (uncapped loops); otherwise the explicit cap. + /// + public static int EffectiveParallelForEachDegreeForLog => + TryParseUserCap(out var cap) ? cap : -1; + + private static bool TryParseUserCap(out int cap) + { + cap = 0; var raw = Environment.GetEnvironmentVariable(EnvironmentVariableName); if (string.IsNullOrWhiteSpace(raw) || !int.TryParse(raw.Trim(), out var parsed)) { - return Math.Max(1, Environment.ProcessorCount); + return false; } if (parsed <= 0) { - return Math.Max(1, Environment.ProcessorCount); + return false; } - return Math.Min(parsed, MaxDegreeCap); + cap = Math.Min(parsed, MaxDegreeCap); + return true; } } } diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index 928518757..184f8f635 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -131,15 +131,18 @@ public void Scan(List folders = null, FilterFilesType filter = FilterFil if (Interlocked.CompareExchange(ref _mediaParallelismLogged, 1, 0) == 0) { var envRaw = Environment.GetEnvironmentVariable(MediaImportParallelism.EnvironmentVariableName); + var loopDeg = MediaImportParallelism.EffectiveParallelForEachDegreeForLog; + var loopDesc = loopDeg < 0 ? "TPL default (-1, uncapped)" : loopDeg.ToString(); _logger.Info( - "Media import parallelism: MaxDegreeOfParallelism={0} ({1}={2}). Set 1–64 to cap IO; unset or 0 uses processor count ({3}).", - MediaImportParallelism.MaxDegreeOfParallelism, + "Media import parallelism: Parallel.ForEach MaxDegreeOfParallelism={0} ({1}; PLINQ degree {2}). Set {3}=1–64 to cap; omit or ≤0 restores pre-cap fork (uncapped loops). Host ProcessorCount={4}.", + loopDesc, + string.IsNullOrEmpty(envRaw) ? $"{MediaImportParallelism.EnvironmentVariableName}=(unset)" : $"{MediaImportParallelism.EnvironmentVariableName}={envRaw}", + MediaImportParallelism.PlinqMaxDegreeOfParallelism, MediaImportParallelism.EnvironmentVariableName, - string.IsNullOrEmpty(envRaw) ? "(unset)" : envRaw, Environment.ProcessorCount); } - Parallel.ForEach(foldersToScan, new ParallelOptions { MaxDegreeOfParallelism = MediaImportParallelism.MaxDegreeOfParallelism }, folder => + Parallel.ForEach(foldersToScan, MediaImportParallelism.GetParallelForEachOptions(), folder => { _logger.ProgressInfo("Scanning {0}", folder); diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs index e752fad1a..15f7da24b 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/Identification/IdentificationService.cs @@ -305,11 +305,10 @@ private void GetBestRelease(LocalAlbumRelease localAlbumRelease, List x.Path))); - var maxParallelism = MediaImportParallelism.MaxDegreeOfParallelism; var scoredCandidates = candidateReleases .Select((candidateRelease, index) => new { candidateRelease, index }) .AsParallel() - .WithDegreeOfParallelism(maxParallelism) + .WithDegreeOfParallelism(MediaImportParallelism.PlinqMaxDegreeOfParallelism) .Select(item => { var release = item.candidateRelease.AlbumRelease; diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs index 3b34824b3..0c903852e 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportDecisionMaker.cs @@ -107,10 +107,9 @@ public Tuple, List>> GetLocalTracks( var processedTracks = new ConcurrentBag<(int Index, LocalTrack Track)>(); var processedDecisions = new ConcurrentBag<(int Index, ImportDecision Decision)>(); var progress = 0; - var maxParallelism = MediaImportParallelism.MaxDegreeOfParallelism; var filesWithIndex = files.Select((file, index) => new { file, index }).ToList(); - Parallel.ForEach(filesWithIndex, new ParallelOptions { MaxDegreeOfParallelism = maxParallelism }, item => + Parallel.ForEach(filesWithIndex, MediaImportParallelism.GetParallelForEachOptions(), item => { var current = Interlocked.Increment(ref progress); _logger.ProgressInfo($"Reading file {current}/{files.Count}");