From c280b9afb049f15fc81b04db0a1abdec7ca78657 Mon Sep 17 00:00:00 2001 From: Ricardo Amaral Date: Tue, 13 Jan 2026 18:15:51 +0000 Subject: [PATCH] New: Merge import list tags across multiple lists Previously, import list tag handling had two issues: 1. Movies already in the library were skipped entirely during list sync, so tags configured on import lists would never be applied to them. 2. When a movie appeared in multiple import lists, only tags from one list would be applied due to premature deduplication in the fetch service. This change ensures that tags from all import lists are properly merged, both for existing movies in the library and for new movies being imported. For existing movies: - Tags from all import lists referencing the movie are collected - New tags are merged with existing movie tags (preserving current tags) - A single batch update is performed for efficiency For new movies appearing in multiple lists: - The movie is added once (using settings from the first list, like usual) - Tags from all lists are merged before the movie is added - Removed premature `DistinctBy` in `FetchAndParseImportListService` that was discarding duplicate entries across lists Performance optimizations: - Changed `dbMovies` from List to `HashSet` for O(1) lookups - Changed moviesToAdd from List to `Dictionary` for O(1) lookups - Tag service is queried once and reused for all logging - Added `FindByTmdbId(List)` to `IMovieService` for batch lookups --- .../FetchAndParseImportListService.cs | 7 +- .../ImportLists/ImportListSyncService.cs | 136 ++++++++++++------ src/NzbDrone.Core/Movies/MovieRepository.cs | 2 +- src/NzbDrone.Core/Movies/MovieService.cs | 6 + 4 files changed, 104 insertions(+), 47 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs index d47aeb71f7..13bb20d6d0 100644 --- a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -101,11 +101,8 @@ public ImportListFetchResult Fetch() if (!importListReports.AnyFailure) { - var alreadyMapped = result.Movies.Where(x => importListReports.Movies.Any(r => r.TmdbId == x.TmdbId)); - var listMovies = MapMovieReports(importListReports.Movies.Where(x => result.Movies.All(r => r.TmdbId != x.TmdbId))).Where(x => x.TmdbId > 0).ToList(); + var listMovies = MapMovieReports(importListReports.Movies).Where(x => x.TmdbId > 0).ToList(); - listMovies.AddRange(alreadyMapped); - listMovies = listMovies.DistinctBy(x => x.TmdbId).ToList(); listMovies.ForEach(m => m.ListId = importList.Definition.Id); result.Movies.AddRange(listMovies); @@ -129,8 +126,6 @@ public ImportListFetchResult Fetch() Task.WaitAll(taskList.ToArray()); - result.Movies = result.Movies.DistinctBy(r => new { r.TmdbId, r.ImdbId, r.Title }).ToList(); - _logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists); return result; diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs index b96e07a0fb..058d69b0fb 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -8,6 +8,7 @@ using NzbDrone.Core.ImportLists.ImportListMovies; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Movies; +using NzbDrone.Core.Tags; namespace NzbDrone.Core.ImportLists { @@ -21,6 +22,7 @@ public class ImportListSyncService : IExecute private readonly IConfigService _configService; private readonly IImportListExclusionService _listExclusionService; private readonly IImportListMovieService _listMovieService; + private readonly ITagService _tagService; public ImportListSyncService(IImportListFactory importListFactory, IFetchAndParseImportList listFetcherAndParser, @@ -29,6 +31,7 @@ public ImportListSyncService(IImportListFactory importListFactory, IConfigService configService, IImportListExclusionService listExclusionService, IImportListMovieService listMovieService, + ITagService tagService, Logger logger) { _importListFactory = importListFactory; @@ -37,6 +40,7 @@ public ImportListSyncService(IImportListFactory importListFactory, _addMovieService = addMovieService; _listExclusionService = listExclusionService; _listMovieService = listMovieService; + _tagService = tagService; _logger = logger; _configService = configService; } @@ -74,7 +78,7 @@ private void SyncList(ImportListDefinition definition) ProcessListItems(listItemsResult); } - private void ProcessMovieReport(ImportListDefinition importList, ImportListMovie report, List listExclusions, List dbMovies, List moviesToAdd) + private void ProcessMovieReport(ImportListDefinition importList, ImportListMovie report, List listExclusions, HashSet dbMovies, Dictionary moviesToAdd, Dictionary> existingMovieTagUpdates, Dictionary allTags) { if (report.TmdbId == 0 || !importList.EnableAuto) { @@ -84,7 +88,20 @@ private void ProcessMovieReport(ImportListDefinition importList, ImportListMovie // Check to see if movie in DB if (dbMovies.Contains(report.TmdbId)) { - _logger.Debug("{0} [{1}] Rejected, Movie Exists in DB", report.TmdbId, report.Title); + _logger.Debug("{0} [{1}] Movie Exists in DB, checking tags", report.TmdbId, report.Title); + + // Collect tags to add to existing movies + if (importList.Tags.Any()) + { + if (!existingMovieTagUpdates.TryGetValue(report.TmdbId, out var tagsToAdd)) + { + tagsToAdd = new HashSet(); + existingMovieTagUpdates[report.TmdbId] = tagsToAdd; + } + + tagsToAdd.UnionWith(importList.Tags); + } + return; } @@ -97,54 +114,59 @@ private void ProcessMovieReport(ImportListDefinition importList, ImportListMovie return; } - // Append Artist if not already in DB or already on add list - if (moviesToAdd.All(s => s.TmdbId != report.TmdbId)) + // Check if movie is already on the add list (from another import list) + if (moviesToAdd.TryGetValue(report.TmdbId, out var existingMovie)) { - var monitorType = importList.Monitor; - - moviesToAdd.Add(new Movie + // Merge tags from this list into the existing movie + if (importList.Tags.Any()) { - Monitored = monitorType != MonitorTypes.None, - RootFolderPath = importList.RootFolderPath, - QualityProfileId = importList.QualityProfileId, - MinimumAvailability = importList.MinimumAvailability, - Tags = importList.Tags, - TmdbId = report.TmdbId, - Title = report.Title, - Year = report.Year, - ImdbId = report.ImdbId, - AddOptions = new AddMovieOptions + var newTags = importList.Tags.Except(existingMovie.Tags).ToList(); + + if (newTags.Any()) { - SearchForMovie = monitorType != MonitorTypes.None && importList.SearchOnAdd, - Monitor = monitorType, - AddMethod = AddMovieMethod.List + var tagNames = newTags.Select(id => allTags.TryGetValue(id, out var name) ? name : id.ToString()); + _logger.Debug("{0} [{1}] Merging {2} tags from list {3}: [{4}]", report.TmdbId, report.Title, newTags.Count, importList.Name, string.Join(", ", tagNames)); + existingMovie.Tags = existingMovie.Tags.Union(importList.Tags).ToHashSet(); } - }); + } + + return; } + + var monitorType = importList.Monitor; + + var initialTagNames = importList.Tags.Select(id => allTags.TryGetValue(id, out var name) ? name : id.ToString()); + _logger.Debug("{0} [{1}] Adding to import queue from list {2} with tags: [{3}]", report.TmdbId, report.Title, importList.Name, string.Join(", ", initialTagNames)); + + moviesToAdd[report.TmdbId] = new Movie + { + Monitored = monitorType != MonitorTypes.None, + RootFolderPath = importList.RootFolderPath, + QualityProfileId = importList.QualityProfileId, + MinimumAvailability = importList.MinimumAvailability, + Tags = importList.Tags, + TmdbId = report.TmdbId, + Title = report.Title, + Year = report.Year, + ImdbId = report.ImdbId, + AddOptions = new AddMovieOptions + { + SearchForMovie = monitorType != MonitorTypes.None && importList.SearchOnAdd, + Monitor = monitorType, + AddMethod = AddMovieMethod.List + } + }; } private void ProcessListItems(ImportListFetchResult listFetchResult) { - listFetchResult.Movies = listFetchResult.Movies.DistinctBy(x => - { - if (x.TmdbId != 0) - { - return x.TmdbId.ToString(); - } - - if (x.ImdbId.IsNotNullOrWhiteSpace()) - { - return x.ImdbId; - } - - return x.Title; - }).ToList(); - var listedMovies = listFetchResult.Movies.ToList(); var importExclusions = _listExclusionService.All(); - var dbMovies = _movieService.AllMovieTmdbIds(); - var moviesToAdd = new List(); + var dbMovies = _movieService.AllMovieTmdbIds().ToHashSet(); + var moviesToAdd = new Dictionary(); + var existingMovieTagUpdates = new Dictionary>(); + var allTags = _tagService.All().ToDictionary(t => t.Id, t => t.Label); var groupedMovies = listedMovies.GroupBy(x => x.ListId); @@ -156,7 +178,7 @@ private void ProcessListItems(ImportListFetchResult listFetchResult) { if (movie.TmdbId != 0) { - ProcessMovieReport(importList, movie, importExclusions, dbMovies, moviesToAdd); + ProcessMovieReport(importList, movie, importExclusions, dbMovies, moviesToAdd, existingMovieTagUpdates, allTags); } } } @@ -164,7 +186,41 @@ private void ProcessListItems(ImportListFetchResult listFetchResult) if (moviesToAdd.Any()) { _logger.ProgressInfo("Adding {0} movies from your auto enabled lists to library", moviesToAdd.Count); - _addMovieService.AddMovies(moviesToAdd, true); + _addMovieService.AddMovies(moviesToAdd.Values.ToList(), true); + } + + if (existingMovieTagUpdates.Any()) + { + UpdateTagsForExistingMovies(existingMovieTagUpdates, allTags); + } + } + + private void UpdateTagsForExistingMovies(Dictionary> existingMovieTagUpdates, Dictionary allTags) + { + var moviesToUpdate = new List(); + var existingMovies = _movieService.FindByTmdbId(existingMovieTagUpdates.Keys.ToList()); + + foreach (var movie in existingMovies) + { + if (existingMovieTagUpdates.TryGetValue(movie.TmdbId, out var tagsFromLists)) + { + var newTags = tagsFromLists.Except(movie.Tags).ToList(); + + if (newTags.Any()) + { + movie.Tags = movie.Tags.Union(tagsFromLists).ToHashSet(); + moviesToUpdate.Add(movie); + + var tagNames = newTags.Select(id => allTags.TryGetValue(id, out var name) ? name : id.ToString()); + _logger.Debug("Adding {0} tags to {1}: {2}", newTags.Count, movie.Title, string.Join(", ", tagNames)); + } + } + } + + if (moviesToUpdate.Any()) + { + _logger.ProgressInfo("Updating tags for {0} movies from import lists", moviesToUpdate.Count); + _movieService.UpdateMovie(moviesToUpdate, true); } } diff --git a/src/NzbDrone.Core/Movies/MovieRepository.cs b/src/NzbDrone.Core/Movies/MovieRepository.cs index ed235a30df..ec9696d7c5 100644 --- a/src/NzbDrone.Core/Movies/MovieRepository.cs +++ b/src/NzbDrone.Core/Movies/MovieRepository.cs @@ -220,7 +220,7 @@ public Movie FindByTmdbId(int tmdbid) public List FindByTmdbId(List tmdbids) { - return Query(x => tmdbids.Contains(x.TmdbId)); + return Query(x => tmdbids.Contains(x.MovieMetadata.Value.TmdbId)); } public List GetMoviesByFileId(int fileId) diff --git a/src/NzbDrone.Core/Movies/MovieService.cs b/src/NzbDrone.Core/Movies/MovieService.cs index e5248df2b7..8775548ac1 100644 --- a/src/NzbDrone.Core/Movies/MovieService.cs +++ b/src/NzbDrone.Core/Movies/MovieService.cs @@ -24,6 +24,7 @@ public interface IMovieService List AddMovies(List newMovies); Movie FindByImdbId(string imdbid); Movie FindByTmdbId(int tmdbid); + List FindByTmdbId(List tmdbids); Movie FindByTitle(string title); Movie FindByTitle(string title, int year); Movie FindByTitle(List titles, int? year, List otherTitles, List candidates); @@ -195,6 +196,11 @@ public Movie FindByTmdbId(int tmdbid) return _movieRepository.FindByTmdbId(tmdbid); } + public List FindByTmdbId(List tmdbids) + { + return _movieRepository.FindByTmdbId(tmdbids); + } + public Movie FindByPath(string path) { return _movieRepository.FindByPath(path);