diff --git a/frontend/src/Indexer/Index/Table/IndexerIndexRow.js b/frontend/src/Indexer/Index/Table/IndexerIndexRow.js
index fa802e254..fda726182 100644
--- a/frontend/src/Indexer/Index/Table/IndexerIndexRow.js
+++ b/frontend/src/Indexer/Index/Table/IndexerIndexRow.js
@@ -244,12 +244,15 @@ class IndexerIndexRow extends Component {
onPress={this.onIndexerInfoPress}
/>
-
+ {
+ indexerUrls ?
+ : null
+ }
("UpdateHistory").RegisterModel();
Mapper.Entity("AppSyncProfiles").RegisterModel();
+ Mapper.Entity("IndexerDefinitionVersions").RegisterModel();
}
private static void RegisterMappers()
diff --git a/src/NzbDrone.Core/HealthCheck/Checks/NoDefinitionCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/NoDefinitionCheck.cs
new file mode 100644
index 000000000..24c49c41d
--- /dev/null
+++ b/src/NzbDrone.Core/HealthCheck/Checks/NoDefinitionCheck.cs
@@ -0,0 +1,48 @@
+using System.Linq;
+using NLog;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Cardigann;
+using NzbDrone.Core.IndexerVersions;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+ [CheckOn(typeof(ProviderDeletedEvent))]
+ public class NoDefinitionCheck : HealthCheckBase
+ {
+ private readonly IIndexerDefinitionUpdateService _indexerDefinitionUpdateService;
+ private readonly IIndexerFactory _indexerFactory;
+
+ public NoDefinitionCheck(IIndexerDefinitionUpdateService indexerDefinitionUpdateService, IIndexerFactory indexerFactory, ILocalizationService localizationService)
+ : base(localizationService)
+ {
+ _indexerDefinitionUpdateService = indexerDefinitionUpdateService;
+ _indexerFactory = indexerFactory;
+ }
+
+ public override HealthCheck Check()
+ {
+ var currentDefs = _indexerDefinitionUpdateService.All();
+
+ var noDefIndexers = _indexerFactory.AllProviders(false)
+ .Where(i => i.Definition.Implementation == "Cardigann" && !currentDefs.Any(d => d.File == ((CardigannSettings)i.Definition.Settings).DefinitionFile)).ToList();
+
+ if (noDefIndexers.Count == 0)
+ {
+ return new HealthCheck(GetType());
+ }
+
+ var healthType = HealthCheckResult.Error;
+ var healthMessage = string.Format(_localizationService.GetLocalizedString("IndexerNoDefCheckMessage"),
+ string.Join(", ", noDefIndexers.Select(v => v.Definition.Name)));
+
+ return new HealthCheck(GetType(),
+ healthType,
+ healthMessage,
+ "#indexers-have-no-definition");
+ }
+
+ public override bool CheckOnSchedule => false;
+ }
+}
diff --git a/src/NzbDrone.Core/HealthCheck/Checks/OutdatedDefinitionCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/OutdatedDefinitionCheck.cs
index 5fd9ea81d..c45c37db6 100644
--- a/src/NzbDrone.Core/HealthCheck/Checks/OutdatedDefinitionCheck.cs
+++ b/src/NzbDrone.Core/HealthCheck/Checks/OutdatedDefinitionCheck.cs
@@ -35,10 +35,13 @@ public override HealthCheck Check()
return new HealthCheck(GetType());
}
+ var healthType = HealthCheckResult.Warning;
+ var healthMessage = string.Format(_localizationService.GetLocalizedString("IndexerObsoleteCheckMessage"),
+ string.Join(", ", oldIndexers.Select(v => v.Definition.Name)));
+
return new HealthCheck(GetType(),
- HealthCheckResult.Warning,
- string.Format(_localizationService.GetLocalizedString("IndexerObsoleteCheckMessage"),
- string.Join(", ", oldIndexers.Select(v => v.Definition.Name))),
+ healthType,
+ healthMessage,
"#indexers-are-obsolete");
}
diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs
index be38c301b..72cd5f817 100644
--- a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs
+++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs
@@ -1,13 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
-using System.IO.Compression;
using System.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnvironmentInfo;
-using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Cardigann;
using NzbDrone.Core.Messaging.Commands;
@@ -19,7 +17,7 @@ namespace NzbDrone.Core.IndexerVersions
public interface IIndexerDefinitionUpdateService
{
List All();
- CardigannDefinition GetDefinition(string fileKey);
+ CardigannDefinition GetCachedDefinition(string fileKey);
List GetBlocklist();
}
@@ -29,6 +27,8 @@ public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, I
private const string DEFINITION_BRANCH = "master";
private const int DEFINITION_VERSION = 3;
+
+ //Used when moving yml to C#
private readonly List _defintionBlocklist = new List()
{
"aither",
@@ -51,6 +51,7 @@ public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, I
private readonly IHttpClient _httpClient;
private readonly IAppFolderInfo _appFolderInfo;
private readonly IDiskProvider _diskProvider;
+ private readonly IIndexerDefinitionVersionService _versionService;
private readonly ICached _cache;
private readonly Logger _logger;
@@ -62,11 +63,13 @@ public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, I
public IndexerDefinitionUpdateService(IHttpClient httpClient,
IAppFolderInfo appFolderInfo,
IDiskProvider diskProvider,
+ IIndexerDefinitionVersionService versionService,
ICacheManager cacheManager,
Logger logger)
{
_appFolderInfo = appFolderInfo;
_diskProvider = diskProvider;
+ _versionService = versionService;
_cache = cacheManager.GetCache(typeof(CardigannDefinition), "definitions");
_httpClient = httpClient;
_logger = logger;
@@ -78,43 +81,24 @@ public List All()
try
{
- var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
- var response = _httpClient.Get>(request);
- indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList();
-
- var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom");
-
- var directoryInfo = new DirectoryInfo(definitionFolder);
-
- if (directoryInfo.Exists)
+ // Grab latest def list from server or fallback to disk
+ try
{
- var files = directoryInfo.GetFiles($"*.yml");
-
- foreach (var file in files)
- {
- _logger.Debug("Loading Custom Cardigann definition " + file.FullName);
-
- try
- {
- var definitionString = File.ReadAllText(file.FullName);
- var definition = _deserializer.Deserialize(definitionString);
-
- definition.File = Path.GetFileNameWithoutExtension(file.Name);
-
- if (indexerList.Any(i => i.File == definition.File || i.Name == definition.Name))
- {
- _logger.Warn("Custom Cardigann definition {0} does not have unique file name or Indexer name", file.FullName);
- continue;
- }
-
- indexerList.Add(definition);
- }
- catch (Exception e)
- {
- _logger.Error($"Error while parsing custom Cardigann definition {file.FullName}\n{e}");
- }
- }
+ var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
+ var response = _httpClient.Get>(request);
+ indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList();
}
+ catch
+ {
+ var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions");
+
+ indexerList = ReadDefinitionsFromDisk(indexerList, definitionFolder);
+ }
+
+ //Check for custom definitions
+ var customDefinitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom");
+
+ indexerList = ReadDefinitionsFromDisk(indexerList, customDefinitionFolder);
}
catch
{
@@ -124,14 +108,14 @@ public List All()
return indexerList;
}
- public CardigannDefinition GetDefinition(string file)
+ public CardigannDefinition GetCachedDefinition(string fileKey)
{
- if (string.IsNullOrEmpty(file))
+ if (string.IsNullOrEmpty(fileKey))
{
- throw new ArgumentNullException(nameof(file));
+ throw new ArgumentNullException(nameof(fileKey));
}
- var definition = _cache.Get(file, () => LoadIndexerDef(file));
+ var definition = _cache.Get(fileKey, () => GetUncachedDefinition(fileKey));
return definition;
}
@@ -141,15 +125,46 @@ public List GetBlocklist()
return _defintionBlocklist;
}
- private CardigannDefinition GetHttpDefinition(string id)
+ private List ReadDefinitionsFromDisk(List defs, string path, SearchOption options = SearchOption.TopDirectoryOnly)
{
- var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}");
- var response = _httpClient.Get(req);
- var definition = _deserializer.Deserialize(response.Content);
- return CleanIndexerDefinition(definition);
+ var indexerList = defs;
+
+ var directoryInfo = new DirectoryInfo(path);
+
+ if (directoryInfo.Exists)
+ {
+ var files = directoryInfo.GetFiles($"*.yml", options);
+
+ foreach (var file in files)
+ {
+ _logger.Debug("Loading definition " + file.FullName);
+
+ try
+ {
+ var definitionString = File.ReadAllText(file.FullName);
+ var definition = _deserializer.Deserialize(definitionString);
+
+ definition.File = Path.GetFileNameWithoutExtension(file.Name);
+
+ if (indexerList.Any(i => i.File == definition.File || i.Name == definition.Name))
+ {
+ _logger.Warn("Definition {0} does not have unique file name or Indexer name", file.FullName);
+ continue;
+ }
+
+ indexerList.Add(definition);
+ }
+ catch (Exception e)
+ {
+ _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
+ }
+ }
+ }
+
+ return indexerList;
}
- private CardigannDefinition LoadIndexerDef(string fileKey)
+ private CardigannDefinition GetUncachedDefinition(string fileKey)
{
if (string.IsNullOrEmpty(fileKey))
{
@@ -184,9 +199,24 @@ private CardigannDefinition LoadIndexerDef(string fileKey)
}
}
+ //Check to ensure it's in versioned defs before we go to web
+ if (!_versionService.All().Any(x => x.File == fileKey))
+ {
+ throw new ArgumentNullException(nameof(fileKey));
+ }
+
+ //No definition was returned locally, go to the web
return GetHttpDefinition(fileKey);
}
+ private CardigannDefinition GetHttpDefinition(string id)
+ {
+ var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}");
+ var response = _httpClient.Get(req);
+ var definition = _deserializer.Deserialize(response.Content);
+ return CleanIndexerDefinition(definition);
+ }
+
private CardigannDefinition CleanIndexerDefinition(CardigannDefinition definition)
{
if (definition.Settings == null)
@@ -242,29 +272,44 @@ private void UpdateLocalDefinitions()
{
var startupFolder = _appFolderInfo.AppDataFolder;
+ var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
+ var response = _httpClient.Get>(request);
+
+ var currentDefs = _versionService.All().ToDictionary(x => x.DefinitionId, x => x.Sha);
+
try
{
EnsureDefinitionsFolder();
- var definitionsFolder = Path.Combine(startupFolder, "Definitions");
- var saveFile = Path.Combine(startupFolder, "Definitions", $"indexers.zip");
-
- _httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/package.zip", saveFile);
-
- using (ZipArchive archive = ZipFile.OpenRead(saveFile))
+ foreach (var def in response.Resource)
{
- archive.ExtractToDirectory(definitionsFolder, true);
+ try
+ {
+ var saveFile = Path.Combine(startupFolder, "Definitions", $"{def.File}.yml");
+
+ if (currentDefs.TryGetValue(def.Id, out var defSha) && defSha == def.Sha)
+ {
+ _logger.Trace("Indexer already up to date: {0}", def.File);
+
+ continue;
+ }
+
+ _httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{def.File}", saveFile);
+
+ _versionService.Upsert(new IndexerDefinitionVersion { Sha = def.Sha, DefinitionId = def.Id, File = def.File, LastUpdated = DateTime.UtcNow });
+
+ _cache.Remove(def.File);
+ _logger.Debug("Updated definition: {0}", def.File);
+ }
+ catch (Exception ex)
+ {
+ _logger.Error("Definition download failed: {0}, {1}", def.File, ex.Message);
+ }
}
-
- _diskProvider.DeleteFile(saveFile);
-
- _cache.Clear();
-
- _logger.Debug("Updated indexer definitions");
}
catch (Exception ex)
{
- _logger.Error(ex, "Definition update failed");
+ _logger.Error(ex, "Definition download failed, error creating definitions folder in {0}", startupFolder);
}
}
}
diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersion.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersion.cs
new file mode 100644
index 000000000..df3d6a932
--- /dev/null
+++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersion.cs
@@ -0,0 +1,13 @@
+using System;
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+ public class IndexerDefinitionVersion : ModelBase
+ {
+ public string File { get; set; }
+ public string Sha { get; set; }
+ public DateTime LastUpdated { get; set; }
+ public string DefinitionId { get; set; }
+ }
+}
diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionRepository.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionRepository.cs
new file mode 100644
index 000000000..8e14d5e13
--- /dev/null
+++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionRepository.cs
@@ -0,0 +1,25 @@
+using System;
+using System.Linq;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+ public interface IIndexerDefinitionVersionRepository : IBasicRepository
+ {
+ public IndexerDefinitionVersion GetByDefId(string defId);
+ }
+
+ public class IndexerDefinitionVersionRepository : BasicRepository, IIndexerDefinitionVersionRepository
+ {
+ public IndexerDefinitionVersionRepository(IMainDatabase database, IEventAggregator eventAggregator)
+ : base(database, eventAggregator)
+ {
+ }
+
+ public IndexerDefinitionVersion GetByDefId(string defId)
+ {
+ return Query(x => x.DefinitionId == defId).SingleOrDefault();
+ }
+ }
+}
diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionService.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionService.cs
new file mode 100644
index 000000000..8b2b06d94
--- /dev/null
+++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionService.cs
@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+ public interface IIndexerDefinitionVersionService
+ {
+ IndexerDefinitionVersion Get(int indexerVersionId);
+ IndexerDefinitionVersion GetByDefId(string defId);
+ List All();
+ IndexerDefinitionVersion Add(IndexerDefinitionVersion defVersion);
+ IndexerDefinitionVersion Upsert(IndexerDefinitionVersion defVersion);
+ void Delete(int indexerVersionId);
+ }
+
+ public class IndexerDefinitionVersionService : IIndexerDefinitionVersionService
+ {
+ private readonly IIndexerDefinitionVersionRepository _repo;
+
+ public IndexerDefinitionVersionService(IIndexerDefinitionVersionRepository repo)
+ {
+ _repo = repo;
+ }
+
+ public IndexerDefinitionVersion Get(int indexerVersionId)
+ {
+ return _repo.Get(indexerVersionId);
+ }
+
+ public IndexerDefinitionVersion GetByDefId(string defId)
+ {
+ return _repo.GetByDefId(defId);
+ }
+
+ public List All()
+ {
+ return _repo.All().ToList();
+ }
+
+ public IndexerDefinitionVersion Add(IndexerDefinitionVersion defVersion)
+ {
+ _repo.Insert(defVersion);
+
+ return defVersion;
+ }
+
+ public IndexerDefinitionVersion Upsert(IndexerDefinitionVersion defVersion)
+ {
+ var existing = _repo.GetByDefId(defVersion.DefinitionId);
+
+ if (existing != null)
+ {
+ defVersion.Id = existing.Id;
+ }
+
+ defVersion = _repo.Upsert(defVersion);
+
+ return defVersion;
+ }
+
+ public void Delete(int indexerVersionId)
+ {
+ _repo.Delete(indexerVersionId);
+ }
+ }
+}
diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
index 9012983f1..6f4797c77 100644
--- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
+++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
@@ -37,7 +37,7 @@ public override IIndexerRequestGenerator GetRequestGenerator()
{
var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
new CardigannRequestGenerator(_configService,
- _definitionService.GetDefinition(Settings.DefinitionFile),
+ _definitionService.GetCachedDefinition(Settings.DefinitionFile),
_logger)
{
HttpClient = _httpClient,
@@ -57,7 +57,7 @@ public override IIndexerRequestGenerator GetRequestGenerator()
public override IParseIndexerResponse GetParser()
{
return new CardigannParser(_configService,
- _definitionService.GetDefinition(Settings.DefinitionFile),
+ _definitionService.GetCachedDefinition(Settings.DefinitionFile),
_logger)
{
Settings = Settings
diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs
index 0140eb289..d08d57031 100644
--- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs
+++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs
@@ -59,7 +59,7 @@ public override List All()
catch
{
// Skip indexer if we fail in Cardigann mapping
- continue;
+ _logger.Debug("Indexer {0} has no definition", definition.Name);
}
}
@@ -96,7 +96,7 @@ protected override List Active()
private void MapCardigannDefinition(IndexerDefinition definition)
{
var settings = (CardigannSettings)definition.Settings;
- var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
+ var defFile = _definitionService.GetCachedDefinition(settings.DefinitionFile);
definition.ExtraFields = defFile.Settings;
if (defFile.Login?.Captcha != null && !definition.ExtraFields.Any(x => x.Type == "cardigannCaptcha"))
diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json
index 202f4b46e..b72ef980e 100644
--- a/src/NzbDrone.Core/Localization/Core/en.json
+++ b/src/NzbDrone.Core/Localization/Core/en.json
@@ -178,6 +178,7 @@
"IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours",
"IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}",
"IndexerObsoleteCheckMessage": "Indexers are obsolete or have been updated: {0}. Please remove and (or) re-add to Prowlarr",
+ "IndexerNoDefCheckMessage": "Indexers have no definition and will not work: {0}. Please remove and (or) re-add to Prowlarr",
"IndexerPriority": "Indexer Priority",
"IndexerPriorityHelpText": "Indexer Priority from 1 (Highest) to 50 (Lowest). Default: 25.",
"IndexerProxies": "Indexer Proxies",
diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs
index 3cd433e29..4dc7cbb08 100644
--- a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs
+++ b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs
@@ -114,7 +114,7 @@ public override IndexerDefinition ToModel(IndexerResource resource)
var settings = (CardigannSettings)definition.Settings;
- var cardigannDefinition = _definitionService.GetDefinition(settings.DefinitionFile);
+ var cardigannDefinition = _definitionService.GetCachedDefinition(settings.DefinitionFile);
foreach (var field in resource.Fields)
{