From 4254a05ea31501baa2451120c2bd1e0ae0694408 Mon Sep 17 00:00:00 2001 From: Yukine Date: Thu, 15 Jul 2021 03:03:49 +0200 Subject: [PATCH] Fixed: (AnimeBytes) cleanup code, fix episode searching & improve Season matching (#329) * refactor/fix(AnimeBytes): use data classes & fix season searching * fix: only append epsisode when season was found * feat: add Episode padding back for Sonarr compatibility * fix: strip Epsiode number from request --- .../Indexers/Definitions/AnimeBytes.cs | 515 ++++++++++++++---- 1 file changed, 420 insertions(+), 95 deletions(-) diff --git a/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs b/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs index 78a378a7c..e70c01baa 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/AnimeBytes.cs @@ -8,6 +8,7 @@ using System.Text.RegularExpressions; using FluentValidation; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Http; @@ -114,7 +115,7 @@ private IEnumerable GetPagedRequests(string searchType, string t { "username", Settings.Username }, { "torrent_pass", Settings.Passkey }, { "type", searchType }, - { "searchstr", term } + { "searchstr", StripEpisodeNumber(term) } }; var queryCats = Capabilities.Categories.MapTorznabCapsToTrackers(categories); @@ -181,6 +182,15 @@ public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchC public Func> GetCookies { get; set; } public Action, DateTime?> CookiesUpdater { get; set; } + + private string StripEpisodeNumber(string term) + { + // Tracer does not support searching with episode number so strip it if we have one + term = Regex.Replace(term, @"\W(\dx)?\d?\d$", string.Empty); + term = Regex.Replace(term, @"\W(S\d\d?E)?\d?\d$", string.Empty); + term = Regex.Replace(term, @"\W\d+$", string.Empty); + return term; + } } public class AnimeBytesParser : IParseIndexerResponse @@ -208,29 +218,17 @@ public IList ParseResponse(IndexerResponse indexerResponse) throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}"); } - //TODO: Create API Resource Type - var json = JsonConvert.DeserializeObject(indexerResponse.Content); + var response = JsonConvert.DeserializeObject(indexerResponse.Content); - if (json["error"] != null) + if (response.Matches > 0) { - throw new Exception(json["error"].ToString()); - } - - var matches = (long)json["Matches"]; - - if (matches > 0) - { - var groups = (JArray)json.Groups; - - foreach (var group in groups) + foreach (var group in response.Groups) { var synonyms = new List(); - var posterStr = (string)group["Image"]; - var poster = string.IsNullOrWhiteSpace(posterStr) ? null : new Uri(posterStr); - var year = (int)group["Year"]; - var groupName = (string)group["GroupName"]; - var seriesName = (string)group["SeriesName"]; - var mainTitle = WebUtility.HtmlDecode((string)group["FullName"]); + var year = group.Year; + var groupName = group.GroupName; + var seriesName = group.SeriesName; + var mainTitle = WebUtility.HtmlDecode(group.FullName); if (seriesName != null) { mainTitle = seriesName; @@ -238,105 +236,122 @@ public IList ParseResponse(IndexerResponse indexerResponse) synonyms.Add(mainTitle); - // TODO: Do we need all these options? - //if (group["Synonymns"].HasValues) - //{ - // if (group["Synonymns"] is JArray) - // { - // var allSyonyms = group["Synonymns"].ToObject>(); - - // if (AddJapaneseTitle && allSyonyms.Count >= 1) - // synonyms.Add(allSyonyms[0]); - // if (AddRomajiTitle && allSyonyms.Count >= 2) - // synonyms.Add(allSyonyms[1]); - // if (AddAlternativeTitles && allSyonyms.Count >= 3) - // synonyms.AddRange(allSyonyms[2].Split(',').Select(t => t.Trim())); - // } - // else - // { - // var allSynonyms = group["Synonymns"].ToObject>(); - - // if (AddJapaneseTitle && allSynonyms.ContainsKey(0)) - // synonyms.Add(allSynonyms[0]); - // if (AddRomajiTitle && allSynonyms.ContainsKey(1)) - // synonyms.Add(allSynonyms[1]); - // if (AddAlternativeTitles && allSynonyms.ContainsKey(2)) - // { - // synonyms.AddRange(allSynonyms[2].Split(',').Select(t => t.Trim())); - // } - // } - //} - List category = null; - var categoryName = (string)group["CategoryName"]; - - var description = (string)group["Description"]; - - foreach (var torrent in group["Torrents"]) + if (group.Synonymns.StringArray != null) { - var releaseInfo = "S01"; + synonyms.AddRange(group.Synonymns.StringArray); + } + else + { + synonyms.AddRange(group.Synonymns.StringMap.Values); + } + + List category = null; + var categoryName = group.CategoryName; + + var description = group.Description; + + foreach (var torrent in group.Torrents) + { + const string defaultReleaseInfo = "S01"; + var releaseInfo = defaultReleaseInfo; string episode = null; int? season = null; - var editionTitle = (string)torrent["EditionData"]["EditionTitle"]; + var editionTitle = torrent.EditionData.EditionTitle; if (!string.IsNullOrWhiteSpace(editionTitle)) { releaseInfo = WebUtility.HtmlDecode(editionTitle); + + var simpleSeasonRegEx = new Regex(@"Season (\d+)", RegexOptions.Compiled); + var simpleSeasonRegExMatch = simpleSeasonRegEx.Match(releaseInfo); + if (simpleSeasonRegExMatch.Success) + { + season = ParseUtil.CoerceInt(simpleSeasonRegExMatch.Groups[1].Value); + } + + var episodeRegEx = new Regex(@"Episode (\d+)", RegexOptions.Compiled); + var episodeRegExMatch = episodeRegEx.Match(releaseInfo); + if (episodeRegExMatch.Success) + { + episode = episodeRegExMatch.Groups[1].Value; + } } - var seasonRegEx = new Regex(@"Season (\d+)", RegexOptions.Compiled); - var seasonRegExMatch = seasonRegEx.Match(releaseInfo); - if (seasonRegExMatch.Success) + var advancedSeasonRegEx = new Regex(@"(\d+)(st|nd|rd|th) Season", RegexOptions.Compiled | RegexOptions.IgnoreCase); + var advancedSeasonRegExMatch = advancedSeasonRegEx.Match(mainTitle); + if (advancedSeasonRegExMatch.Success) { - season = ParseUtil.CoerceInt(seasonRegExMatch.Groups[1].Value); + season = ParseUtil.CoerceInt(advancedSeasonRegExMatch.Groups[1].Value); } - var episodeRegEx = new Regex(@"Episode (\d+)", RegexOptions.Compiled); - var episodeRegExMatch = episodeRegEx.Match(releaseInfo); - if (episodeRegExMatch.Success) + var seasonCharactersRegEx = new Regex(@"(I{2,})$", RegexOptions.Compiled); + var seasonCharactersRegExMatch = seasonCharactersRegEx.Match(mainTitle); + if (seasonCharactersRegExMatch.Success) { - episode = episodeRegExMatch.Groups[1].Value; + season = seasonCharactersRegExMatch.Groups[1].Value.Length; } - releaseInfo = releaseInfo.Replace("Episode ", ""); + var seasonNumberRegEx = new Regex(@"([2-9])$", RegexOptions.Compiled); + var seasonNumberRegExMatch = seasonNumberRegEx.Match(mainTitle); + if (seasonNumberRegExMatch.Success) + { + season = ParseUtil.CoerceInt(seasonNumberRegExMatch.Groups[1].Value); + } + + var foundSeason = false; + + if (season != null) + { + releaseInfo = $"Season {season}"; + + foundSeason = true; + } + + if (episode != null) + { + var epString = $"Episode {episode}"; + + if (foundSeason) + { + releaseInfo += $" {epString}"; + } + else + { + releaseInfo = epString; + } + } + + releaseInfo = releaseInfo.Replace("Episode ", string.Empty); releaseInfo = releaseInfo.Replace("Season ", "S"); releaseInfo = releaseInfo.Trim(); - //if (PadEpisode && int.TryParse(releaseInfo, out _) && releaseInfo.Length == 1) - //{ - // releaseInfo = "0" + releaseInfo; - //} + if (int.TryParse(releaseInfo, out _) && releaseInfo.Length == 1) + { + releaseInfo = "0" + releaseInfo; + } - //if (FilterSeasonEpisode) - //{ - // if (query.Season != 0 && season != null && season != query.Season) // skip if season doesn't match - // continue; - // if (query.Episode != null && episode != null && episode != query.Episode) // skip if episode doesn't match - // continue; - //} - var torrentId = (long)torrent["ID"]; - var property = ((string)torrent["Property"]).Replace(" | Freeleech", ""); - var link = (string)torrent["Link"]; - var linkUri = new Uri(link); - var uploadTimeString = (string)torrent["UploadTime"]; - var uploadTime = DateTime.ParseExact(uploadTimeString, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture); - var publishDate = DateTime.SpecifyKind(uploadTime, DateTimeKind.Utc).ToLocalTime(); + var torrentId = torrent.Id; + var property = torrent.Property.Replace(" | Freeleech", string.Empty); + var link = torrent.Link; + var uploadTime = torrent.UploadTime; + var publishDate = DateTime.SpecifyKind(uploadTime.DateTime, DateTimeKind.Utc).ToLocalTime(); var details = new Uri(_settings.BaseUrl + "torrent/" + torrentId + "/group"); - var size = (long)torrent["Size"]; - var snatched = (int)torrent["Snatched"]; - var seeders = (int)torrent["Seeders"]; - var leechers = (int)torrent["Leechers"]; - var fileCount = (int)torrent["FileCount"]; + var size = torrent.Size; + var snatched = torrent.Snatched; + var seeders = torrent.Seeders; + var leechers = torrent.Leechers; + var fileCount = torrent.FileCount; var peers = seeders + leechers; - var rawDownMultiplier = (int?)torrent["RawDownMultiplier"] ?? 0; - var rawUpMultiplier = (int?)torrent["RawUpMultiplier"] ?? 0; + var rawDownMultiplier = torrent.RawDownMultiplier; + var rawUpMultiplier = torrent.RawUpMultiplier; + // Ignore these categories as they'll cause hell with the matcher + // TV Special, ONA, DVD Special, BD Special if (groupName == "TV Series" || groupName == "OVA") { category = new List { NewznabStandardCategory.TVAnime }; } - // Ignore these categories as they'll cause hell with the matcher - // TV Special, OVA, ONA, DVD Special, BD Special if (groupName == "Movie" || groupName == "Live Action Movie") { category = new List { NewznabStandardCategory.Movies }; @@ -422,7 +437,7 @@ public IList ParseResponse(IndexerResponse indexerResponse) //{ // continue; //} - var infoString = releaseTags.Aggregate("", (prev, cur) => prev + "[" + cur + "]"); + var infoString = releaseTags.Aggregate(string.Empty, (prev, cur) => prev + "[" + cur + "]"); var minimumSeedTime = 259200; // Additional 5 hours per GB @@ -443,7 +458,7 @@ public IList ParseResponse(IndexerResponse indexerResponse) Title = releaseTitle, InfoUrl = details.AbsoluteUri, Guid = guid.AbsoluteUri, - DownloadUrl = linkUri.AbsoluteUri, + DownloadUrl = link.AbsoluteUri, PublishDate = publishDate, Categories = category, Description = description, @@ -453,7 +468,7 @@ public IList ParseResponse(IndexerResponse indexerResponse) Grabs = snatched, Files = fileCount, DownloadVolumeFactor = rawDownMultiplier, - UploadVolumeFactor = rawUpMultiplier + UploadVolumeFactor = rawUpMultiplier, }; torrentInfos.Add(release); @@ -507,4 +522,314 @@ public NzbDroneValidationResult Validate() return new NzbDroneValidationResult(Validator.Validate(this)); } } + + public class AnimeBytesResponse + { + [JsonProperty("Matches")] + public long Matches { get; set; } + + [JsonProperty("Limit")] + public long Limit { get; set; } + + [JsonProperty("Results")] + [JsonConverter(typeof(ParseStringConverter))] + public long Results { get; set; } + + [JsonProperty("Groups")] + public Group[] Groups { get; set; } + } + + public class Group + { + [JsonProperty("ID")] + public long Id { get; set; } + + [JsonProperty("CategoryName")] + public string CategoryName { get; set; } + + [JsonProperty("FullName")] + public string FullName { get; set; } + + [JsonProperty("GroupName")] + public string GroupName { get; set; } + + [JsonProperty("SeriesID")] + [JsonConverter(typeof(ParseStringConverter))] + public long SeriesId { get; set; } + + [JsonProperty("SeriesName")] + public string SeriesName { get; set; } + + [JsonProperty("Artists")] + public object Artists { get; set; } + + [JsonProperty("Year")] + [JsonConverter(typeof(ParseStringConverter))] + public long Year { get; set; } + + [JsonProperty("Image")] + public Uri Image { get; set; } + + [JsonProperty("Synonymns")] + [JsonConverter(typeof(SynonymnsConverter))] + public Synonymns Synonymns { get; set; } + + [JsonProperty("Snatched")] + public long Snatched { get; set; } + + [JsonProperty("Comments")] + public long Comments { get; set; } + + [JsonProperty("Links")] + public LinksUnion Links { get; set; } + + [JsonProperty("Votes")] + public long Votes { get; set; } + + [JsonProperty("AvgVote")] + public double AvgVote { get; set; } + + [JsonProperty("Associations")] + public object Associations { get; set; } + + [JsonProperty("Description")] + public string Description { get; set; } + + [JsonProperty("DescriptionHTML")] + public string DescriptionHtml { get; set; } + + [JsonProperty("EpCount")] + public long EpCount { get; set; } + + [JsonProperty("StudioList")] + public string StudioList { get; set; } + + [JsonProperty("PastWeek")] + public long PastWeek { get; set; } + + [JsonProperty("Incomplete")] + public bool Incomplete { get; set; } + + [JsonProperty("Ongoing")] + public bool Ongoing { get; set; } + + [JsonProperty("Tags")] + public List Tags { get; set; } + + [JsonProperty("Torrents")] + public List Torrents { get; set; } + } + + public class LinksClass + { + [JsonProperty("ANN", NullValueHandling = NullValueHandling.Ignore)] + public Uri Ann { get; set; } + + [JsonProperty("Manga-Updates", NullValueHandling = NullValueHandling.Ignore)] + public Uri MangaUpdates { get; set; } + + [JsonProperty("Wikipedia", NullValueHandling = NullValueHandling.Ignore)] + public Uri Wikipedia { get; set; } + + [JsonProperty("MAL", NullValueHandling = NullValueHandling.Ignore)] + public Uri Mal { get; set; } + + [JsonProperty("AniDB", NullValueHandling = NullValueHandling.Ignore)] + public Uri AniDb { get; set; } + } + + public class Torrent + { + [JsonProperty("ID")] + public long Id { get; set; } + + [JsonProperty("EditionData")] + public EditionData EditionData { get; set; } + + [JsonProperty("RawDownMultiplier")] + public double? RawDownMultiplier { get; set; } + + [JsonProperty("RawUpMultiplier")] + public double? RawUpMultiplier { get; set; } + + [JsonProperty("Link")] + public Uri Link { get; set; } + + [JsonProperty("Property")] + public string Property { get; set; } + + [JsonProperty("Snatched")] + public int Snatched { get; set; } + + [JsonProperty("Seeders")] + public int Seeders { get; set; } + + [JsonProperty("Leechers")] + public int Leechers { get; set; } + + [JsonProperty("Size")] + public long Size { get; set; } + + [JsonProperty("FileCount")] + public int FileCount { get; set; } + + [JsonProperty("UploadTime")] + public DateTimeOffset UploadTime { get; set; } + } + + public class EditionData + { + [JsonProperty("EditionTitle")] + public string EditionTitle { get; set; } + } + + public struct LinksUnion + { + public List AnythingArray; + public LinksClass LinksClass; + + public static implicit operator LinksUnion(List anythingArray) => new LinksUnion { AnythingArray = anythingArray }; + + public static implicit operator LinksUnion(LinksClass linksClass) => new LinksUnion { LinksClass = linksClass }; + } + + public struct Synonymns + { + public List StringArray; + public Dictionary StringMap; + + public static implicit operator Synonymns(List stringArray) => new Synonymns { StringArray = stringArray }; + + public static implicit operator Synonymns(Dictionary stringMap) => new Synonymns { StringMap = stringMap }; + } + + internal static class Converter + { + public static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + MetadataPropertyHandling = MetadataPropertyHandling.Ignore, + DateParseHandling = DateParseHandling.None, + Converters = + { + LinksUnionConverter.Singleton, + SynonymnsConverter.Singleton, + new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } + }, + }; + } + + internal class LinksUnionConverter : JsonConverter + { + public override bool CanConvert(Type t) => t == typeof(LinksUnion) || t == typeof(LinksUnion?); + + public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer) + { + switch (reader.TokenType) + { + case JsonToken.StartObject: + var objectValue = serializer.Deserialize(reader); + return new LinksUnion { LinksClass = objectValue }; + case JsonToken.StartArray: + var arrayValue = serializer.Deserialize>(reader); + return new LinksUnion { AnythingArray = arrayValue }; + } + + throw new Exception("Cannot unmarshal type LinksUnion"); + } + + public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer) + { + var value = (LinksUnion)untypedValue; + if (value.AnythingArray != null) + { + serializer.Serialize(writer, value.AnythingArray); + return; + } + + if (value.LinksClass == null) + { + throw new Exception("Cannot marshal type LinksUnion"); + } + + serializer.Serialize(writer, value.LinksClass); + return; + } + + public static readonly LinksUnionConverter Singleton = new LinksUnionConverter(); + } + + internal class ParseStringConverter : JsonConverter + { + public override bool CanConvert(Type t) => t == typeof(long) || t == typeof(long?); + + public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + var value = serializer.Deserialize(reader); + if (long.TryParse(value, out var l)) + { + return l; + } + + throw new Exception("Cannot unmarshal type long"); + } + + public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer) + { + if (untypedValue == null) + { + serializer.Serialize(writer, null); + return; + } + + var value = (long)untypedValue; + serializer.Serialize(writer, value.ToString()); + return; + } + + public static readonly ParseStringConverter Singleton = new ParseStringConverter(); + } + + internal class SynonymnsConverter : JsonConverter + { + public override bool CanConvert(Type t) => t == typeof(Synonymns) || t == typeof(Synonymns?); + + public override object ReadJson(JsonReader reader, Type t, object existingValue, JsonSerializer serializer) + { + switch (reader.TokenType) + { + case JsonToken.StartObject: + var objectValue = serializer.Deserialize>(reader); + return new Synonymns { StringMap = objectValue }; + case JsonToken.StartArray: + var arrayValue = serializer.Deserialize>(reader); + return new Synonymns { StringArray = arrayValue }; + } + + throw new Exception("Cannot unmarshal type Synonymns"); + } + + public override void WriteJson(JsonWriter writer, object untypedValue, JsonSerializer serializer) + { + var value = (Synonymns)untypedValue; + if (value.StringArray != null) + { + serializer.Serialize(writer, value.StringArray); + return; + } + + if (value.StringMap == null) + { + throw new Exception("Cannot marshal type Synonymns"); + } + + serializer.Serialize(writer, value.StringMap); + } + + public static readonly SynonymnsConverter Singleton = new SynonymnsConverter(); + } }