Backup_Repos/Readarr
1
0
Fork 0
mirror of https://github.com/Readarr/Readarr synced 2025-12-24 01:03:32 +01:00
Code Issues Projects Releases Wiki Activity

Move all data fetching to BookInfo v2

Browse source
This commit is contained in:
BookInfo 2022-01-12 21:38:30 +00:00 committed by ta264
parent 33e1c4a537
commit 1491788081
43 changed files with 904 additions and 941 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
frontend/src/Search/Book/AddNewBookModalContent.css
View file

@ -17,10 +17,15 @@
font-size: 36px;
}
.series {
font-weight: 300;
font-size: 24px;
}
.authorName {
margin-bottom: 20px;
font-weight: 300;
font-size: 20px;
font-size: 24px;
}
.disambiguation {

9
frontend/src/Search/Book/AddNewBookModalContent.js
View file

@ -43,6 +43,7 @@ class AddNewBookModalContent extends Component {
render() {
const {
bookTitle,
seriesTitle,
authorName,
disambiguation,
overview,
@ -84,6 +85,13 @@ class AddNewBookModalContent extends Component {
<span className={styles.disambiguation}>({disambiguation})</span>
}
{
!!seriesTitle &&
<div className={styles.series}>
{seriesTitle}
</div>
}
<div>
<span className={styles.authorName}> By: {authorName}</span>
</div>
@ -144,6 +152,7 @@ class AddNewBookModalContent extends Component {
AddNewBookModalContent.propTypes = {
bookTitle: PropTypes.string.isRequired,
seriesTitle: PropTypes.string,
authorName: PropTypes.string.isRequired,
disambiguation: PropTypes.string,
overview: PropTypes.string,

5
frontend/src/Search/Book/AddNewBookSearchResult.css
View file

@ -52,6 +52,11 @@
font-size: 36px;
}
.series {
font-weight: 300;
font-size: 24px;
}
.year {
margin-left: 10px;
color: $disabledColor;

10
frontend/src/Search/Book/AddNewBookSearchResult.js
View file

@ -74,6 +74,7 @@ class AddNewBookSearchResult extends Component {
foreignBookId,
titleSlug,
title,
seriesTitle,
releaseDate,
disambiguation,
overview,
@ -151,6 +152,13 @@ class AddNewBookSearchResult extends Component {
</div>
</div>
{
seriesTitle &&
<div className={styles.series}>
{seriesTitle}
</div>
}
<div>
<Label size={sizes.LARGE}>
<HeartRating
@ -188,6 +196,7 @@ class AddNewBookSearchResult extends Component {
isExistingAuthor={isExistingAuthor}
foreignBookId={foreignBookId}
bookTitle={title}
seriesTitle={seriesTitle}
disambiguation={disambiguation}
authorName={author.authorName}
overview={overview}
@ -203,6 +212,7 @@ AddNewBookSearchResult.propTypes = {
foreignBookId: PropTypes.string.isRequired,
titleSlug: PropTypes.string.isRequired,
title: PropTypes.string.isRequired,
seriesTitle: PropTypes.string,
releaseDate: PropTypes.string,
disambiguation: PropTypes.string,
overview: PropTypes.string,

61
src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs
View file

@ -7,6 +7,7 @@
using NzbDrone.Core.ImportLists;
using NzbDrone.Core.ImportLists.Exclusions;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.MetadataSource.Goodreads;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Test.Framework;
@ -30,27 +31,23 @@ public void SetUp()
.Setup(v => v.Fetch())
.Returns(_importListReports);
Mocker.GetMock<ISearchForNewAuthor>()
.Setup(v => v.SearchForNewAuthor(It.IsAny<string>()))
.Returns(new List<Author>());
Mocker.GetMock<IGoodreadsSearchProxy>()
.Setup(v => v.Search(It.IsAny<string>()))
.Returns(new List<SearchJsonResource>());
Mocker.GetMock<ISearchForNewBook>()
.Setup(v => v.SearchForNewBook(It.IsAny<string>(), It.IsAny<string>()))
.Returns(new List<Book>());
Mocker.GetMock<ISearchForNewBook>()
.Setup(v => v.SearchByGoodreadsId(It.IsAny<int>()))
.Returns<int>(x => Builder<Book>
.CreateListOfSize(1)
.TheFirst(1)
.With(b => b.ForeignBookId = "4321")
.With(b => b.Editions = Builder<Edition>
.CreateListOfSize(1)
.TheFirst(1)
.With(e => e.ForeignEditionId = x.ToString())
.With(e => e.Monitored = true)
.BuildList())
.BuildList());
Mocker.GetMock<IGoodreadsProxy>()
.Setup(v => v.GetBookInfo(It.IsAny<string>(), true))
.Returns<string, bool>((id, useCache) => Builder<Book>
.CreateNew()
.With(b => b.AuthorMetadata = Builder<AuthorMetadata>.CreateNew().Build())
.With(b => b.ForeignBookId = "4321")
.With(b => b.Editions = Builder<Edition>
.CreateListOfSize(1)
.TheFirst(1)
.With(e => e.ForeignEditionId = id.ToString())
.With(e => e.Monitored = true)
.BuildList())
.Build());
Mocker.GetMock<IImportListFactory>()
.Setup(v => v.Get(It.IsAny<int>()))
@ -111,8 +108,8 @@ private void WithExistingAuthor()
private void WithExistingBook()
{
Mocker.GetMock<IBookService>()
.Setup(v => v.FindById(_importListReports.First().EditionGoodreadsId))
.Returns(new Book { Id = 1, ForeignBookId = _importListReports.First().EditionGoodreadsId });
.Setup(v => v.FindById("4321"))
.Returns(new Book { Id = 1, ForeignBookId = _importListReports.First().BookGoodreadsId });
}
private void WithExcludedAuthor()
@ -153,8 +150,8 @@ public void should_search_if_author_title_and_no_author_id()
{
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewAuthor>()
.Verify(v => v.SearchForNewAuthor(It.IsAny<string>()), Times.Once());
Mocker.GetMock<IGoodreadsSearchProxy>()
.Verify(v => v.Search(It.IsAny<string>()), Times.Once());
}
[Test]
@ -173,8 +170,8 @@ public void should_search_if_book_title_and_no_book_id()
WithBook();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewBook>()
.Verify(v => v.SearchForNewBook(It.IsAny<string>(), It.IsAny<string>()), Times.Once());
Mocker.GetMock<IGoodreadsSearchProxy>()
.Verify(v => v.Search(It.IsAny<string>()), Times.Once());
}
[Test]
@ -184,8 +181,8 @@ public void should_not_search_if_book_title_and_book_id()
WithBookId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewBook>()
.Verify(v => v.SearchForNewBook(It.IsAny<string>(), It.IsAny<string>()), Times.Never());
Mocker.GetMock<IGoodreadsSearchProxy>()
.Verify(v => v.Search(It.IsAny<string>()), Times.Never());
}
[Test]
@ -196,11 +193,11 @@ public void should_not_search_if_all_info()
WithBookId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewAuthor>()
.Verify(v => v.SearchForNewAuthor(It.IsAny<string>()), Times.Never());
Mocker.GetMock<IGoodreadsSearchProxy>()
.Verify(v => v.Search(It.IsAny<string>()), Times.Never());
Mocker.GetMock<ISearchForNewBook>()
.Verify(v => v.SearchForNewBook(It.IsAny<string>(), It.IsAny<string>()), Times.Never());
Mocker.GetMock<IGoodreadsSearchProxy>()
.Verify(v => v.Search(It.IsAny<string>()), Times.Never());
}
[Test]

2
src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/CandidateServiceFixture.cs
View file

@ -17,7 +17,7 @@ public class CandidateServiceFixture : CoreTest<CandidateService>
public void should_not_throw_on_goodreads_exception()
{
Mocker.GetMock<ISearchForNewBook>()
.Setup(s => s.SearchForNewBook(It.IsAny<string>(), It.IsAny<string>()))
.Setup(s => s.SearchForNewBook(It.IsAny<string>(), It.IsAny<string>(), true))
.Throws(new GoodreadsException("Bad search"));
var edition = new LocalEdition

2
src/NzbDrone.Core.Test/MediaFiles/TrackImport/Identification/IdentificationServiceFixture.cs
View file

@ -60,7 +60,7 @@ public void SetUp()
Mocker.SetConstant<IConfigService>(Mocker.Resolve<IConfigService>());
Mocker.SetConstant<IProvideAuthorInfo>(Mocker.Resolve<BookInfoProxy>());
Mocker.SetConstant<IProvideBookInfo>(Mocker.Resolve<GoodreadsProxy>());
Mocker.SetConstant<IProvideBookInfo>(Mocker.Resolve<BookInfoProxy>());
_addAuthorService = Mocker.Resolve<AddAuthorService>();

19
src/NzbDrone.Core.Test/MetadataSource/Goodreads/GoodreadsProxyFixture.cs → src/NzbDrone.Core.Test/MetadataSource/BookInfo/BookInfoProxyFixture.cs
View file

@ -4,17 +4,16 @@
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Books;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.MetadataSource.Goodreads;
using NzbDrone.Core.MetadataSource.BookInfo;
using NzbDrone.Core.Profiles.Metadata;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.MetadataSource.Goodreads
{
[TestFixture]
public class GoodreadsProxyFixture : CoreTest<GoodreadsProxy>
public class BookInfoProxyFixture : CoreTest<BookInfoProxy>
{
private MetadataProfile _metadataProfile;
@ -45,8 +44,8 @@ public void should_be_able_to_get_author_detail(string mbId, string name)
details.Name.Should().Be(name);
}
[TestCase("64216", "Guards! Guards!")]
[TestCase("1371", "Ιλιάς")]
[TestCase("1128601", "Guards! Guards!")]
[TestCase("3293141", "Ιλιάς")]
public void should_be_able_to_get_book_detail(string mbId, string name)
{
var details = Subject.GetBookInfo(mbId);
@ -56,13 +55,13 @@ public void should_be_able_to_get_book_detail(string mbId, string name)
details.Item2.Title.Should().Be(name);
}
[TestCase("54837483", "The Book of Dust", "1")]
[TestCase("28360360", "October Daye", "9.3")]
[TestCase("14190696", "The Book of Dust", "1")]
[TestCase("48427681", "October Daye Chronological Order", "7.1")]
public void should_parse_series_from_title(string id, string series, string position)
{
var result = Subject.GetBookInfo(id);
var link = result.Item2.SeriesLinks.Value.First();
var link = result.Item2.SeriesLinks.Value.OrderBy(x => x.SeriesPosition).First();
link.Series.Value.Title.Should().Be(series);
link.Position.Should().Be(position);
}
@ -70,13 +69,13 @@ public void should_parse_series_from_title(string id, string series, string posi
[Test]
public void getting_details_of_invalid_author()
{
Assert.Throws<AuthorNotFoundException>(() => Subject.GetAuthorInfo("66c66aaa-6e2f-4930-8610-912e24c63ed1"));
Assert.Throws<AuthorNotFoundException>(() => Subject.GetAuthorInfo("1"));
}
[Test]
public void getting_details_of_invalid_book()
{
Assert.Throws<BookNotFoundException>(() => Subject.GetBookInfo("66c66aaa-6e2f-4930-8610-912e24c63ed1"));
Assert.Throws<BookNotFoundException>(() => Subject.GetBookInfo("99999999"));
}
private void ValidateAuthor(Author author)

111
src/NzbDrone.Core.Test/MetadataSource/BookInfo/BookInfoProxySearchFixture.cs Normal file
View file

@ -0,0 +1,111 @@
using System;
using System.Collections.Generic;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.Http;
using NzbDrone.Core.Books;
using NzbDrone.Core.Http;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.MetadataSource.BookInfo;
using NzbDrone.Core.MetadataSource.Goodreads;
using NzbDrone.Core.Profiles.Metadata;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Test.Common;
namespace NzbDrone.Core.Test.MetadataSource.Goodreads
{
[TestFixture]
public class BookInfoProxySearchFixture : CoreTest<BookInfoProxy>
{
[SetUp]
public void Setup()
{
UseRealHttp();
Mocker.SetConstant<IGoodreadsSearchProxy>(Mocker.Resolve<GoodreadsSearchProxy>());
var httpClient = Mocker.Resolve<IHttpClient>();
Mocker.GetMock<ICachedHttpResponseService>()
.Setup(x => x.Get<List<SearchJsonResource>>(It.IsAny<HttpRequest>(), It.IsAny<bool>(), It.IsAny<TimeSpan>()))
.Returns((HttpRequest request, bool useCache, TimeSpan ttl) => httpClient.Get<List<SearchJsonResource>>(request));
var metadataProfile = new MetadataProfile();
Mocker.GetMock<IMetadataProfileService>()
.Setup(s => s.All())
.Returns(new List<MetadataProfile> { metadataProfile });
Mocker.GetMock<IMetadataProfileService>()
.Setup(s => s.Get(It.IsAny<int>()))
.Returns(metadataProfile);
}
[TestCase("Robert Harris", "Robert Harris")]
[TestCase("James Patterson", "James Patterson")]
[TestCase("Antoine de Saint-Exupéry", "Antoine de Saint-Exupéry")]
public void successful_author_search(string title, string expected)
{
var result = Subject.SearchForNewAuthor(title);
result.Should().NotBeEmpty();
result[0].Name.Should().Be(expected);
ExceptionVerification.IgnoreWarns();
}
[TestCase("Harry Potter and the sorcerer's stone", null, "Harry Potter and the Sorcerer's Stone")]
[TestCase("edition:3", null, "Harry Potter and the Sorcerer's Stone")]
[TestCase("edition: 3", null, "Harry Potter and the Sorcerer's Stone")]
[TestCase("asin:B0192CTMYG", null, "Harry Potter and the Sorcerer's Stone")]
[TestCase("isbn:9780439554930", null, "Harry Potter and the Sorcerer's Stone")]
public void successful_book_search(string title, string author, string expected)
{
var result = Subject.SearchForNewBook(title, author, false);
result.Should().NotBeEmpty();
result[0].Editions.Value[0].Title.Should().Be(expected);
ExceptionVerification.IgnoreWarns();
ExceptionVerification.IgnoreErrors();
}
[TestCase("edition:")]
[TestCase("edition: 99999999999999999999")]
[TestCase("edition: 0")]
[TestCase("edition: -12")]
[TestCase("edition: aaaa")]
[TestCase("adjalkwdjkalwdjklawjdlKAJD")]
public void no_author_search_result(string term)
{
var result = Subject.SearchForNewAuthor(term);
result.Should().BeEmpty();
ExceptionVerification.IgnoreWarns();
}
[TestCase("Philip Pullman", 0, typeof(Author), "Philip Pullman")]
[TestCase("Philip Pullman", 1, typeof(Book), "Northern Lights")]
public void successful_combined_search(string query, int position, Type resultType, string expected)
{
var result = Subject.SearchForNewEntity(query);
result.Should().NotBeEmpty();
result[position].GetType().Should().Be(resultType);
if (resultType == typeof(Author))
{
var cast = result[position] as Author;
cast.Should().NotBeNull();
cast.Name.Should().Be(expected);
}
else
{
var cast = result[position] as Book;
cast.Should().NotBeNull();
cast.Title.Should().Be(expected);
}
}
}
}

93
src/NzbDrone.Core.Test/MetadataSource/Goodreads/GoodreadsProxySearchFixture.cs
View file

@ -1,15 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Common.Http;
using NzbDrone.Core.Books;
using NzbDrone.Core.Http;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.MetadataSource.Goodreads;
using NzbDrone.Core.Profiles.Metadata;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Test.Common;
@ -23,52 +19,35 @@ public void Setup()
{
UseRealHttp();
Mocker.SetConstant<IProvideBookInfo>(Mocker.Resolve<GoodreadsProxy>());
var httpClient = Mocker.Resolve<IHttpClient>();
Mocker.GetMock<ICachedHttpResponseService>()
.Setup(x => x.Get<List<SearchJsonResource>>(It.IsAny<HttpRequest>(), It.IsAny<bool>(), It.IsAny<TimeSpan>()))
.Returns((HttpRequest request, bool useCache, TimeSpan ttl) => httpClient.Get<List<SearchJsonResource>>(request));
var metadataProfile = new MetadataProfile();
Mocker.GetMock<IMetadataProfileService>()
.Setup(s => s.All())
.Returns(new List<MetadataProfile> { metadataProfile });
Mocker.GetMock<IMetadataProfileService>()
.Setup(s => s.Get(It.IsAny<int>()))
.Returns(metadataProfile);
.Returns((HttpRequest request, bool useCache, TimeSpan ttl) => Mocker.Resolve<IHttpClient>().Get<List<SearchJsonResource>>(request));
}
[TestCase("Robert Harris", "Robert Harris")]
[TestCase("James Patterson", "James Patterson")]
[TestCase("Antoine de Saint-Exupéry", "Antoine de Saint-Exupéry")]
public void successful_author_search(string title, string expected)
[TestCase("Robert Harris", 575)]
[TestCase("James Patterson", 3780)]
[TestCase("Antoine de Saint-Exupéry", 1020792)]
public void successful_author_search(string title, int expected)
{
var result = Subject.SearchForNewAuthor(title);
var result = Subject.Search(title);
result.Should().NotBeEmpty();
result[0].Name.Should().Be(expected);
result[0].Author.Id.Should().Be(expected);
ExceptionVerification.IgnoreWarns();
}
[TestCase("Harry Potter and the sorcerer's stone", null, "Harry Potter and the Sorcerer's Stone")]
[TestCase("readarr:3", null, "Harry Potter and the Philosopher's Stone")]
[TestCase("readarr: 3", null, "Harry Potter and the Philosopher's Stone")]
[TestCase("readarrid:3", null, "Harry Potter and the Philosopher's Stone")]
[TestCase("goodreads:3", null, "Harry Potter and the Philosopher's Stone")]
[TestCase("asin:B0192CTMYG", null, "Harry Potter and the Sorcerer's Stone")]
[TestCase("isbn:9780439554930", null, "Harry Potter and the Sorcerer's Stone")]
public void successful_book_search(string title, string author, string expected)
[TestCase("Harry Potter and the sorcerer's stone", 3)]
[TestCase("B0192CTMYG", 42844155)]
[TestCase("9780439554930", 48517161)]
public void successful_book_search(string title, int expected)
{
var result = Subject.SearchForNewBook(title, author);
var result = Subject.Search(title);
result.Should().NotBeEmpty();
result[0].Title.Should().Be(expected);
result[0].BookId.Should().Be(expected);
ExceptionVerification.IgnoreWarns();
}
@ -81,54 +60,10 @@ public void successful_book_search(string title, string author, string expected)
[TestCase("adjalkwdjkalwdjklawjdlKAJD")]
public void no_author_search_result(string term)
{
var result = Subject.SearchForNewAuthor(term);
var result = Subject.Search(term);
result.Should().BeEmpty();
ExceptionVerification.IgnoreWarns();
}
[TestCase("Philip Pullman", 0, typeof(Author), "Philip Pullman")]
[TestCase("Philip Pullman", 1, typeof(Book), "The Golden Compass")]
public void successful_combined_search(string query, int position, Type resultType, string expected)
{
var result = Subject.SearchForNewEntity(query);
result.Should().NotBeEmpty();
result[position].GetType().Should().Be(resultType);
if (resultType == typeof(Author))
{
var cast = result[position] as Author;
cast.Should().NotBeNull();
cast.Name.Should().Be(expected);
}
else
{
var cast = result[position] as Book;
cast.Should().NotBeNull();
cast.Title.Should().Be(expected);
}
}
[TestCase("B01N390U59", "The Book of Dust", "1")]
[TestCase("B0191WS1EE", "October Daye", "9.3")]
public void should_parse_series_from_title(string query, string series, string position)
{
var result = Subject.SearchByField("field", query);
var link = result.First().SeriesLinks.Value.First();
link.Series.Value.Title.Should().Be(series);
link.Position.Should().Be(position);
}
[TestCase("Imperium: A Novel of Ancient Rome (Cicero, #1)", "Imperium: A Novel of Ancient Rome", "Cicero", "1")]
[TestCase("Sons of Valor (The Tier One Shared-World Series Book 1)", "Sons of Valor", "Tier One Shared-World", "1")]
public void should_map_series_for_search(string title, string titleWithoutSeries, string series, string position)
{
var result = GoodreadsProxy.MapSearchSeries(title, titleWithoutSeries);
result.Should().HaveCount(1);
result[0].Series.Value.Title.Should().Be(series);
result[0].Position.Should().Be(position);
}
}
}

10
src/NzbDrone.Core.Test/MusicTests/AddAlbumFixture.cs
View file

@ -30,20 +30,20 @@ public void Setup()
.Build();
}
private void GivenValidBook(string readarrId)
private void GivenValidBook(string bookId, string editionId)
{
_fakeBook = Builder<Book>
.CreateNew()
.With(x => x.Editions = Builder<Edition>
.CreateListOfSize(1)
.TheFirst(1)
.With(e => e.ForeignEditionId = readarrId)
.With(e => e.ForeignEditionId = editionId)
.With(e => e.Monitored = true)
.BuildList())
.Build();
Mocker.GetMock<IProvideBookInfo>()
.Setup(s => s.GetBookInfo(readarrId, true))
.Setup(s => s.GetBookInfo(bookId))
.Returns(Tuple.Create(_fakeAuthor.Metadata.Value.ForeignAuthorId,
_fakeBook,
new List<AuthorMetadata> { _fakeAuthor.Metadata.Value }));
@ -85,7 +85,7 @@ public void should_be_able_to_add_a_book_without_passing_in_name()
{
var newBook = BookToAdd("edition", "book", "author");
GivenValidBook("edition");
GivenValidBook("book", "edition");
GivenValidPath();
var book = Subject.AddBook(newBook);
@ -99,7 +99,7 @@ public void should_throw_if_book_cannot_be_found()
var newBook = BookToAdd("edition", "book", "author");
Mocker.GetMock<IProvideBookInfo>()
.Setup(s => s.GetBookInfo("edition", true))
.Setup(s => s.GetBookInfo("book"))
.Throws(new BookNotFoundException("edition"));
Assert.Throws<ValidationException>(() => Subject.AddBook(newBook));

8
src/NzbDrone.Core.Test/MusicTests/AddArtistFixture.cs
View file

@ -28,12 +28,16 @@ public void Setup()
.With(s => s.Path = null)
.Build();
_fakeAuthor.Books = new List<Book>();
Mocker.GetMock<IAuthorService>()
.Setup(s => s.AddAuthor(It.IsAny<Author>(), It.IsAny<bool>()))
.Returns<Author, bool>((author, _) => author);
}
private void GivenValidAuthor(string readarrId)
{
Mocker.GetMock<IProvideAuthorInfo>()
.Setup(s => s.GetAuthorInfo(readarrId, true, false))
.Setup(s => s.GetAuthorInfo(readarrId, false))
.Returns(_fakeAuthor);
}
@ -113,7 +117,7 @@ public void should_throw_if_author_cannot_be_found()
};
Mocker.GetMock<IProvideAuthorInfo>()
.Setup(s => s.GetAuthorInfo(newAuthor.ForeignAuthorId, true, false))
.Setup(s => s.GetAuthorInfo(newAuthor.ForeignAuthorId, false))
.Throws(new AuthorNotFoundException(newAuthor.ForeignAuthorId));
Mocker.GetMock<IAddAuthorValidator>()

4
src/NzbDrone.Core.Test/MusicTests/RefreshArtistServiceFixture.cs
View file

@ -65,7 +65,7 @@ public void Setup()
.Returns(_remoteBooks);
Mocker.GetMock<IProvideAuthorInfo>()
.Setup(s => s.GetAuthorAndBooks(It.IsAny<string>(), It.IsAny<double>()))
.Setup(s => s.GetAuthorInfo(It.IsAny<string>(), true))
.Callback(() => { throw new AuthorNotFoundException(_author.ForeignAuthorId); });
Mocker.GetMock<IMediaFileService>()
@ -92,7 +92,7 @@ public void Setup()
private void GivenNewAuthorInfo(Author author)
{
Mocker.GetMock<IProvideAuthorInfo>()
.Setup(s => s.GetAuthorAndBooks(_author.ForeignAuthorId, It.IsAny<double>()))
.Setup(s => s.GetAuthorInfo(_author.ForeignAuthorId, true))
.Returns(author);
}

2
src/NzbDrone.Core/Books/Model/Book.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text.Json.Serialization;
using Equ;
using NzbDrone.Common.Extensions;
@ -8,6 +9,7 @@
namespace NzbDrone.Core.Books
{
[DebuggerDisplay("{GetType().FullName} ID = {Id} [{ForeignBookId}][{Title}]")]
public class Book : Entity<Book>
{
public Book()

2
src/NzbDrone.Core/Books/Model/Series.cs
View file

@ -1,10 +1,12 @@
using System.Collections.Generic;
using System.Diagnostics;
using Equ;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Datastore;
namespace NzbDrone.Core.Books
{
[DebuggerDisplay("{GetType().FullName} ID = {Id} [{ForeignSeriesId}][{Title}]")]
public class Series : Entity<Series>
{
public string ForeignSeriesId { get; set; }

6
src/NzbDrone.Core/Books/Services/AddAuthorService.cs
View file

@ -58,9 +58,7 @@ public Author AddAuthor(Author newAuthor, bool doRefresh = true)
newAuthor.AuthorMetadataId = newAuthor.Metadata.Value.Id;
// add the author itself
_authorService.AddAuthor(newAuthor, doRefresh);
return newAuthor;
return _authorService.AddAuthor(newAuthor, doRefresh);
}
public List<Author> AddAuthors(List<Author> newAuthors, bool doRefresh = true)
@ -97,7 +95,7 @@ private Author AddSkyhookData(Author newAuthor)
try
{
author = _authorInfo.GetAuthorInfo(newAuthor.Metadata.Value.ForeignAuthorId, includeBooks: false);
author = _authorInfo.GetAuthorInfo(newAuthor.Metadata.Value.ForeignAuthorId, false);
}
catch (AuthorNotFoundException)
{

12
src/NzbDrone.Core/Books/Services/AddBookService.cs
View file

@ -44,16 +44,13 @@ public Book AddBook(Book book, bool doRefresh = true)
{
_logger.Debug($"Adding book {book}");
book = AddSkyhookData(book);
// we allow adding extra editions, so check if the book already exists
var dbBook = _bookService.FindById(book.ForeignBookId);
if (dbBook != null)
{
dbBook.Editions = book.Editions;
book = dbBook;
}
else
{
book = AddSkyhookData(book);
book.UseDbFieldsFrom(dbBook);
}
// Remove any import list exclusions preventing addition
@ -107,10 +104,11 @@ public List<Book> AddBooks(List<Book> books, bool doRefresh = true)
private Book AddSkyhookData(Book newBook)
{
var editionId = newBook.Editions.Value.Single(x => x.Monitored).ForeignEditionId;
Tuple<string, Book, List<AuthorMetadata>> tuple = null;
try
{
tuple = _bookInfo.GetBookInfo(editionId);
tuple = _bookInfo.GetBookInfo(newBook.ForeignBookId);
}
catch (BookNotFoundException)
{

8
src/NzbDrone.Core/Books/Services/RefreshAuthorService.cs
View file

@ -84,11 +84,11 @@ public RefreshAuthorService(IProvideAuthorInfo authorInfo,
_logger = logger;
}
private Author GetSkyhookData(string foreignId, double minPopularity)
private Author GetSkyhookData(string foreignId)
{
try
{
return _authorInfo.GetAuthorAndBooks(foreignId, minPopularity);
return _authorInfo.GetAuthorInfo(foreignId);
}
catch (AuthorNotFoundException)
{
@ -347,7 +347,7 @@ private void RefreshSelectedAuthors(List<int> authorIds, bool isNew, CommandTrig
{
try
{
var data = GetSkyhookData(author.ForeignAuthorId, author.MetadataProfile.Value.MinPopularity);
var data = GetSkyhookData(author.ForeignAuthorId);
updated |= RefreshEntityInfo(author, null, data, true, false, null);
}
catch (Exception e)
@ -397,7 +397,7 @@ public void Execute(RefreshAuthorCommand message)
try
{
LogProgress(author);
var data = GetSkyhookData(author.ForeignAuthorId, author.MetadataProfile.Value.MinPopularity);
var data = GetSkyhookData(author.ForeignAuthorId);
updated |= RefreshEntityInfo(author, null, data, manualTrigger, false, message.LastStartTime);
}
catch (Exception e)

16
src/NzbDrone.Core/Books/Services/RefreshBookService.cs
View file

@ -75,12 +75,10 @@ public RefreshBookService(IBookService bookService,
private Author GetSkyhookData(Book book)
{
var foreignId = book.Editions.Value.First().ForeignEditionId;
try
{
var tuple = _bookInfo.GetBookInfo(foreignId, false);
var author = _authorInfo.GetAuthorInfo(tuple.Item1, false);
var tuple = _bookInfo.GetBookInfo(book.ForeignBookId);
var author = _authorInfo.GetAuthorInfo(tuple.Item1);
var newbook = tuple.Item2;
newbook.Author = author;
@ -88,19 +86,12 @@ private Author GetSkyhookData(Book book)
newbook.AuthorMetadataId = book.AuthorMetadataId;
newbook.AuthorMetadata.Value.Id = book.AuthorMetadataId;
// make sure to grab editions data for any other existing editions
foreach (var edition in book.Editions.Value.Skip(1))
{
tuple = _bookInfo.GetBookInfo(edition.ForeignEditionId, false);
newbook.Editions.Value.AddRange(tuple.Item2.Editions.Value);
}
author.Books = new List<Book> { newbook };
return author;
}
catch (BookNotFoundException)
{
_logger.Error($"Could not find book with id {foreignId}");
_logger.Error($"Could not find book with id {book.ForeignBookId}");
}
return null;
@ -119,6 +110,7 @@ protected override RemoteData GetRemoteData(Book local, List<Book> remote, Autho
if (book == null)
{
data = GetSkyhookData(local);
book = data.Books.Value.SingleOrDefault(x => x.ForeignBookId == local.ForeignBookId);
}

4
src/NzbDrone.Core/Datastore/ModelBase.cs
View file

@ -1,8 +1,8 @@
using System.Diagnostics;
using System.Diagnostics;
namespace NzbDrone.Core.Datastore
{
[DebuggerDisplay("{GetType()} ID = {Id}")]
[DebuggerDisplay("{GetType().FullName} ID = {Id}")]
public abstract class ModelBase
{
public int Id { get; set; }

2
src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs
View file

@ -18,6 +18,8 @@ public void Clean()
{
mapper.Execute(@"DELETE FROM HttpResponse WHERE Expiry < date('now')");
}
_database.Vacuum();
}
}
}

7
src/NzbDrone.Core/Http/CachedHttpResponseService.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Net;
using NLog;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.Http
@ -15,12 +16,15 @@ public class CachedHttpResponseService : ICachedHttpResponseService
{
private readonly ICachedHttpResponseRepository _repo;
private readonly IHttpClient _httpClient;
private readonly Logger _logger;
public CachedHttpResponseService(ICachedHttpResponseRepository httpResponseRepository,
IHttpClient httpClient)
IHttpClient httpClient,
Logger logger)
{
_repo = httpResponseRepository;
_httpClient = httpClient;
_logger = logger;
}
public HttpResponse Get(HttpRequest request, bool useCache, TimeSpan ttl)
@ -29,6 +33,7 @@ public HttpResponse Get(HttpRequest request, bool useCache, TimeSpan ttl)
if (useCache && cached != null && cached.Expiry > DateTime.UtcNow)
{
_logger.Trace($"Returning cached response for [GET] {request.Url}");
return new HttpResponse(request, new HttpHeader(), cached.Value, (HttpStatusCode)cached.StatusCode);
}

97
src/NzbDrone.Core/ImportLists/ImportListSyncService.cs
View file

@ -6,11 +6,12 @@
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.Books;
using NzbDrone.Core.Books.Commands;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.ImportLists.Exclusions;
using NzbDrone.Core.IndexerSearch;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.MetadataSource.Goodreads;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.ImportLists
@ -20,10 +21,11 @@ public class ImportListSyncService : IExecute<ImportListSyncCommand>
private readonly IImportListFactory _importListFactory;
private readonly IImportListExclusionService _importListExclusionService;
private readonly IFetchAndParseImportList _listFetcherAndParser;
private readonly ISearchForNewBook _bookSearchService;
private readonly ISearchForNewAuthor _authorSearchService;
private readonly IGoodreadsProxy _goodreadsProxy;
private readonly IGoodreadsSearchProxy _goodreadsSearchProxy;
private readonly IAuthorService _authorService;
private readonly IBookService _bookService;
private readonly IEditionService _editionService;
private readonly IAddAuthorService _addAuthorService;
private readonly IAddBookService _addBookService;
private readonly IEventAggregator _eventAggregator;
@ -33,10 +35,11 @@ public class ImportListSyncService : IExecute<ImportListSyncCommand>
public ImportListSyncService(IImportListFactory importListFactory,
IImportListExclusionService importListExclusionService,
IFetchAndParseImportList listFetcherAndParser,
ISearchForNewBook bookSearchService,
ISearchForNewAuthor authorSearchService,
IGoodreadsProxy goodreadsProxy,
IGoodreadsSearchProxy goodreadsSearchProxy,
IAuthorService authorService,
IBookService bookService,
IEditionService editionService,
IAddAuthorService addAuthorService,
IAddBookService addBookService,
IEventAggregator eventAggregator,
@ -46,10 +49,11 @@ public ImportListSyncService(IImportListFactory importListFactory,
_importListFactory = importListFactory;
_importListExclusionService = importListExclusionService;
_listFetcherAndParser = listFetcherAndParser;
_bookSearchService = bookSearchService;
_authorSearchService = authorSearchService;
_goodreadsProxy = goodreadsProxy;
_goodreadsSearchProxy = goodreadsSearchProxy;
_authorService = authorService;
_bookService = bookService;
_editionService = editionService;
_addAuthorService = addAuthorService;
_addBookService = addBookService;
_eventAggregator = eventAggregator;
@ -137,32 +141,55 @@ private List<Book> ProcessReports(List<ImportListItemInfo> reports)
private void MapBookReport(ImportListItemInfo report)
{
Book mappedBook;
if (report.EditionGoodreadsId.IsNotNullOrWhiteSpace() && int.TryParse(report.EditionGoodreadsId, out var goodreadsId))
{
var search = _bookSearchService.SearchByGoodreadsId(goodreadsId);
mappedBook = search.FirstOrDefault(x => x.Editions.Value.Any(e => int.TryParse(e.ForeignEditionId, out var editionId) && editionId == goodreadsId));
// check the local DB
var edition = _editionService.GetEditionByForeignEditionId(report.EditionGoodreadsId);
if (edition != null)
{
var book = edition.Book.Value;
report.BookGoodreadsId = book.ForeignBookId;
report.Book = edition.Title;
report.Author ??= book.AuthorMetadata.Value.Name;
report.AuthorGoodreadsId ??= book.AuthorMetadata.Value.ForeignAuthorId;
return;
}
try
{
var remoteBook = _goodreadsProxy.GetBookInfo(report.EditionGoodreadsId);
_logger.Trace($"Mapped {report.EditionGoodreadsId} to [{remoteBook.ForeignBookId}] {remoteBook.Title}");
report.BookGoodreadsId = remoteBook.ForeignBookId;
report.Book = remoteBook.Title;
report.Author ??= remoteBook.AuthorMetadata.Value.Name;
report.AuthorGoodreadsId ??= remoteBook.AuthorMetadata.Value.Name;
}
catch (BookNotFoundException)
{
_logger.Debug($"Nothing found for edition [{report.EditionGoodreadsId}]");
report.EditionGoodreadsId = null;
}
}
else
{
mappedBook = _bookSearchService.SearchForNewBook(report.Book, report.Author).FirstOrDefault();
var mappedBook = _goodreadsSearchProxy.Search($"{report.Book} {report.Author}").FirstOrDefault();
if (mappedBook == null)
{
_logger.Trace($"Nothing found for {report.Author} - {report.Book}");
return;
}
_logger.Trace($"Mapped {report.EditionGoodreadsId} to [{mappedBook.WorkId}] {mappedBook.BookTitleBare}");
report.BookGoodreadsId = mappedBook.WorkId.ToString();
report.Book = mappedBook.BookTitleBare;
report.Author ??= mappedBook.Author.Name;
report.AuthorGoodreadsId ??= mappedBook.Author.Id.ToString();
}
// Break if we are looking for a book and cant find it. This will avoid us from adding the author and possibly getting it wrong.
if (mappedBook == null)
{
_logger.Trace($"Nothing found for {report.EditionGoodreadsId}");
report.EditionGoodreadsId = null;
return;
}
_logger.Trace($"Mapped {report.EditionGoodreadsId} to {mappedBook}");
report.BookGoodreadsId = mappedBook.ForeignBookId;
report.Book = mappedBook.Title;
report.Author ??= mappedBook.AuthorMetadata?.Value?.Name;
report.AuthorGoodreadsId ??= mappedBook.AuthorMetadata?.Value?.ForeignAuthorId;
}
private void ProcessBookReport(ImportListDefinition importList, ImportListItemInfo report, List<ImportListExclusion> listExclusions, List<Book> booksToAdd, List<Author> authorsToAdd)
@ -297,10 +324,18 @@ private void ProcessBookReport(ImportListDefinition importList, ImportListItemIn
private void MapAuthorReport(ImportListItemInfo report)
{
var mappedAuthor = _authorSearchService.SearchForNewAuthor(report.Author)
.FirstOrDefault();
report.AuthorGoodreadsId = mappedAuthor?.Metadata.Value?.ForeignAuthorId;
report.Author = mappedAuthor?.Metadata.Value?.Name;
var mappedBook = _goodreadsSearchProxy.Search(report.Author).FirstOrDefault();
if (mappedBook == null)
{
_logger.Trace($"Nothing found for {report.Author}");
return;
}
_logger.Trace($"Mapped {report.Author} to [{mappedBook.Author.Name}]");
report.Author = mappedBook.Author.Name;
report.AuthorGoodreadsId = mappedBook.Author.Id.ToString();
}
private Author ProcessAuthorReport(ImportListDefinition importList, ImportListItemInfo report, List<ImportListExclusion> listExclusions, List<Author> authorsToAdd)

2
src/NzbDrone.Core/Localization/Core/en.json
View file

@ -582,7 +582,7 @@
"Search": "Search",
"SearchAll": "Search All",
"SearchBook": "Search Book",
"SearchBoxPlaceHolder": "eg. War and Peace, goodreads:656, isbn:067003469X, asin:B00JCDK5ME",
"SearchBoxPlaceHolder": "eg. War and Peace, edition:656, work:4912783, author:128382, isbn:067003469X, asin:B00JCDK5ME",
"SearchForAllCutoffUnmetBooks": "Search for all Cutoff Unmet books",
"SearchForAllMissingBooks": "Search for all missing books",
"SearchForMissing": "Search for Missing",

2
src/NzbDrone.Core/MediaFiles/BookImport/Identification/CandidateService.cs
View file

@ -254,7 +254,7 @@ public IEnumerable<CandidateEdition> GetRemoteCandidates(LocalEdition localEditi
try
{
remoteBooks = _bookSearchService.SearchByGoodreadsId(id);
remoteBooks = _bookSearchService.SearchByGoodreadsBookId(id, true);
}
catch (GoodreadsException e)
{

13
src/NzbDrone.Core/MediaFiles/BookImport/Identification/IdentificationService.cs
View file

@ -141,7 +141,9 @@ private void IdentifyRelease(LocalEdition localBookRelease, IdentificationOverri
usedRemote = true;
}
if (!candidateReleases.Any())
GetBestRelease(localBookRelease, candidateReleases, allLocalTracks, out var seenCandidate);
if (!seenCandidate)
{
// can't find any candidates even after using remote search
// populate the overrides and return
@ -155,8 +157,6 @@ private void IdentifyRelease(LocalEdition localBookRelease, IdentificationOverri
return;
}
GetBestRelease(localBookRelease, candidateReleases, allLocalTracks);
// If the result isn't great and we haven't tried remote candidates, try looking for remote candidates
// Goodreads may have a better edition of a local book
if (localBookRelease.Distance.NormalizedDistance() > 0.15 && !usedRemote)
@ -169,7 +169,7 @@ private void IdentifyRelease(LocalEdition localBookRelease, IdentificationOverri
candidateReleases = candidateReleases.Where(x => x.Edition.Book.Value.Id > 0);
}
GetBestRelease(localBookRelease, candidateReleases, allLocalTracks);
GetBestRelease(localBookRelease, candidateReleases, allLocalTracks, out _);
}
_logger.Debug($"Best release found in {watch.ElapsedMilliseconds}ms");
@ -179,7 +179,7 @@ private void IdentifyRelease(LocalEdition localBookRelease, IdentificationOverri
_logger.Debug($"IdentifyRelease done in {watch.ElapsedMilliseconds}ms");
}
private void GetBestRelease(LocalEdition localBookRelease, IEnumerable<CandidateEdition> candidateReleases, List<LocalBook> extraTracksOnDisk)
private void GetBestRelease(LocalEdition localBookRelease, IEnumerable<CandidateEdition> candidateReleases, List<LocalBook> extraTracksOnDisk, out bool seenCandidate)
{
var watch = System.Diagnostics.Stopwatch.StartNew();
@ -187,9 +187,12 @@ private void GetBestRelease(LocalEdition localBookRelease, IEnumerable<Candidate
_logger.Trace("Processing files:\n{0}", string.Join("\n", localBookRelease.LocalBooks.Select(x => x.Path)));
double bestDistance = 1.0;
seenCandidate = false;
foreach (var candidateRelease in candidateReleases)
{
seenCandidate = true;
var release = candidateRelease.Edition;
_logger.Debug($"Trying Release {release}");
var rwatch = System.Diagnostics.Stopwatch.StartNew();

3
src/NzbDrone.Core/MediaFiles/BookImport/ImportApprovedBooks.cs
View file

@ -346,6 +346,9 @@ private Author EnsureAuthorAdded(List<ImportDecision<LocalBook>> decisions, List
try
{
dbAuthor = _addAuthorService.AddAuthor(author, false);
// this looks redundant but is necessary to get the LazyLoads populated
dbAuthor = _authorService.GetAuthor(dbAuthor.Id);
addedAuthors.Add(dbAuthor);
}
catch (Exception e)

2
src/NzbDrone.Core/MediaFiles/BookImport/Manual/ManualImportService.cs
View file

@ -308,7 +308,7 @@ public void Execute(ManualImportCommand message)
var edition = _editionService.GetEditionByForeignEditionId(file.ForeignEditionId);
if (edition == null)
{
var tuple = _bookInfo.GetBookInfo(file.ForeignEditionId);
var tuple = _bookInfo.GetBookInfo(book.ForeignBookId);
edition = tuple.Item2.Editions.Value.SingleOrDefault(x => x.ForeignEditionId == file.ForeignEditionId);
}

10
src/NzbDrone.Core/MediaFiles/EpubTag/Readers/PackageReader.cs
View file

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Threading.Tasks;
@ -27,7 +27,12 @@ public static async Task<EpubPackage> ReadPackageAsync(ZipArchive epubArchive, s
XNamespace opfNamespace = "http://www.idpf.org/2007/opf";
var packageNode = containerDocument.Element(opfNamespace + "package");
var result = new EpubPackage();
if (packageNode == null)
{
throw new Exception("Invalid epub file");
}
var epubVersionValue = packageNode.Attribute("version").Value;
EpubVersion epubVersion;
switch (epubVersionValue)
@ -46,6 +51,7 @@ public static async Task<EpubPackage> ReadPackageAsync(ZipArchive epubArchive, s
throw new Exception($"Unsupported EPUB version: {epubVersionValue}.");
}
var result = new EpubPackage();
result.EpubVersion = epubVersion;
var metadataNode = packageNode.Element(opfNamespace + "metadata");
if (metadataNode == null)

580
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoProxy.cs
View file

@ -3,32 +3,59 @@
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Books;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http;
using NzbDrone.Core.MediaCover;
using NzbDrone.Core.MetadataSource.Goodreads;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class BookInfoProxy : IProvideAuthorInfo
public class BookInfoProxy : IProvideAuthorInfo, IProvideBookInfo, ISearchForNewBook, ISearchForNewAuthor, ISearchForNewEntity
{
private static readonly JsonSerializerOptions SerializerSettings = new JsonSerializerOptions
{
PropertyNameCaseInsensitive = false,
Converters = { new STJUtcConverter() }
};
private readonly IHttpClient _httpClient;
private readonly ICachedHttpResponseService _cachedHttpClient;
private readonly IGoodreadsSearchProxy _goodreadsSearchProxy;
private readonly IAuthorService _authorService;
private readonly IBookService _bookService;
private readonly IEditionService _editionService;
private readonly Logger _logger;
private readonly IMetadataRequestBuilder _requestBuilder;
private readonly ICached<HashSet<string>> _cache;
private readonly ICached<Author> _authorCache;
public BookInfoProxy(IHttpClient httpClient,
IMetadataRequestBuilder requestBuilder,
Logger logger,
ICacheManager cacheManager)
ICachedHttpResponseService cachedHttpClient,
IGoodreadsSearchProxy goodreadsSearchProxy,
IAuthorService authorService,
IBookService bookService,
IEditionService editionService,
IMetadataRequestBuilder requestBuilder,
Logger logger,
ICacheManager cacheManager)
{
_httpClient = httpClient;
_cachedHttpClient = cachedHttpClient;
_goodreadsSearchProxy = goodreadsSearchProxy;
_authorService = authorService;
_bookService = bookService;
_editionService = editionService;
_requestBuilder = requestBuilder;
_cache = cacheManager.GetCache<HashSet<string>>(GetType());
_authorCache = cacheManager.GetRollingCache<Author>(GetType(), "authorCache", TimeSpan.FromMinutes(5));
_logger = logger;
}
@ -51,17 +78,393 @@ public HashSet<string> GetChangedAuthors(DateTime startTime)
return new HashSet<string>(httpResponse.Resource.Ids.Select(x => x.ToString()));
}
public Author GetAuthorInfo(string foreignAuthorId, bool useCache = false, bool includeBooks = true)
public Author GetAuthorInfo(string foreignAuthorId, bool useCache = true)
{
_logger.Debug("Getting Author details GoodreadsId of {0}", foreignAuthorId);
return PollAuthor(foreignAuthorId, includeBooks);
if (useCache)
{
return PollAuthor(foreignAuthorId);
}
return PollAuthorUncached(foreignAuthorId);
}
private Author PollAuthor(string foreignAuthorId, bool includeBooks)
public HashSet<string> GetChangedBooks(DateTime startTime)
{
return _cache.Get("ChangedBooks", () => GetChangedBooksUncached(startTime), TimeSpan.FromMinutes(30));
}
private HashSet<string> GetChangedBooksUncached(DateTime startTime)
{
return null;
}
public Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string foreignBookId)
{
return PollBook(foreignBookId);
}
public List<object> SearchForNewEntity(string title)
{
var books = SearchForNewBook(title, null, false);
var result = new List<object>();
foreach (var book in books)
{
var author = book.Author.Value;
if (!result.Contains(author))
{
result.Add(author);
}
result.Add(book);
}
return result;
}
public List<Author> SearchForNewAuthor(string title)
{
var books = SearchForNewBook(title, null);
return books.Select(x => x.Author.Value).ToList();
}
public List<Book> SearchForNewBook(string title, string author, bool getAllEditions = true)
{
var q = title.ToLower().Trim();
if (author != null)
{
q += " " + author;
}
try
{
var lowerTitle = title.ToLowerInvariant();
var split = lowerTitle.Split(':');
var prefix = split[0];
if (split.Length == 2 && new[] { "author", "work", "edition", "isbn", "asin" }.Contains(prefix))
{
var slug = split[1].Trim();
if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace))
{
return new List<Book>();
}
if (prefix == "author" || prefix == "work" || prefix == "edition")
{
var isValid = int.TryParse(slug, out var searchId);
if (!isValid)
{
return new List<Book>();
}
if (prefix == "author")
{
return SearchByGoodreadsAuthorId(searchId);
}
if (prefix == "work")
{
return SearchByGoodreadsWorkId(searchId);
}
if (prefix == "edition")
{
return SearchByGoodreadsBookId(searchId, getAllEditions);
}
}
// to handle isbn / asin
q = slug;
}
return Search(q, getAllEditions);
}
catch (HttpException ex)
{
_logger.Warn(ex, ex.Message);
throw new GoodreadsException("Search for '{0}' failed. Unable to communicate with Goodreads.", title);
}
catch (Exception ex)
{
_logger.Warn(ex, ex.Message);
throw new GoodreadsException("Search for '{0}' failed. Invalid response received from Goodreads.",
title);
}
}
public List<Book> SearchByIsbn(string isbn)
{
return Search(isbn, true);
}
public List<Book> SearchByAsin(string asin)
{
return Search(asin, true);
}
private List<Book> Search(string query, bool getAllEditions)
{
var result = _goodreadsSearchProxy.Search(query);
var books = new List<Book>();
if (getAllEditions)
{
// Slower but more exhaustive, less intensive on metadata API
var bookIds = result.Select(x => x.WorkId).ToList();
var idMap = result.Select(x => new { AuthorId = x.Author.Id, BookId = x.WorkId })
.GroupBy(x => x.AuthorId)
.ToDictionary(x => x.Key, x => x.Select(i => i.BookId.ToString()).ToList());
List<Book> authorBooks;
foreach (var author in idMap.Keys)
{
authorBooks = SearchByGoodreadsAuthorId(author);
books.AddRange(authorBooks.Where(b => idMap[author].Contains(b.ForeignBookId)));
}
var missingBooks = bookIds.ExceptBy(x => x.ToString(), books, x => x.ForeignBookId, StringComparer.Ordinal).ToList();
foreach (var book in missingBooks)
{
books.AddRange(SearchByGoodreadsWorkId(book));
}
return books;
}
else
{
// Use sparingly, hits metadata API quite hard
var ids = result.Select(x => x.BookId).ToList();
if (ids.Count == 0)
{
return new List<Book>();
}
if (ids.Count == 1)
{
try
{
return SearchByGoodreadsBookId(ids[0], false);
}
catch (BookNotFoundException)
{
_logger.Debug($"Couldn't fetch book info for {ids[0]}");
return new List<Book>();
}
}
return MapSearchResult(ids);
}
}
private List<Book> SearchByGoodreadsAuthorId(int id)
{
try
{
var authorId = id.ToString();
var result = GetAuthorInfo(authorId);
var books = result.Books.Value;
var authors = new Dictionary<string, AuthorMetadata> { { authorId, result.Metadata.Value } };
foreach (var book in books)
{
AddDbIds(authorId, book, authors);
}
return books;
}
catch (AuthorNotFoundException)
{
return new List<Book>();
}
}
public List<Book> SearchByGoodreadsWorkId(int id)
{
try
{
var tuple = GetBookInfo(id.ToString());
AddDbIds(tuple.Item1, tuple.Item2, tuple.Item3.ToDictionary(x => x.ForeignAuthorId));
return new List<Book> { tuple.Item2 };
}
catch (BookNotFoundException)
{
return new List<Book>();
}
}
public List<Book> SearchByGoodreadsBookId(int id, bool getAllEditions)
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"book/{id}")
.Build();
httpRequest.SuppressHttpError = true;
// we expect a redirect
var httpResponse = _httpClient.Get(httpRequest);
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
return new List<Book>();
}
if (!httpResponse.HasHttpRedirect)
{
throw new BookInfoException($"Unexpected response from {httpRequest.Url}");
}
var location = httpResponse.Headers.GetSingleValue("Location");
var split = location.Split('/');
var type = split[0];
var newId = split[1];
Book book;
List<AuthorMetadata> authors;
if (type == "author")
{
var author = PollAuthor(newId);
book = author.Books.Value.Where(b => b.Editions.Value.Any(e => e.ForeignEditionId == id.ToString())).FirstOrDefault();
authors = new List<AuthorMetadata> { author.Metadata.Value };
}
else if (type == "book")
{
var tuple = PollBook(newId);
book = tuple.Item2;
authors = tuple.Item3;
}
else
{
throw new NotImplementedException($"Unexpected response from {httpResponse.Request.Url}");
}
if (book == null)
{
return new List<Book>();
}
if (!getAllEditions)
{
var trimmed = new Book();
trimmed.UseMetadataFrom(book);
trimmed.SeriesLinks = book.SeriesLinks;
var edition = book.Editions.Value.SingleOrDefault(e => e.ForeignEditionId == id.ToString());
if (edition == null)
{
return new List<Book>();
}
trimmed.Editions = new List<Edition> { edition };
return new List<Book> { trimmed };
}
var authorDict = authors.ToDictionary(x => x.ForeignAuthorId);
AddDbIds(book.AuthorMetadata.Value.ForeignAuthorId, book, authorDict);
return new List<Book> { book };
}
private List<Book> MapSearchResult(List<int> ids)
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"book/bulk")
.SetHeader("Content-Type", "application/json")
.Build();
httpRequest.SetContent(ids.ToJson());
httpRequest.AllowAutoRedirect = true;
var httpResponse = _httpClient.Post<BulkBookResource>(httpRequest);
var mapped = MapBulkBook(httpResponse.Resource);
var idStr = ids.Select(x => x.ToString()).ToList();
return mapped.OrderBy(b => idStr.IndexOf(b.Editions.Value.First().ForeignEditionId)).ToList();
}
private List<Book> MapBulkBook(BulkBookResource resource)
{
var authors = resource.Authors.Select(MapAuthorMetadata).ToDictionary(x => x.ForeignAuthorId, x => x);
var series = resource.Series.Select(MapSeries).ToList();
var books = new List<Book>();
foreach (var work in resource.Works)
{
var book = MapBook(work);
var authorId = work.Books.OrderByDescending(b => b.AverageRating * b.RatingCount).First().Contributors.First().ForeignId.ToString();
AddDbIds(authorId, book, authors);
books.Add(book);
}
MapSeriesLinks(series, books, resource.Series);
return books;
}
private void AddDbIds(string authorId, Book book, Dictionary<string, AuthorMetadata> authors)
{
var dbBook = _bookService.FindById(book.ForeignBookId);
if (dbBook != null)
{
book.UseDbFieldsFrom(dbBook);
var editions = _editionService.GetEditionsByBook(dbBook.Id).ToDictionary(x => x.ForeignEditionId);
foreach (var edition in book.Editions.Value)
{
if (editions.TryGetValue(edition.ForeignEditionId, out var dbEdition))
{
edition.UseDbFieldsFrom(dbEdition);
}
}
}
var author = _authorService.FindById(authorId);
if (author == null)
{
var metadata = authors[authorId];
author = new Author
{
CleanName = Parser.Parser.CleanAuthorName(metadata.Name),
Metadata = metadata
};
}
book.Author = author;
book.AuthorMetadata = author.Metadata.Value;
book.AuthorMetadataId = author.AuthorMetadataId;
}
private Author PollAuthor(string foreignAuthorId)
{
return _authorCache.Get(foreignAuthorId, () => PollAuthorUncached(foreignAuthorId));
}
private Author PollAuthorUncached(string foreignAuthorId)
{
AuthorResource resource = null;
var useCache = true;
for (var i = 0; i < 60; i++)
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
@ -71,7 +474,7 @@ private Author PollAuthor(string foreignAuthorId, bool includeBooks)
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _httpClient.Get<AuthorResource>(httpRequest);
var httpResponse = _cachedHttpClient.Get(httpRequest, useCache, TimeSpan.FromMinutes(30));
if (httpResponse.HasHttpError)
{
@ -89,15 +492,16 @@ private Author PollAuthor(string foreignAuthorId, bool includeBooks)
}
}
resource = httpResponse.Resource;
resource = JsonSerializer.Deserialize<AuthorResource>(httpResponse.Content, SerializerSettings);
if (resource.Works != null || !includeBooks)
if (resource.Works != null)
{
resource.Works ??= new List<WorkResource>();
resource.Series ??= new List<SeriesResource>();
break;
}
useCache = false;
Thread.Sleep(2000);
}
@ -109,27 +513,91 @@ private Author PollAuthor(string foreignAuthorId, bool includeBooks)
return MapAuthor(resource);
}
public Author GetAuthorAndBooks(string foreignAuthorId, double minPopularity = 0)
private Tuple<string, Book, List<AuthorMetadata>> PollBook(string foreignBookId)
{
return GetAuthorInfo(foreignAuthorId);
WorkResource resource = null;
for (var i = 0; i < 60; i++)
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"work/{foreignBookId}")
.Build();
httpRequest.SuppressHttpError = true;
// this may redirect to an author
var httpResponse = _httpClient.Get(httpRequest);
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new BookNotFoundException(foreignBookId);
}
if (httpResponse.HasHttpRedirect)
{
var location = httpResponse.Headers.GetSingleValue("Location");
var split = location.Split('/');
var type = split[0];
var newId = split[1];
if (type == "author")
{
var author = PollAuthor(newId);
var authorBook = author.Books.Value.SingleOrDefault(x => x.ForeignBookId == foreignBookId);
if (authorBook == null)
{
throw new BookNotFoundException(foreignBookId);
}
var authorMetadata = new List<AuthorMetadata> { author.Metadata.Value };
return Tuple.Create(author.ForeignAuthorId, authorBook, authorMetadata);
}
else
{
throw new NotImplementedException($"Unexpected response from {httpResponse.Request.Url}");
}
}
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignBookId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
resource = JsonSerializer.Deserialize<WorkResource>(httpResponse.Content, SerializerSettings);
if (resource.Books != null)
{
break;
}
Thread.Sleep(2000);
}
if (resource?.Books == null)
{
throw new BookInfoException($"Failed to get books for {foreignBookId}");
}
var book = MapBook(resource);
var authorId = resource.Books.OrderByDescending(x => x.AverageRating * x.RatingCount).First().Contributors.First().ForeignId.ToString();
var metadata = resource.Authors.Select(MapAuthorMetadata).ToList();
var series = resource.Series.Select(MapSeries).ToList();
MapSeriesLinks(series, new List<Book> { book }, resource.Series);
return Tuple.Create(authorId, book, metadata);
}
public HashSet<string> GetChangedBooks(DateTime startTime)
{
return _cache.Get("ChangedBooks", () => GetChangedBooksUncached(startTime), TimeSpan.FromMinutes(30));
}
private HashSet<string> GetChangedBooksUncached(DateTime startTime)
{
return null;
}
public Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string foreignBookId)
{
return null;
}
private Author MapAuthor(AuthorResource resource)
private static AuthorMetadata MapAuthorMetadata(AuthorResource resource)
{
var metadata = new AuthorMetadata
{
@ -159,6 +627,13 @@ private Author MapAuthor(AuthorResource resource)
metadata.Links.Add(new Links { Url = resource.Url, Name = "Goodreads" });
}
return metadata;
}
private static Author MapAuthor(AuthorResource resource)
{
var metadata = MapAuthorMetadata(resource);
var books = resource.Works
.Where(x => x.ForeignId > 0 && GetAuthorId(x) == resource.ForeignId)
.Select(MapBook)
@ -168,7 +643,7 @@ private Author MapAuthor(AuthorResource resource)
var series = resource.Series.Select(MapSeries).ToList();
MapSeriesLinks(series, books, resource);
MapSeriesLinks(series, books, resource.Series);
var result = new Author
{
@ -181,17 +656,22 @@ private Author MapAuthor(AuthorResource resource)
return result;
}
private static void MapSeriesLinks(List<Series> series, List<Book> books, AuthorResource resource)
private static void MapSeriesLinks(List<Series> series, List<Book> books, List<SeriesResource> resource)
{
var bookDict = books.ToDictionary(x => x.ForeignBookId);
var seriesDict = series.ToDictionary(x => x.ForeignSeriesId);
foreach (var book in books)
{
book.SeriesLinks = new List<SeriesBookLink>();
}
// only take series where there are some works
foreach (var s in resource.Series.Where(x => x.LinkItems.Any()))
foreach (var s in resource.Where(x => x.LinkItems.Any()))
{
if (seriesDict.TryGetValue(s.ForeignId.ToString(), out var curr))
{
curr.LinkItems = s.LinkItems.Where(x => x.ForeignWorkId.IsNotNullOrWhiteSpace() && bookDict.ContainsKey(x.ForeignWorkId.ToString())).Select(l => new SeriesBookLink
curr.LinkItems = s.LinkItems.Where(x => x.ForeignWorkId != 0 && bookDict.ContainsKey(x.ForeignWorkId.ToString())).Select(l => new SeriesBookLink
{
Book = bookDict[l.ForeignWorkId.ToString()],
Series = curr,
@ -199,6 +679,11 @@ private static void MapSeriesLinks(List<Series> series, List<Book> books, Author
Position = l.PositionInSeries,
SeriesPosition = l.SeriesPosition
}).ToList();
foreach (var l in curr.LinkItems.Value)
{
l.Book.Value.SeriesLinks.Value.Add(l);
}
}
}
}
@ -234,8 +719,8 @@ private static Book MapBook(WorkResource resource)
{
book.Editions = resource.Books.Select(x => MapEdition(x)).ToList();
// monitor the most rated release
var mostPopular = book.Editions.Value.OrderByDescending(x => x.Ratings.Votes).FirstOrDefault();
// monitor the most popular release
var mostPopular = book.Editions.Value.OrderByDescending(x => x.Ratings.Popularity).FirstOrDefault();
if (mostPopular != null)
{
mostPopular.Monitored = true;
@ -252,17 +737,24 @@ private static Book MapBook(WorkResource resource)
book.Editions = new List<Edition>();
}
// sometimes the work release date is after the earliest good edition release
var editionReleases = book.Editions.Value
.Where(x => x.ReleaseDate.HasValue && x.ReleaseDate.Value.Month != 1 && x.ReleaseDate.Value.Day != 1)
.ToList();
if (editionReleases.Any())
// If we are missing the book release date, set as the earliest edition release date
if (!book.ReleaseDate.HasValue)
{
var earliestRelease = editionReleases.Min(x => x.ReleaseDate.Value);
if (earliestRelease < book.ReleaseDate)
var editionReleases = book.Editions.Value
.Where(x => x.ReleaseDate.HasValue && x.ReleaseDate.Value.Month != 1 && x.ReleaseDate.Value.Day != 1)
.ToList();
if (editionReleases.Any())
{
book.ReleaseDate = earliestRelease;
book.ReleaseDate = editionReleases.Min(x => x.ReleaseDate.Value);
}
else
{
editionReleases = book.Editions.Value.Where(x => x.ReleaseDate.HasValue).ToList();
if (editionReleases.Any())
{
book.ReleaseDate = editionReleases.Min(x => x.ReleaseDate.Value);
}
}
}
@ -322,7 +814,7 @@ private static Edition MapEdition(BookResource resource)
return edition;
}
private int GetAuthorId(WorkResource b)
private static int GetAuthorId(WorkResource b)
{
return b.Books.OrderByDescending(x => x.RatingCount * x.AverageRating).First().Contributors.FirstOrDefault()?.ForeignId ?? 0;
}

10
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoResource/AuthorResource.cs
View file

@ -7,22 +7,12 @@ public class AuthorResource
{
public int ForeignId { get; set; }
public string Name { get; set; }
public string TitleSlug { get; set; }
public string Description { get; set; }
public string ImageUrl { get; set; }
public string Url { get; set; }
public int ReviewCount { get; set; }
public int RatingCount { get; set; }
public double AverageRating { get; set; }
public DateTime LastChange { get; set; }
public DateTime LastRefresh { get; set; }
public List<WorkResource> Works { get; set; }
public List<SeriesResource> Series { get; set; }
}
}

2
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoResource/BookResource.cs
View file

@ -6,7 +6,6 @@ namespace NzbDrone.Core.MetadataSource.BookInfo
public class BookResource
{
public int ForeignId { get; set; }
public string TitleSlug { get; set; }
public string Asin { get; set; }
public string Description { get; set; }
public string Isbn13 { get; set; }
@ -18,7 +17,6 @@ public class BookResource
public string ImageUrl { get; set; }
public bool IsEbook { get; set; }
public int? NumPages { get; set; }
public int ReviewsCount { get; set; }
public int RatingCount { get; set; }
public double AverageRating { get; set; }
public string Url { get; set; }

11
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoResource/BulkBookResource.cs Normal file
View file

@ -0,0 +1,11 @@
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class BulkBookResource
{
public List<WorkResource> Works { get; set; }
public List<SeriesResource> Series { get; set; }
public List<AuthorResource> Authors { get; set; }
}
}

3
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoResource/SeriesResource.cs
View file

@ -13,8 +13,7 @@ public class SeriesResource
public class SeriesWorkLinkResource
{
public string ForeignSeriesId { get; set; }
public string ForeignWorkId { get; set; }
public int ForeignWorkId { get; set; }
public string PositionInSeries { get; set; }
public int SeriesPosition { get; set; }
public bool Primary { get; set; }

5
src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoResource/WorkResource.cs
View file

@ -7,11 +7,12 @@ public class WorkResource
{
public int ForeignId { get; set; }
public string Title { get; set; }
public string TitleSlug { get; set; }
public string Url { get; set; }
public DateTime? ReleaseDate { get; set; }
public List<string> Genres { get; set; }
public List<int> RelatedWorks { get; set; }
public List<BookResource> Books { get; set; } = new List<BookResource>();
public List<BookResource> Books { get; set; }
public List<SeriesResource> Series { get; set; } = new List<SeriesResource>();
public List<AuthorResource> Authors { get; set; } = new List<AuthorResource>();
}
}

403
src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs
View file

@ -3,54 +3,30 @@
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.Books;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http;
using NzbDrone.Core.MediaCover;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.MetadataSource.Goodreads
{
public class GoodreadsProxy : IProvideBookInfo, IProvideSeriesInfo, IProvideListInfo
public interface IGoodreadsProxy
{
private static readonly RegexReplace FullSizeImageRegex = new RegexReplace(@"\._[SU][XY]\d+_.jpg$",
".jpg",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DuplicateSpacesRegex = new Regex(@"\s{2,}", RegexOptions.Compiled);
private static readonly Regex NoPhotoRegex = new Regex(@"/nophoto/(book|user)/",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly List<Regex> SeriesRegex = new List<Regex>
{
new Regex(@"\((?<series>[^,]+),\s+#(?<position>[\w\d\.]+)\)$", RegexOptions.Compiled),
new Regex(@"(The\s+(?<series>.+)\s+Series\s+Book\s+(?<position>[\w\d\.]+)\)$)", RegexOptions.Compiled)
};
Book GetBookInfo(string foreignEditionId, bool useCache = true);
}
public class GoodreadsProxy : IGoodreadsProxy, IProvideSeriesInfo, IProvideListInfo
{
private readonly ICachedHttpResponseService _cachedHttpClient;
private readonly Logger _logger;
private readonly IAuthorService _authorService;
private readonly IEditionService _editionService;
private readonly IHttpRequestBuilderFactory _requestBuilder;
private readonly ICached<HashSet<string>> _cache;
public GoodreadsProxy(ICachedHttpResponseService cachedHttpClient,
IAuthorService authorService,
IEditionService editionService,
Logger logger,
ICacheManager cacheManager)
Logger logger)
{
_cachedHttpClient = cachedHttpClient;
_authorService = authorService;
_editionService = editionService;
_cache = cacheManager.GetCache<HashSet<string>>(GetType());
_logger = logger;
_requestBuilder = new HttpRequestBuilder("https://www.goodreads.com/{route}")
@ -61,252 +37,6 @@ public GoodreadsProxy(ICachedHttpResponseService cachedHttpClient,
.CreateFactory();
}
public HashSet<string> GetChangedAuthors(DateTime startTime)
{
return null;
}
public Author GetAuthorInfo(string foreignAuthorId, bool useCache = true)
{
_logger.Debug("Getting Author details GoodreadsId of {0}", foreignAuthorId);
var httpRequest = _requestBuilder.Create()
.SetSegment("route", $"author/show/{foreignAuthorId}.xml")
.AddQueryParam("exclude_books", "true")
.Build();
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _cachedHttpClient.Get(httpRequest, useCache, TimeSpan.FromDays(30));
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new AuthorNotFoundException(foreignAuthorId);
}
else if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignAuthorId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
var resource = httpResponse.Deserialize<AuthorResource>();
var author = new Author
{
Metadata = MapAuthor(resource)
};
author.CleanName = Parser.Parser.CleanAuthorName(author.Metadata.Value.Name);
// we can only get a rating from the author list page...
var listResource = GetAuthorBooksPageResource(foreignAuthorId, 10, 1);
var authorResource = listResource.List.SelectMany(x => x.Authors).FirstOrDefault(a => a.Id.ToString() == foreignAuthorId);
author.Metadata.Value.Ratings = new Ratings
{
Votes = authorResource?.RatingsCount ?? 0,
Value = authorResource?.AverageRating ?? 0
};
return author;
}
public Author GetAuthorAndBooks(string foreignAuthorId, double minPopularity = 0)
{
var author = GetAuthorInfo(foreignAuthorId);
var bookList = GetAuthorBooks(foreignAuthorId, minPopularity);
var books = bookList.Select((x, i) =>
{
_logger.ProgressDebug($"{author}: Fetching book {i}/{bookList.Count}");
return GetBookInfo(x.Editions.Value.First().ForeignEditionId).Item2;
}).ToList();
var existingAuthor = _authorService.FindById(foreignAuthorId);
if (existingAuthor != null)
{
var existingEditions = _editionService.GetEditionsByAuthor(existingAuthor.Id);
var extraEditionIds = existingEditions
.Select(x => x.ForeignEditionId)
.Except(books.Select(x => x.Editions.Value.First().ForeignEditionId))
.ToList();
_logger.Debug($"Getting data for extra editions {extraEditionIds.ConcatToString()}");
var extraEditions = new List<Tuple<string, Book, List<AuthorMetadata>>>();
foreach (var id in extraEditionIds)
{
if (TryGetBookInfo(id, true, out var result))
{
extraEditions.Add(result);
}
}
var bookDict = books.ToDictionary(x => x.ForeignBookId);
foreach (var edition in extraEditions)
{
var b = edition.Item2;
if (bookDict.TryGetValue(b.ForeignBookId, out var book))
{
book.Editions.Value.Add(b.Editions.Value.First());
}
else
{
bookDict.Add(b.ForeignBookId, b);
}
}
books = bookDict.Values.ToList();
}
books.ForEach(x => x.AuthorMetadata = author.Metadata.Value);
author.Books = books;
author.Series = GetAuthorSeries(foreignAuthorId, author.Books);
return author;
}
private List<Book> GetAuthorBooks(string foreignAuthorId, double minPopularity)
{
var perPage = 100;
var page = 0;
var result = new List<Book>();
List<Book> current;
IEnumerable<Book> filtered;
do
{
current = GetAuthorBooksPage(foreignAuthorId, perPage, ++page);
filtered = current.Where(x => x.Editions.Value.First().Ratings.Popularity >= minPopularity);
result.AddRange(filtered);
}
while (current.Count == perPage && filtered.Any());
return result;
}
private List<Book> GetAuthorBooksPage(string foreignAuthorId, int perPage, int page)
{
var resource = GetAuthorBooksPageResource(foreignAuthorId, perPage, page);
var books = resource?.List.Where(x => x.Authors.First().Id.ToString() == foreignAuthorId)
.Select(MapBook)
.ToList() ??
new List<Book>();
books.ForEach(x => x.CleanTitle = x.Title.CleanAuthorName());
return books;
}
private AuthorBookListResource GetAuthorBooksPageResource(string foreignAuthorId, int perPage, int page)
{
_logger.Debug("Getting Author Books with GoodreadsId of {0}", foreignAuthorId);
var httpRequest = _requestBuilder.Create()
.SetSegment("route", $"author/list/{foreignAuthorId}.xml")
.AddQueryParam("per_page", perPage)
.AddQueryParam("page", page)
.AddQueryParam("sort", "popularity")
.Build();
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _cachedHttpClient.Get(httpRequest, true, TimeSpan.FromDays(7));
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new AuthorNotFoundException(foreignAuthorId);
}
else if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignAuthorId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
return httpResponse.Deserialize<AuthorBookListResource>();
}
private List<Series> GetAuthorSeries(string foreignAuthorId, List<Book> books)
{
_logger.Debug("Getting Author Series with GoodreadsId of {0}", foreignAuthorId);
var httpRequest = _requestBuilder.Create()
.SetSegment("route", $"series/list/{foreignAuthorId}.xml")
.Build();
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _cachedHttpClient.Get(httpRequest, true, TimeSpan.FromDays(90));
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new AuthorNotFoundException(foreignAuthorId);
}
else if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignAuthorId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
var resource = httpResponse.Deserialize<AuthorSeriesListResource>();
var result = new List<Series>();
var bookDict = books.ToDictionary(x => x.ForeignBookId);
// only take series where there are some works
// and the title is not null
// e.g. https://www.goodreads.com/series/work/6470221?format=xml is in series 260494
// which has a null title and is not shown anywhere on goodreads webpage
foreach (var seriesResource in resource.List.Where(x => x.Title.IsNotNullOrWhiteSpace() && x.Works.Any()))
{
var series = MapSeries(seriesResource);
series.LinkItems = new List<SeriesBookLink>();
var works = seriesResource.Works
.Where(x => x.BestBook.AuthorId.ToString() == foreignAuthorId &&
bookDict.ContainsKey(x.Id.ToString()));
foreach (var work in works)
{
series.LinkItems.Value.Add(new SeriesBookLink
{
Book = bookDict[work.Id.ToString()],
Series = series,
IsPrimary = true,
Position = work.UserPosition
});
}
if (series.LinkItems.Value.Any())
{
result.Add(series);
}
}
return result;
}
public SeriesResource GetSeriesInfo(int foreignSeriesId, bool useCache = true)
{
_logger.Debug("Getting Series with GoodreadsId of {0}", foreignSeriesId);
@ -380,22 +110,7 @@ public ListResource GetListInfo(int foreignListId, int page, bool useCache = tru
return httpResponse.Deserialize<ListResource>();
}
private bool TryGetBookInfo(string foreignEditionId, bool useCache, out Tuple<string, Book, List<AuthorMetadata>> result)
{
try
{
result = GetBookInfo(foreignEditionId, useCache);
return true;
}
catch (BookNotFoundException e)
{
result = null;
_logger.Warn(e, "Book not found");
return false;
}
}
public Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string foreignEditionId, bool useCache = true)
public Book GetBookInfo(string foreignEditionId, bool useCache = true)
{
_logger.Debug("Getting Book with GoodreadsId of {0}", foreignEditionId);
@ -433,40 +148,7 @@ public Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string foreignEditi
var authors = resource.Authors.SelectList(MapAuthor);
book.AuthorMetadata = authors.First();
return new Tuple<string, Book, List<AuthorMetadata>>(resource.Authors.First().Id.ToString(), book, authors);
}
private static AuthorMetadata MapAuthor(AuthorResource resource)
{
var author = new AuthorMetadata
{
ForeignAuthorId = resource.Id.ToString(),
TitleSlug = resource.Id.ToString(),
Name = resource.Name.CleanSpaces(),
Overview = resource.About,
Gender = resource.Gender,
Hometown = resource.Hometown,
Born = resource.BornOnDate,
Died = resource.DiedOnDate,
Status = resource.DiedOnDate < DateTime.UtcNow ? AuthorStatusType.Ended : AuthorStatusType.Continuing
};
author.SortName = author.Name.ToLower();
author.NameLastFirst = author.Name.ToLastFirst();
author.SortNameLastFirst = author.NameLastFirst.ToLower();
if (!NoPhotoRegex.IsMatch(resource.LargeImageUrl))
{
author.Images.Add(new MediaCover.MediaCover
{
Url = FullSizeImageRegex.Replace(resource.LargeImageUrl),
CoverType = MediaCoverTypes.Poster
});
}
author.Links.Add(new Links { Url = resource.Link, Name = "Goodreads" });
return author;
return book;
}
private static AuthorMetadata MapAuthor(AuthorSummaryResource resource)
@ -491,33 +173,9 @@ private static AuthorMetadata MapAuthor(AuthorSummaryResource resource)
};
}
if (!NoPhotoRegex.IsMatch(resource.ImageUrl))
{
author.Images.Add(new MediaCover.MediaCover
{
Url = FullSizeImageRegex.Replace(resource.ImageUrl),
CoverType = MediaCoverTypes.Poster
});
}
return author;
}
private static Series MapSeries(SeriesResource resource)
{
var series = new Series
{
ForeignSeriesId = resource.Id.ToString(),
Title = resource.Title,
Description = resource.Description,
Numbered = resource.IsNumbered,
WorkCount = resource.SeriesWorksCount,
PrimaryWorkCount = resource.PrimaryWorksCount
};
return series;
}
private static Book MapBook(BookResource resource)
{
var book = new Book
@ -554,58 +212,13 @@ private static Book MapBook(BookResource resource)
Monitored = true
};
if (resource.ImageUrl.IsNotNullOrWhiteSpace() && !NoPhotoRegex.IsMatch(resource.ImageUrl))
{
edition.Images.Add(new MediaCover.MediaCover
{
Url = FullSizeImageRegex.Replace(resource.ImageUrl),
CoverType = MediaCoverTypes.Cover
});
}
edition.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Book" });
book.Editions = new List<Edition> { edition };
Debug.Assert(!book.Editions.Value.Any() || book.Editions.Value.Count(x => x.Monitored) == 1, "one edition monitored");
book.SeriesLinks = MapSearchSeries(resource.Title, resource.TitleWithoutSeries);
return book;
}
public static List<SeriesBookLink> MapSearchSeries(string title, string titleWithoutSeries)
{
if (title != titleWithoutSeries &&
title.Substring(0, titleWithoutSeries.Length) == titleWithoutSeries)
{
var seriesText = title.Substring(titleWithoutSeries.Length);
foreach (var regex in SeriesRegex)
{
var match = regex.Match(seriesText);
if (match.Success)
{
var series = match.Groups["series"].Value;
var position = match.Groups["position"].Value;
return new List<SeriesBookLink>
{
new SeriesBookLink
{
Series = new Series
{
Title = series
},
Position = position
}
};
}
}
}
return new List<SeriesBookLink>();
}
}
}

281
src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/GoodreadsSearchProxy.cs
View file

@ -1,59 +1,26 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Books;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http;
using NzbDrone.Core.MediaCover;
using NzbDrone.Core.Parser;
namespace NzbDrone.Core.MetadataSource.Goodreads
{
public class GoodreadsSearchProxy : ISearchForNewAuthor, ISearchForNewBook, ISearchForNewEntity
public interface IGoodreadsSearchProxy
{
private static readonly RegexReplace FullSizeImageRegex = new RegexReplace(@"\._[SU][XY]\d+_.jpg$",
".jpg",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex DuplicateSpacesRegex = new Regex(@"\s{2,}", RegexOptions.Compiled);
private static readonly Regex NoPhotoRegex = new Regex(@"/nophoto/(book|user)/",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly List<Regex> SeriesRegex = new List<Regex>
{
new Regex(@"\((?<series>[^,]+),\s+#(?<position>[\w\d\.]+)\)$", RegexOptions.Compiled),
new Regex(@"(The\s+(?<series>.+)\s+Series\s+Book\s+(?<position>[\w\d\.]+)\)$)", RegexOptions.Compiled)
};
public List<SearchJsonResource> Search(string query);
}
public class GoodreadsSearchProxy : IGoodreadsSearchProxy
{
private readonly ICachedHttpResponseService _cachedHttpClient;
private readonly Logger _logger;
private readonly IProvideBookInfo _bookInfo;
private readonly IAuthorService _authorService;
private readonly IBookService _bookService;
private readonly IEditionService _editionService;
private readonly IHttpRequestBuilderFactory _searchBuilder;
private readonly ICached<HashSet<string>> _cache;
public GoodreadsSearchProxy(ICachedHttpResponseService cachedHttpClient,
IProvideBookInfo bookInfo,
IAuthorService authorService,
IBookService bookService,
IEditionService editionService,
Logger logger,
ICacheManager cacheManager)
Logger logger)
{
_cachedHttpClient = cachedHttpClient;
_bookInfo = bookInfo;
_authorService = authorService;
_bookService = bookService;
_editionService = editionService;
_cache = cacheManager.GetCache<HashSet<string>>(GetType());
_logger = logger;
_searchBuilder = new HttpRequestBuilder("https://www.goodreads.com/book/auto_complete")
@ -64,127 +31,7 @@ public GoodreadsSearchProxy(ICachedHttpResponseService cachedHttpClient,
.CreateFactory();
}
public List<Author> SearchForNewAuthor(string title)
{
var books = SearchForNewBook(title, null);
return books.Select(x => x.Author.Value).ToList();
}
public List<Book> SearchForNewBook(string title, string author)
{
try
{
var lowerTitle = title.ToLowerInvariant();
var split = lowerTitle.Split(':');
var prefix = split[0];
if (split.Length == 2 && new[] { "readarr", "readarrid", "goodreads", "isbn", "asin" }.Contains(prefix))
{
var slug = split[1].Trim();
if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace))
{
return new List<Book>();
}
if (prefix == "goodreads" || prefix == "readarr" || prefix == "readarrid")
{
var isValid = int.TryParse(slug, out var searchId);
if (!isValid)
{
return new List<Book>();
}
return SearchByGoodreadsId(searchId);
}
else if (prefix == "isbn")
{
return SearchByIsbn(slug);
}
else if (prefix == "asin")
{
return SearchByAsin(slug);
}
}
var q = title.ToLower().Trim();
if (author != null)
{
q += " " + author;
}
return SearchByField("all", q);
}
catch (HttpException)
{
throw new GoodreadsException("Search for '{0}' failed. Unable to communicate with Goodreads.", title);
}
catch (Exception ex)
{
_logger.Warn(ex, ex.Message);
throw new GoodreadsException("Search for '{0}' failed. Invalid response received from Goodreads.",
title);
}
}
public List<Book> SearchByIsbn(string isbn)
{
return SearchByField("isbn", isbn, e => e.Isbn13 = isbn);
}
public List<Book> SearchByAsin(string asin)
{
return SearchByField("asin", asin, e => e.Asin = asin);
}
public List<Book> SearchByGoodreadsId(int id)
{
try
{
var remote = _bookInfo.GetBookInfo(id.ToString());
var book = _bookService.FindById(remote.Item2.ForeignBookId);
var result = book ?? remote.Item2;
// at this point, book could have the wrong edition.
// Check if we already have the correct edition.
var remoteEdition = remote.Item2.Editions.Value.Single(x => x.Monitored);
var localEdition = _editionService.GetEditionByForeignEditionId(remoteEdition.ForeignEditionId);
if (localEdition != null)
{
result.Editions = new List<Edition> { localEdition };
}
// If we don't have the correct edition in the response, add it in.
if (!result.Editions.Value.Any(x => x.ForeignEditionId == remoteEdition.ForeignEditionId))
{
result.Editions.Value.ForEach(x => x.Monitored = false);
result.Editions.Value.Add(remoteEdition);
}
var author = _authorService.FindById(remote.Item1);
if (author == null)
{
author = new Author
{
CleanName = Parser.Parser.CleanAuthorName(remote.Item2.AuthorMetadata.Value.Name),
Metadata = remote.Item2.AuthorMetadata.Value
};
}
result.Author = author;
return new List<Book> { result };
}
catch (BookNotFoundException)
{
return new List<Book>();
}
}
public List<Book> SearchByField(string field, string query, Action<Edition> applyData = null)
public List<SearchJsonResource> Search(string query)
{
try
{
@ -194,125 +41,17 @@ public List<Book> SearchByField(string field, string query, Action<Edition> appl
var response = _cachedHttpClient.Get<List<SearchJsonResource>>(httpRequest, true, TimeSpan.FromDays(5));
return response.Resource.SelectList(x =>
MapJsonSearchResult(x, response.Resource.Count == 1 ? applyData : null));
return response.Resource;
}
catch (HttpException)
{
throw new GoodreadsException("Search for {0} '{1}' failed. Unable to communicate with Goodreads.", field, query);
throw new GoodreadsException("Search for '{0}' failed. Unable to communicate with Goodreads.", query);
}
catch (Exception ex)
{
_logger.Warn(ex, ex.Message);
throw new GoodreadsException("Search for {0} '{1}' failed. Invalid response received from Goodreads.", field, query);
throw new GoodreadsException("Search for '{0}' failed. Invalid response received from Goodreads.", query);
}
}
public List<object> SearchForNewEntity(string title)
{
var books = SearchForNewBook(title, null);
var result = new List<object>();
foreach (var book in books)
{
var author = book.Author.Value;
if (!result.Contains(author))
{
result.Add(author);
}
result.Add(book);
}
return result;
}
private Book MapJsonSearchResult(SearchJsonResource resource, Action<Edition> applyData = null)
{
var book = _bookService.FindById(resource.WorkId.ToString());
var edition = _editionService.GetEditionByForeignEditionId(resource.BookId.ToString());
if (edition == null)
{
edition = new Edition
{
ForeignEditionId = resource.BookId.ToString(),
Title = resource.BookTitleBare,
TitleSlug = resource.BookId.ToString(),
Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating },
PageCount = resource.PageCount,
Overview = resource.Description?.Html ?? string.Empty
};
if (applyData != null)
{
applyData(edition);
}
}
edition.Monitored = true;
edition.ManualAdd = true;
if (resource.ImageUrl.IsNotNullOrWhiteSpace() && !NoPhotoRegex.IsMatch(resource.ImageUrl))
{
edition.Images.Add(new MediaCover.MediaCover
{
Url = FullSizeImageRegex.Replace(resource.ImageUrl),
CoverType = MediaCoverTypes.Cover
});
}
if (book == null)
{
book = new Book
{
ForeignBookId = resource.WorkId.ToString(),
Title = resource.BookTitleBare,
TitleSlug = resource.WorkId.ToString(),
Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating },
AnyEditionOk = true
};
}
if (book.Editions != null)
{
if (book.Editions.Value.Any())
{
edition.Monitored = false;
}
book.Editions.Value.Add(edition);
}
else
{
book.Editions = new List<Edition> { edition };
}
var authorId = resource.Author.Id.ToString();
var author = _authorService.FindById(authorId);
if (author == null)
{
author = new Author
{
CleanName = Parser.Parser.CleanAuthorName(resource.Author.Name),
Metadata = new AuthorMetadata()
{
ForeignAuthorId = resource.Author.Id.ToString(),
Name = DuplicateSpacesRegex.Replace(resource.Author.Name, " "),
TitleSlug = resource.Author.Id.ToString()
}
};
}
book.Author = author;
book.AuthorMetadata = book.Author.Value.Metadata.Value;
book.AuthorMetadataId = author.AuthorMetadataId;
book.CleanTitle = book.Title.CleanAuthorName();
book.SeriesLinks = GoodreadsProxy.MapSearchSeries(resource.Title, resource.BookTitleBare);
return book;
}
}
}

3
src/NzbDrone.Core/MetadataSource/IProvideAuthorInfo.cs
View file

@ -6,8 +6,7 @@ namespace NzbDrone.Core.MetadataSource
{
public interface IProvideAuthorInfo
{
Author GetAuthorInfo(string readarrId, bool useCache = true, bool includeBooks = true);
Author GetAuthorAndBooks(string readarrId, double minPopularity = 0);
Author GetAuthorInfo(string readarrId, bool useCache = true);
HashSet<string> GetChangedAuthors(DateTime startTime);
}
}

2
src/NzbDrone.Core/MetadataSource/IProvideBookInfo.cs
View file

@ -6,6 +6,6 @@ namespace NzbDrone.Core.MetadataSource
{
public interface IProvideBookInfo
{
Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string id, bool useCache = true);
Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string id);
}
}

4
src/NzbDrone.Core/MetadataSource/ISearchForNewBook.cs
View file

@ -5,9 +5,9 @@ namespace NzbDrone.Core.MetadataSource
{
public interface ISearchForNewBook
{
List<Book> SearchForNewBook(string title, string author);
List<Book> SearchForNewBook(string title, string author, bool getAllEditions = true);
List<Book> SearchByIsbn(string isbn);
List<Book> SearchByAsin(string asin);
List<Book> SearchByGoodreadsId(int goodreadsId);
List<Book> SearchByGoodreadsBookId(int goodreadsId, bool getAllEditions);
}
}

8
src/NzbDrone.Integration.Test/ApiTests/AuthorFixture.cs
View file

@ -16,7 +16,7 @@ public void add_author_with_tags_should_store_them()
EnsureNoAuthor("14586394", "Andrew Hunter Murray");
var tag = EnsureTag("abc");
var author = Author.Lookup("readarr:43765115").Single();
var author = Author.Lookup("edition:43765115").Single();
author.QualityProfileId = 1;
author.MetadataProfileId = 1;
@ -36,7 +36,7 @@ public void add_author_without_profileid_should_return_badrequest()
{
EnsureNoAuthor("14586394", "Andrew Hunter Murray");
var author = Author.Lookup("readarr:43765115").Single();
var author = Author.Lookup("edition:43765115").Single();
author.Path = Path.Combine(AuthorRootFolder, author.AuthorName);
@ -49,7 +49,7 @@ public void add_author_without_path_should_return_badrequest()
{
EnsureNoAuthor("14586394", "Andrew Hunter Murray");
var author = Author.Lookup("readarr:43765115").Single();
var author = Author.Lookup("edition:43765115").Single();
author.QualityProfileId = 1;
@ -62,7 +62,7 @@ public void add_author()
{
EnsureNoAuthor("14586394", "Andrew Hunter Murray");
var author = Author.Lookup("readarr:43765115").Single();
var author = Author.Lookup("edition:43765115").Single();
author.QualityProfileId = 1;
author.MetadataProfileId = 1;

2
src/NzbDrone.Integration.Test/ApiTests/AuthorLookupFixture.cs
View file

@ -19,7 +19,7 @@ public void lookup_new_author_by_name(string term, string name)
[Test]
public void lookup_new_author_by_goodreads_book_id()
{
var author = Author.Lookup("readarr:1");
var author = Author.Lookup("edition:1");
author.Should().NotBeEmpty();
author.Should().Contain(c => c.AuthorName == "J.K. Rowling");

2
src/NzbDrone.Integration.Test/IntegrationTestBase.cs
View file

@ -243,7 +243,7 @@ public AuthorResource EnsureAuthor(string authorId, string goodreadsEditionId, s
if (result == null)
{
var lookup = Author.Lookup("readarr:" + goodreadsEditionId);
var lookup = Author.Lookup("edition:" + goodreadsEditionId);
var author = lookup.First();
author.QualityProfileId = 1;
author.MetadataProfileId = 1;