diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 2c5ef043..26466f63 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -941,6 +941,12 @@ always_use_forumtags:false ## 10. #reader_posts_per_page:10 +## XF2 sites let you change the number of threadmarks shown per page. +## This reduces the number of network requests considerably for +## stories with lots of threadmarks. Site default is 25, FFF default +## is 200, the common maximum. +#threadmarks_per_page:200 + ## xenforoforum has categories of threadmarks. This setting allows ## you to leave out categories you don't want. Skipping categories ## will also speed downloads as categories other than 'Threadmarks' @@ -2187,6 +2193,10 @@ cover_exclusion_regexp:/css/bir.png #username:YourName #password:yourpassword +## forum.questionablequesting.com allows a larger maximum +## threadmarks_per_page than other XF2 sites. +threadmarks_per_page:400 + [forums.spacebattles.com] ## see [base_xenforoforum] @@ -3364,6 +3374,10 @@ slow_down_sleep_time:2 #username:YourName #password:yourpassword +## alternatehistory.com allows a smaller maximum threadmarks_per_page +## than other XF2 sites. +threadmarks_per_page:50 + [www.aneroticstory.com] use_basic_cache:true ## Some sites do not require a login, but do require the user to @@ -4289,6 +4303,10 @@ use_basic_cache:true ## the-sietch.com shows more posts per reader page than other XF sites. reader_posts_per_page:15 +## the-sietch.com allows a smaller maximum threadmarks_per_page than +## other XF2 sites. +threadmarks_per_page:50 + [www.thedelphicexpanse.com] ## Site dedicated to these categories/characters/ships extracategories:Star Trek: Enterprise diff --git a/fanficfare/adapters/base_xenforo2forum_adapter.py b/fanficfare/adapters/base_xenforo2forum_adapter.py index caafcf7a..5bc0edc7 100644 --- a/fanficfare/adapters/base_xenforo2forum_adapter.py +++ b/fanficfare/adapters/base_xenforo2forum_adapter.py @@ -592,6 +592,14 @@ class BaseXenForo2ForumAdapter(BaseSiteAdapter): # logger.debug("fetch_threadmarks(%s,tmcat_num=%s,passed_tmcat_index:%s,url=%s,dedup=%s)\nDuplicate threadmark URL, skipping"%(tmcat_name,tmcat_num, passed_tmcat_index, url, dedup)) return threadmarks dedup = dedup + [url] + ## + threadmarks_per_page = int(self.getConfig("threadmarks_per_page",'200')) + if threadmarks_per_page and 'per_page' not in url: + if '?' in url: + url = url + '&' + else: + url = url + '?' + url = url + "per_page=%s"%threadmarks_per_page soupmarks = self.make_soup(self.get_request(url)) tm_list = self.get_threadmarks_list(soupmarks) if not tm_list: # load-range don't match diff --git a/fanficfare/configurable.py b/fanficfare/configurable.py index e550dc00..c0204480 100644 --- a/fanficfare/configurable.py +++ b/fanficfare/configurable.py @@ -572,6 +572,7 @@ def get_valid_keywords(): 'use_reader_mode', 'author_avatar_cover', 'reader_posts_per_page', + 'threadmarks_per_page', 'remove_spoilers', 'legend_spoilers', 'details_spoilers', diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index 935b8bd4..1e075b24 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -931,6 +931,12 @@ always_use_forumtags:false ## 10. #reader_posts_per_page:10 +## XF2 sites let you change the number of threadmarks shown per page. +## This reduces the number of network requests considerably for +## stories with lots of threadmarks. Site default is 25, FFF default +## is 200, the common maximum. +#threadmarks_per_page:200 + ## xenforoforum has categories of threadmarks. This setting allows ## you to leave out categories you don't want. Skipping categories ## will also speed downloads as categories other than 'Threadmarks' @@ -2180,6 +2186,10 @@ cover_exclusion_regexp:/css/bir.png #username:YourName #password:yourpassword +## forum.questionablequesting.com allows a larger maximum +## threadmarks_per_page than other XF2 sites. +threadmarks_per_page:400 + [forums.spacebattles.com] ## see [base_xenforoforum] @@ -3357,6 +3367,10 @@ slow_down_sleep_time:2 #username:YourName #password:yourpassword +## alternatehistory.com allows a smaller maximum threadmarks_per_page +## than other XF2 sites. +threadmarks_per_page:50 + [www.aneroticstory.com] use_basic_cache:true ## Some sites do not require a login, but do require the user to @@ -4262,6 +4276,10 @@ use_basic_cache:true ## the-sietch.com shows more posts per reader page than other XF sites. reader_posts_per_page:15 +## the-sietch.com allows a smaller maximum threadmarks_per_page than +## other XF2 sites. +threadmarks_per_page:50 + [www.thedelphicexpanse.com] ## Site dedicated to these categories/characters/ships extracategories:Star Trek: Enterprise