XF2: Add threadmarks_per_page setting

calibre-plugin/plugin-defaults.ini

@@ -941,6 +941,12 @@ always_use_forumtags:false
 ## 10.
 #reader_posts_per_page:10
 
+## XF2 sites let you change the number of threadmarks shown per page.
+## This reduces the number of network requests considerably for
+## stories with lots of threadmarks.  Site default is 25, FFF default
+## is 200, the common maximum.
+#threadmarks_per_page:200
+
 ## xenforoforum has categories of threadmarks.  This setting allows
 ## you to leave out categories you don't want.  Skipping categories
 ## will also speed downloads as categories other than 'Threadmarks'
@@ -2187,6 +2193,10 @@ cover_exclusion_regexp:/css/bir.png
 #username:YourName
 #password:yourpassword
 
+## forum.questionablequesting.com allows a larger maximum
+## threadmarks_per_page than other XF2 sites.
+threadmarks_per_page:400
+
 [forums.spacebattles.com]
 ## see [base_xenforoforum]
 
@@ -3364,6 +3374,10 @@ slow_down_sleep_time:2
 #username:YourName
 #password:yourpassword
 
+## alternatehistory.com allows a smaller maximum threadmarks_per_page
+## than other XF2 sites.
+threadmarks_per_page:50
+
 [www.aneroticstory.com]
 use_basic_cache:true
 ## Some sites do not require a login, but do require the user to
@@ -4289,6 +4303,10 @@ use_basic_cache:true
 ## the-sietch.com shows more posts per reader page than other XF sites.
 reader_posts_per_page:15
 
+## the-sietch.com allows a smaller maximum threadmarks_per_page than
+## other XF2 sites.
+threadmarks_per_page:50
+
 [www.thedelphicexpanse.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek: Enterprise

fanficfare/adapters/base_xenforo2forum_adapter.py

@@ -592,6 +592,14 @@ class BaseXenForo2ForumAdapter(BaseSiteAdapter):
             # logger.debug("fetch_threadmarks(%s,tmcat_num=%s,passed_tmcat_index:%s,url=%s,dedup=%s)\nDuplicate threadmark URL, skipping"%(tmcat_name,tmcat_num, passed_tmcat_index, url, dedup))
             return threadmarks
         dedup = dedup + [url]
+        ##
+        threadmarks_per_page = int(self.getConfig("threadmarks_per_page",'200'))
+        if threadmarks_per_page and 'per_page' not in url:
+            if '?' in url:
+                url = url + '&'
+            else:
+                url = url + '?'
+            url = url + "per_page=%s"%threadmarks_per_page
         soupmarks = self.make_soup(self.get_request(url))
         tm_list = self.get_threadmarks_list(soupmarks)
         if not tm_list: # load-range don't match

fanficfare/configurable.py

@@ -572,6 +572,7 @@ def get_valid_keywords():
                  'use_reader_mode',
                  'author_avatar_cover',
                  'reader_posts_per_page',
+                 'threadmarks_per_page',
                  'remove_spoilers',
                  'legend_spoilers',
                  'details_spoilers',

fanficfare/defaults.ini

@@ -931,6 +931,12 @@ always_use_forumtags:false
 ## 10.
 #reader_posts_per_page:10
 
+## XF2 sites let you change the number of threadmarks shown per page.
+## This reduces the number of network requests considerably for
+## stories with lots of threadmarks.  Site default is 25, FFF default
+## is 200, the common maximum.
+#threadmarks_per_page:200
+
 ## xenforoforum has categories of threadmarks.  This setting allows
 ## you to leave out categories you don't want.  Skipping categories
 ## will also speed downloads as categories other than 'Threadmarks'
@@ -2180,6 +2186,10 @@ cover_exclusion_regexp:/css/bir.png
 #username:YourName
 #password:yourpassword
 
+## forum.questionablequesting.com allows a larger maximum
+## threadmarks_per_page than other XF2 sites.
+threadmarks_per_page:400
+
 [forums.spacebattles.com]
 ## see [base_xenforoforum]
 
@@ -3357,6 +3367,10 @@ slow_down_sleep_time:2
 #username:YourName
 #password:yourpassword
 
+## alternatehistory.com allows a smaller maximum threadmarks_per_page
+## than other XF2 sites.
+threadmarks_per_page:50
+
 [www.aneroticstory.com]
 use_basic_cache:true
 ## Some sites do not require a login, but do require the user to
@@ -4262,6 +4276,10 @@ use_basic_cache:true
 ## the-sietch.com shows more posts per reader page than other XF sites.
 reader_posts_per_page:15
 
+## the-sietch.com allows a smaller maximum threadmarks_per_page than
+## other XF2 sites.
+threadmarks_per_page:50
+
 [www.thedelphicexpanse.com]
 ## Site dedicated to these categories/characters/ships
 extracategories:Star Trek: Enterprise