Allow tweak_fg_sleep etc with any site, add defaults settings for basexenforo, fictionalley and fictionpress.

calibre-plugin/fff_plugin.py

@@ -38,6 +38,7 @@ from functools import partial
 from datetime import datetime, time
 from string import Template
 import traceback
+from collections import defaultdict
 
 try:
     from PyQt5.Qt import (QApplication, QMenu, QTimer, QToolButton)
@@ -1101,7 +1102,9 @@ class FanFicFarePlugin(InterfaceAction):
             books = self.convert_urls_to_books(url_list)
 
         ## for tweak_fg_sleep
-        options['ffnetcount']=sum(1 for x in books if x['site']=='www.fanfiction.net')
+        d = options['site_counts'] = defaultdict(int)
+        for b in books:
+            d[b['site']] += 1
 
         options['version'] = self.version
         logger.debug(self.version)
@@ -1304,20 +1307,20 @@ class FanFicFarePlugin(InterfaceAction):
             story = self.get_story_metadata_only(adapter)
             bgmeta = False
         else:
-            # reduce foreground sleep time for ffnet when few books.
-            if 'ffnetcount' in options and \
-                    adapter.getConfig('tweak_fg_sleep') and \
-                    adapter.getSiteDomain() == 'www.fanfiction.net':
-                minslp = float(adapter.getConfig('min_fg_sleep'))
-                maxslp = float(adapter.getConfig('max_fg_sleep'))
-                dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
-                m = (maxslp-minslp) / (dwnlds-1)
-                b = minslp - m
-                slp = min(maxslp,m*float(options['ffnetcount'])+b)
-                #print("m:%s b:%s = %s"%(m,b,slp))
-                configuration.set_sleep_override(slp)
-
             if not bgmeta:
+                # reduce foreground sleep time for configured sites when few books.
+                if adapter.getConfig('tweak_fg_sleep'):
+                    minslp = float(adapter.getConfig('min_fg_sleep'))
+                    maxslp = float(adapter.getConfig('max_fg_sleep'))
+                    dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
+                    m = (maxslp-minslp) / (dwnlds-1)
+                    b = minslp - m
+                    slp = min(maxslp,m*float(options['site_counts'][book['site']])+b)
+                    # logger.debug("tweak_fg_sleep count:%s"%options['site_counts'][book['site']])
+                    # logger.debug("m:%s b:%s = %s"%(m,b,slp))
+                    logger.debug("tweak_fg_sleep: Set FG sleep override time %s"%slp)
+                    configuration.set_sleep_override(slp)
+
                 story = self.get_story_metadata_only(adapter)
                 book['title'] = story.getMetadata('title')
                 book['author'] = [story.getMetadata('author')]

calibre-plugin/plugin-defaults.ini

@@ -629,6 +629,15 @@ use_basic_cache:true
 ## addition to whatever slow_down_sleep_time is.
 slow_down_sleep_time:6
 
+## sites are sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:1.0
+max_fg_sleep:4.0
+max_fg_sleep_at_downloads:4
+
 ## exclude emoji and default avatars.
 cover_exclusion_regexp:(/styles/|xenforo/avatars/avatar.*\.png|https://cdn\.jsdelivr\.net/gh/|https://cdn\.jsdelivr\.net/emojione)
 
@@ -2975,6 +2984,15 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 website_encodings:Windows-1252,utf8
 slow_down_sleep_time:10
 
+## sites are sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:2.0
+max_fg_sleep:5.0
+max_fg_sleep_at_downloads:4
+
 [www.fictionpress.com]
 ## Using cloudscraper can satisfy the first couple levels of
 ## Cloudflare bot-proofing, but not all levels.  Older versions of
@@ -3005,6 +3023,15 @@ cover_exclusion_regexp:(/imageu/|d_60_90\.jpg)
 ## reducing this sleep.
 slow_down_sleep_time:8
 
+## ffnet is sensitive to too many hits.  Users are sensitive to long
+## waits during the initial metadata collection in the foreground.
+## When used, these settings will speed up metadata downloads in the
+## foreground linearly.
+tweak_fg_sleep:true
+min_fg_sleep:4.0
+max_fg_sleep:12.0
+max_fg_sleep_at_downloads:4
+
 ## Date formats used by FanFicFare.  Published and Update don't usually have
 ## time, but they do now on ffnet.
 ## See http://docs.python.org/library/datetime.html#strftime-strptime-behavior

fanficfare/fetcher.py

@@ -121,7 +121,7 @@ class SleepDecorator(FetcherDecorator):
 
     ## used by plugin for ffnet variable timing
     def set_sleep_override(self,val):
-        logger.debug("\n===========\n set sleep time %s\n==========="%val)
+        # logger.debug("\n===========\n set sleep time %s\n==========="%val)
         self.sleep_override = val
 
     def fetcher_do_request(self,