From f02b85434313ef4941c90d0787db4b9c06899be4 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sun, 31 Jan 2021 11:34:38 -0600 Subject: [PATCH] Remove 'extrasleep' feature--it was ugly--increase slow_down_sleep_time for the couple sites that used it. We can trust the users, right? --- calibre-plugin/plugin-defaults.ini | 9 ++++---- fanficfare/adapters/adapter_fanfictionnet.py | 11 +--------- fanficfare/adapters/adapter_test1.py | 2 +- .../adapters/base_xenforoforum_adapter.py | 10 --------- fanficfare/defaults.ini | 5 +++-- fanficfare/fetcher.py | 21 ------------------- fanficfare/requestable.py | 14 ++++--------- 7 files changed, 14 insertions(+), 58 deletions(-) diff --git a/calibre-plugin/plugin-defaults.ini b/calibre-plugin/plugin-defaults.ini index 5d1ce2ac..c8affe7f 100644 --- a/calibre-plugin/plugin-defaults.ini +++ b/calibre-plugin/plugin-defaults.ini @@ -581,7 +581,7 @@ use_pagecache:true ## download fewer stories less often you can likely get by with ## reducing this sleep. There's also a hard-coded 2sec sleep in ## addition to whatever slow_down_sleep_time is. -slow_down_sleep_time:4 +slow_down_sleep_time:6 ## exclude emoji and default avatars. cover_exclusion_regexp:(/styles/|xenforo/avatars/avatar.*\.png|https://cdn\.jsdelivr\.net/gh/|https://cdn\.jsdelivr\.net/emojione) @@ -1127,6 +1127,7 @@ windows_eol: true [test1.com] use_pagecache:true +slow_down_sleep_time:2 extratags: FanFiction,Testing # extracategories:Fafner @@ -2833,15 +2834,15 @@ skip_author_cover:true ## fanfiction.net is blocking people more aggressively. If you ## download fewer stories less often you can likely get by with ## reducing this sleep. -slow_down_sleep_time:8 +slow_down_sleep_time:12 ## ffnet is sensitive to too many hits. Users are sensitive to long ## waits during the initial metadata collection in the foreground. ## When used, these settings will speed up metadata downloads in the ## foreground linearly. tweak_fg_sleep:true -min_fg_sleep:3.0 -max_fg_sleep:8.0 +min_fg_sleep:4.0 +max_fg_sleep:12.0 max_fg_sleep_at_downloads:4 ## Extra metadata that this adapter knows about. See [archiveofourown.org] diff --git a/fanficfare/adapters/adapter_fanfictionnet.py b/fanficfare/adapters/adapter_fanfictionnet.py index c80eb7c7..7374ff51 100644 --- a/fanficfare/adapters/adapter_fanfictionnet.py +++ b/fanficfare/adapters/adapter_fanfictionnet.py @@ -89,14 +89,6 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter): def getSiteURLPattern(self): return self._get_site_url_pattern() - def get_request(self,url,extrasleep=1.0,usecache=True): - ## ffnet(and, I assume, fpcom) tends to fail more if hit too - ## fast. This is in additional to what ever the - ## slow_down_sleep_time setting is. - return BaseSiteAdapter.get_request(self,url, - extrasleep=extrasleep, - usecache=usecache) - ## not actually putting urltitle on multi-chapters below, but ## one-shots will have it, so this is still useful. normalized ## chapter URLs do NOT contain the story title. @@ -372,8 +364,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter): ## AND explicitly put title URL back on chapter URL for fetch ## *only*--normalized chapter URL does NOT have urltitle - data = self.get_request(url+self.urltitle, - extrasleep=4.0) + data = self.get_request(url+self.urltitle) if "Please email this error message in full to support@fanfiction.com" in data: raise exceptions.FailedToDownload("Error downloading Chapter: %s! FanFiction.net Site Error!" % url) diff --git a/fanficfare/adapters/adapter_test1.py b/fanficfare/adapters/adapter_test1.py index 82cc136b..90c3f61f 100644 --- a/fanficfare/adapters/adapter_test1.py +++ b/fanficfare/adapters/adapter_test1.py @@ -361,7 +361,7 @@ Some more longer description. "I suck at summaries!" "Better than it sounds!" elif 'test1.com' not in url: ## for chapter_urls setting. origurl = url - (data,url) = self.get_request_redirected(url,extrasleep=2.0) + (data,url) = self.get_request_redirected(url) if '#' in origurl and '#' not in url: url = url + origurl[origurl.index('#'):] if url != origurl: diff --git a/fanficfare/adapters/base_xenforoforum_adapter.py b/fanficfare/adapters/base_xenforoforum_adapter.py index 7d21febf..a66735db 100644 --- a/fanficfare/adapters/base_xenforoforum_adapter.py +++ b/fanficfare/adapters/base_xenforoforum_adapter.py @@ -88,16 +88,6 @@ class BaseXenForoForumAdapter(BaseSiteAdapter): ## need to accept http and https still. return re.escape(self.getURLPrefix()).replace("https","https?")+r"(?Pthreads|posts)/(?P.+\.)?(?P<id>\d+)/?[^#]*?(#?post-(?P<anchorpost>\d+))?$" - def get_request_redirected(self, url, - usecache=True, - extrasleep=2.0): - ## We've been requested by the site(s) admin to rein in hits. - ## This is in additional to what ever the slow_down_sleep_time - ## setting is. - return BaseSiteAdapter.get_request_redirected(self,url, - usecache=usecache, - extrasleep=extrasleep) - ## For adapters, especially base_xenforoforum to override. Make ## sure to return unchanged URL if it's NOT a chapter URL. This ## is most helpful for xenforoforum because threadmarks use diff --git a/fanficfare/defaults.ini b/fanficfare/defaults.ini index cff18eba..30d3fefe 100644 --- a/fanficfare/defaults.ini +++ b/fanficfare/defaults.ini @@ -608,7 +608,7 @@ use_pagecache:true ## download fewer stories less often you can likely get by with ## reducing this sleep. There's also a hard-coded 2sec sleep in ## addition to whatever slow_down_sleep_time is. -slow_down_sleep_time:4 +slow_down_sleep_time:6 ## exclude emoji and default avatars. cover_exclusion_regexp:(/styles/|xenforo/avatars/avatar.*\.png|https://cdn\.jsdelivr\.net/gh/|https://cdn\.jsdelivr\.net/emojione) @@ -1158,6 +1158,7 @@ windows_eol: true [test1.com] use_pagecache:true +slow_down_sleep_time:2 extratags: FanFiction,Testing # extracategories:Fafner @@ -2864,7 +2865,7 @@ skip_author_cover:true ## fanfiction.net is blocking people more aggressively. If you ## download fewer stories less often you can likely get by with ## reducing this sleep. -slow_down_sleep_time:8 +slow_down_sleep_time:12 ## Extra metadata that this adapter knows about. See [archiveofourown.org] ## for examples of how to use them. diff --git a/fanficfare/fetcher.py b/fanficfare/fetcher.py index f20d99be..2d5eef69 100644 --- a/fanficfare/fetcher.py +++ b/fanficfare/fetcher.py @@ -78,7 +78,6 @@ class FetcherDecorator(object): method, url, parameters=None, - extrasleep=None, referer=None, usecache=True): ## can use fetcher.getConfig()/getConfigList(). @@ -86,7 +85,6 @@ class FetcherDecorator(object): method, url, parameters=parameters, - extrasleep=extrasleep, referer=referer, usecache=usecache) @@ -99,7 +97,6 @@ class ProgressBarDecorator(FetcherDecorator): method, url, parameters=None, - extrasleep=None, referer=None, usecache=True): logger.debug("ProgressBarDecorator fetcher_do_request") @@ -107,7 +104,6 @@ class ProgressBarDecorator(FetcherDecorator): method, url, parameters=parameters, - extrasleep=extrasleep, referer=referer, usecache=usecache) ## added ages ago for CLI to give a line of dots showing it's @@ -135,7 +131,6 @@ class SleepDecorator(FetcherDecorator): method, url, parameters=None, - extrasleep=None, referer=None, usecache=True): logger.debug("SleepDecorator fetcher_do_request") @@ -143,7 +138,6 @@ class SleepDecorator(FetcherDecorator): method, url, parameters=parameters, - extrasleep=extrasleep, referer=referer, usecache=usecache) @@ -152,9 +146,6 @@ class SleepDecorator(FetcherDecorator): # and other intermediate caches. if not fetchresp.fromcache: t = None - if extrasleep: - logger.debug("extra sleep:%s"%extrasleep) - time.sleep(float(extrasleep)) if self.sleep_override: t = float(self.sleep_override) elif fetcher.getConfig('slow_down_sleep_time'): @@ -233,7 +224,6 @@ class BasicCacheDecorator(FetcherDecorator): method, url, parameters=None, - extrasleep=None, referer=None, usecache=True): ''' @@ -254,7 +244,6 @@ class BasicCacheDecorator(FetcherDecorator): method, url, parameters=parameters, - extrasleep=extrasleep, referer=referer, usecache=usecache) @@ -336,14 +325,8 @@ class Fetcher(object): def do_request(self, method, url, parameters=None, - extrasleep=None, referer=None, usecache=True): - ''' - extrasleep is primarily for ffnet adapter which has extra - sleeps. Passed into fetchs so it can be bypassed when - cache hits. - ''' logger.debug("fetcher do_request") headers = self.make_headers(url,referer=referer) fetchresp = self.request(method,url, @@ -363,22 +346,18 @@ class Fetcher(object): def post_request(self, url, parameters=None, - extrasleep=None, usecache=True): fetchresp = self.do_request('POST', self.condition_url(url), parameters=parameters, - extrasleep=extrasleep, usecache=usecache) return fetchresp.content def get_request_redirected(self, url, - extrasleep=None, referer=None, usecache=True): fetchresp = self.do_request('GET', self.condition_url(url), - extrasleep=extrasleep, referer=referer, usecache=usecache) return (fetchresp.content,fetchresp.redirecturl) diff --git a/fanficfare/requestable.py b/fanficfare/requestable.py index f7b7be5a..4141b732 100644 --- a/fanficfare/requestable.py +++ b/fanficfare/requestable.py @@ -101,29 +101,23 @@ class Requestable(Configurable): return data def get_request_redirected(self, url, - usecache=True, - extrasleep=None): + usecache=True): (data,rurl) = self.configuration.get_fetcher().get_request_redirected( url, - usecache=usecache, - extrasleep=extrasleep)[:2] + usecache=usecache)[:2] data = self.decode_data(data) return (data,rurl) def get_request(self, url, - usecache=True, - extrasleep=None): + usecache=True): return self.get_request_redirected(url, - usecache, - extrasleep)[0] + usecache)[0] def get_request_raw(self, url, - extrasleep=None, referer=None, usecache=True): ## referer is used with raw for images. return self.configuration.get_fetcher().get_request_redirected( url, - extrasleep=extrasleep, referer=referer, usecache=usecache)[0]