Add delays for base_xenforoforum_adapter.

This commit is contained in:
Jim Miller 2017-08-29 11:02:02 -05:00
parent 4d36f24d37
commit 15b3a02edb
3 changed files with 26 additions and 0 deletions

View file

@ -463,6 +463,12 @@ add_to_extra_titlepage_entries:,storynotes
[base_xenforoforum]
## We've been requested by the site(s) admin to rein in hits. If you
## download fewer stories less often you can likely get by with
## reducing this sleep. There's also a hard-coded 2sec sleep in
## addition to whatever slow_down_sleep_time is.
slow_down_sleep_time:4
## exclude emoji and default avatars.
cover_exclusion_regexp:(/styles/|xenforo/avatars/avatar.*\.png)

View file

@ -84,6 +84,20 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
## need to accept http and https still.
return re.escape(self.getURLPrefix()).replace("https","https?")+r"/(?P<tp>threads|posts)/(.+\.)?(?P<id>\d+)/?[^#]*?(#post-(?P<anchorpost>\d+))?$"
def _fetchUrlOpened(self, url,
parameters=None,
usecache=True,
extrasleep=2.0,
referer=None):
## We've been requested by the site(s) admin to rein in hits.
## This is in additional to what ever the slow_down_sleep_time
## setting is.
return BaseSiteAdapter._fetchUrlOpened(self,url,
parameters=parameters,
usecache=usecache,
extrasleep=extrasleep,
referer=referer)
## For adapters, especially base_xenforoforum to override. Make
## sure to return unchanged URL if it's NOT a chapter URL. This
## is most helpful for xenforoforum because threadmarks use

View file

@ -478,6 +478,12 @@ add_to_extra_titlepage_entries:,storynotes
[base_xenforoforum]
## We've been requested by the site(s) admin to rein in hits. If you
## download fewer stories less often you can likely get by with
## reducing this sleep. There's also a hard-coded 2sec sleep in
## addition to whatever slow_down_sleep_time is.
slow_down_sleep_time:4
## exclude emoji and default avatars.
cover_exclusion_regexp:(/styles/|xenforo/avatars/avatar.*\.png)