Backup_Repos/FanFicFare
1
0
Fork 0
mirror of https://github.com/JimmXinu/FanFicFare.git synced 2025-12-06 08:52:55 +01:00
Code Issues Projects Releases Wiki Activity

Install hook to make urlfetch deadline 10s, add limited retry on fetch error.

Browse source
This commit is contained in:
Jim Miller 2011-06-02 16:10:19 -05:00
parent 7133189601
commit fb5740c549
1 changed files with 34 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
fanficdownloader/adapters/base_adapter.py
View file

@ -18,10 +18,27 @@
import re
import datetime
import time
import logging
import urllib
import urllib2 as u2
import urlparse as up
try:
from google.appengine.api import apiproxy_stub_map
def urlfetch_timeout_hook(service, call, request, response):
if call != 'Fetch':
return
# Make the default deadline 10 seconds instead of 5.
if not request.has_deadline():
request.set_deadline(10.0)
apiproxy_stub_map.apiproxy.GetPreCallHooks().Append(
'urlfetch_timeout_hook', urlfetch_timeout_hook, 'urlfetch')
logging.info("Hook to make default deadline 10.0 installed.")
except:
pass
#logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine")
from fanficdownloader.story import Story
from fanficdownloader.configurable import Configurable
from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
@ -86,11 +103,23 @@ class BaseSiteAdapter(Configurable):
def _fetchUrl(self, url, parameters=None):
if self.getConfig('slow_down_sleep_time'):
time.sleep(float(self.getConfig('slow_down_sleep_time')))
excpt=None
for sleeptime in [0, 0.5, 4, 9]:
time.sleep(sleeptime)
try:
if parameters:
return self.opener.open(url,urllib.urlencode(parameters))\
.read().decode(self.decode)
else:
return self.opener.open(url).read().decode(self.decode)
except Exception, e:
excpt=e
logging.warn("Caught an exception reading URL: %s Exception %s."%(unicode(url),unicode(e)))
logging.error("Giving up on %s" %url)
logging.exception(excpt)
raise(excpt)
# Does the download the first time it's called.
def getStory(self):