From 666e9666cd35ea96168176afee25beb3b90ea9db Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Fri, 22 Apr 2011 17:03:43 -0500 Subject: [PATCH] Attempting to help with ffnet failures. Increase fetch deadline, add retries with backoff, 1/2 sleep. Also remove <> from error message--HTML eats it. --- fanficdownloader/adapter.py | 2 +- fanficdownloader/ffnet.py | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/fanficdownloader/adapter.py b/fanficdownloader/adapter.py index f4dd116a..16b4adb5 100644 --- a/fanficdownloader/adapter.py +++ b/fanficdownloader/adapter.py @@ -76,7 +76,7 @@ class FanfictionSiteAdapter: if not self.appEngine: return self.opener.open(url).read().decode('utf-8') else: - return googlefetch(url).content + return googlefetch(url,deadline=10).content def requiresLogin(self, url = None): return False diff --git a/fanficdownloader/ffnet.py b/fanficdownloader/ffnet.py index 7b51a48b..47679516 100644 --- a/fanficdownloader/ffnet.py +++ b/fanficdownloader/ffnet.py @@ -277,17 +277,22 @@ class FFNet(FanfictionSiteAdapter): return urls def getText(self, url): - # time.sleep( 2.0 ) - data = '' - try: - logging.debug("Fetching URL: %s" % url) - data = self.fetchUrl(url) - except Exception, e: - data = '' - logging.error("Caught an exception reading URL " + url + ". Exception " + unicode(e) + ".") - logging.error("Data downloaded: <%s>" % data) + data = None + + # try up to three times, with longer sleeps first. + for sleeptime in [0.5, 4, 9]: + time.sleep(sleeptime) + try: + logging.debug("Fetching URL: %s sleeptime: %f" % (url, sleeptime)) + data = self.fetchUrl(url) + if data is not None: + break + except Exception, e: + logging.error("Caught an exception reading URL " + url + ". Exception " + unicode(e) + ".") + logging.error("Data downloaded: <%s>" % data) + if data is None: - raise FailedToDownload("Error downloading Chapter: <%s>! Problem getting page!" % url) + raise FailedToDownload("Error downloading Chapter: %s! Problem getting page!" % url) lines = data.split('\n') @@ -310,7 +315,7 @@ class FFNet(FanfictionSiteAdapter): div = soup.find('div', {'id' : 'storytext'}) if None == div: logging.debug(data) - raise FailedToDownload("Error downloading Chapter: <%s>! Missing required element!" % url) + raise FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return div.__str__('utf8')