From 10aedfbce611a5a0f564661c522e4d6a0cad1bf1 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sat, 1 Mar 2014 17:00:07 -0600 Subject: [PATCH] Configurable connection timeout as per FaceDeer's recommendation. --- defaults.ini | 5 +++++ fanficdownloader/adapters/base_adapter.py | 6 +++--- plugin-defaults.ini | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/defaults.ini b/defaults.ini index 7538a77d..bbb5ba15 100644 --- a/defaults.ini +++ b/defaults.ini @@ -182,6 +182,11 @@ extratags: FanFiction ## useful if pulling large numbers of stories or if the site is slow. #slow_down_sleep_time:0.5 +## How long to wait for each HTTP connection to finish. Longer times +## are better for sites that are slow to respond. Shorter times +## prevent excessive wait when your network or the site is down. +connect_timeout:60.0 + ## For use only with stand-alone CLI version--run a command on the ## generated file after it's produced. All of the titlepage_entries ## values are available, plus output_filename. diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index 448be87b..c32f1979 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -160,13 +160,13 @@ class BaseSiteAdapter(Configurable): req = u2.Request(url, data=urllib.urlencode(parameters), headers=headers) - return self._decode(self.opener.open(req).read()) + return self._decode(self.opener.open(req,None,float(self.getConfig('connect_timeout',30.0))).read()) def _fetchUrlRaw(self, url, parameters=None): if parameters != None: - return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters)).read() + return self.opener.open(url.replace(' ','%20'),urllib.urlencode(parameters),float(self.getConfig('connect_timeout',30.0))).read() else: - return self.opener.open(url.replace(' ','%20')).read() + return self.opener.open(url.replace(' ','%20'),None,float(self.getConfig('connect_timeout',30.0))).read() def set_sleep(self,val): print("\n===========\n set sleep time %s\n==========="%val) diff --git a/plugin-defaults.ini b/plugin-defaults.ini index e0239a9d..fcdbaded 100644 --- a/plugin-defaults.ini +++ b/plugin-defaults.ini @@ -154,6 +154,11 @@ extratags: FanFiction ## useful if pulling large numbers of stories or if the site is slow. #slow_down_sleep_time:0.5 +## How long to wait for each HTTP connection to finish. Longer times +## are better for sites that are slow to respond. Shorter times +## prevent excessive wait when your network or the site is down. +connect_timeout:60.0 + ## Use regular expressions to find and replace (or remove) metadata. ## For example, you could change Sci-Fi=>SF, remove *-Centered tags, ## etc. See http://docs.python.org/library/re.html (look for re.sub)