Actually adding _filelist feature.

fanficfare/adapters/adapter_test1.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team, 2016 FanFicFare team
+# Copyright 2011 Fanficdownloader team, 2017 FanFicFare team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -48,6 +48,9 @@ class TestSiteAdapter(BaseSiteAdapter):
     def getSiteURLPattern(self):
         return BaseSiteAdapter.getSiteURLPattern(self)+r'/?\?sid=\d+$'
 
+    def use_pagecache(self):
+        return True
+
     def extractChapterUrlsAndMetadata(self):
         idstr = self.story.getMetadata('storyId')
         idnum = int(idstr)

fanficfare/configurable.py

@@ -537,11 +537,16 @@ class Configuration(ConfigParser.SafeConfigParser):
                 return True
             except:
                 try:
-                    self.get(section,"add_to_"+key)
-                    #print("found add_to_%s in section [%s]"%(key,section))
+                    self.get(section,key+"_filelist")
+                    #print("found %s_filelist in section [%s]"%(key,section))
                     return True
                 except:
-                    pass
+                    try:
+                        self.get(section,"add_to_"+key)
+                        #print("found add_to_%s in section [%s]"%(key,section))
+                        return True
+                    except:
+                        pass
 
         return False
 
@@ -551,15 +556,40 @@ class Configuration(ConfigParser.SafeConfigParser):
 
     def get_config(self, sections, key, default=""):
         val = default
-        for section in sections:
-            try:
-                val = self.get(section,key)
-                if val and val.lower() == "false":
-                    val = False
-                #print "getConfig(%s)=[%s]%s" % (key,section,val)
-                break
-            except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
-                pass
+
+        val_files = []
+        if not key.endswith("_filelist"):
+            ## <key>_filelist overrides <key>, but add_to_<key> is
+            ## still used.  By using self.get_config_list(),
+            ## add_to_<key>_filelist also works. (But not
+            ## <key>_filelist_filelist--that way lies madness--and
+            ## infinite recursion.)  self.get_config_list() also does
+            ## the list split & clean up.
+            val_files = self.get_config_list(sections, key+"_filelist")
+
+        file_val = False
+        if val_files:
+            val = ''
+            for v in val_files:
+                try:
+                    val = val + self._fetchUrl(v)
+                    file_val = True
+                except:
+                    pass
+            if not file_val:
+                logger.warn("All files for (%s) failed!  Using (%s) instead. Filelist: (%s)"%
+                            (key+"_filelist",key,val_files))
+
+        if not file_val:
+            for section in sections:
+                try:
+                    val = self.get(section,key)
+                    if val and val.lower() == "false":
+                        val = False
+                    #print "getConfig(%s)=[%s]%s" % (key,section,val)
+                    break
+                except (ConfigParser.NoOptionError, ConfigParser.NoSectionError), e:
+                    pass
 
         for section in sections[::-1]:
             # 'martian smiley' [::-1] reverses list by slicing whole list with -1 step.
@@ -921,6 +951,15 @@ class Configuration(ConfigParser.SafeConfigParser):
         self._set_to_pagecache(cachekey,data,url)
         return data
 
+    def _fetchUrl(self, url,
+                  parameters=None,
+                  usecache=True,
+                  extrasleep=None):
+        return self._fetchUrlOpened(url,
+                                    parameters,
+                                    usecache,
+                                    extrasleep)[0]
+
     def _fetchUrlRawOpened(self, url,
                            parameters=None,
                            extrasleep=None,
@@ -985,7 +1024,12 @@ class Configuration(ConfigParser.SafeConfigParser):
                         extrasleep=None):
 
         excpt=None
-        for sleeptime in [0, 0.5, 4, 9]:
+        if url.startswith("file://"):
+            # only one try for file:s.
+            sleeptimes = [0]
+        else:
+            sleeptimes = [0, 0.5, 4, 9]
+        for sleeptime in sleeptimes:
             time.sleep(sleeptime)
             try:
                 (data,opened)=self._fetchUrlRawOpened(url,