Add XenForo(QuestionableQuesting) user/pass. Add 'Story' to XenForo 'Thread' title removal.

calibre-plugin/plugin-defaults.ini

@@ -405,7 +405,7 @@ add_to_replace_metadata:
 # remove [] or () blocks and leading/trailing spaces/dashes/colons
  title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
 # remove 'Thread' and the next word, usually "Thread 2", "Thread
-# four", "Thread iv", etc
+# four", "Thread iv", "Story Thread", etc
  title,tagsfromtitle=>[-: ]*(Story *)?[Tt]hread [^ ]+[-: ]*=>
 
 add_to_extra_titlepage_entries:,tagsfromtitle,forumtags
@@ -1357,6 +1357,13 @@ extracategories:The Pretender
 [questionablequesting.com]
 ## see [base_xenforoforum]
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [samandjack.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In

fanficfare/adapters/base_adapter.py

@@ -338,7 +338,7 @@ class BaseSiteAdapter(Configurable):
                 return (self._decode(data),opened)
             except u2.HTTPError, he:
                 excpt=he
-                if he.code == 404:
+                if he.code in (403,404):
                     logger.warn("Caught an exception reading URL: %s  Exception %s."%(unicode(url),unicode(he)))
                     break # break out on 404
             except Exception, e:

fanficfare/adapters/base_xenforoforum_adapter.py

@@ -87,6 +87,42 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
         '''
         return True
 
+    def performLogin(self):
+        params = {}
+
+        if self.password:
+            params['login'] = self.username
+            params['password'] = self.password
+        else:
+            params['login'] = self.getConfig("username")
+            params['password'] = self.getConfig("password")
+        params['register'] = '0'
+        params['cookie_check'] = '1'
+        params['_xfToken'] = ''
+        params['redirect'] = 'https://' + self.getSiteDomain() + '/'
+
+        if not params['password']:
+            return
+
+        ## https://forum.questionablequesting.com/login/login
+        loginUrl = 'https://' + self.getSiteDomain() + '/login/login'
+        logger.debug("Will now login to URL (%s) as (%s)" % (loginUrl,
+                                                              params['login']))
+
+        # soup = self.make_soup(self._fetchUrl(loginUrl))
+        # params['ctkn']=soup.find('input', {'name':'ctkn'})['value']
+        # params[soup.find('input', {'id':'password'})['name']] = params['password']
+
+        d = self._fetchUrl(loginUrl, params)
+    
+        if "Log Out" not in d :
+            logger.info("Failed to login to URL %s as %s" % (loginUrl,
+                                                             params['login']))
+            raise exceptions.FailedToLogin(self.url,params['login'])
+            return False
+        else:
+            return True
+
     ## Getting the chapter list and the meta data, plus 'is adult' checking.
     def extractChapterUrlsAndMetadata(self):
 
@@ -100,8 +136,13 @@ class BaseXenForoForumAdapter(BaseSiteAdapter):
         except urllib2.HTTPError, e:
             if e.code == 404:
                 raise exceptions.StoryDoesNotExist(self.url)
+            elif e.code == 403:
+                self.performLogin()
+                (data,opened) = self._fetchUrlOpened(useurl)
+                useurl = opened.geturl()
+                logger.info("use useurl: "+useurl)
             else:
-                raise e
+                raise
 
         # use BeautifulSoup HTML parser to make everything easier to find.
         soup = self.make_soup(data)

fanficfare/defaults.ini

@@ -404,7 +404,7 @@ add_to_replace_metadata:
 # remove [] or () blocks and leading/trailing spaces/dashes/colons
  title=>[-: ]*[\(\[]([^\]\)]+)[\)\]][-: ]*=>
 # remove 'Thread' and the next word, usually "Thread 2", "Thread
-# four", "Thread iv", etc
+# four", "Thread iv", "Story Thread", etc
  title,tagsfromtitle=>[-: ]*(Story *)?[Tt]hread [^ ]+[-: ]*=>
 
 add_to_extra_titlepage_entries:,tagsfromtitle,forumtags
@@ -1345,6 +1345,13 @@ extracategories:The Pretender
 [questionablequesting.com]
 ## see [base_xenforoforum]
 
+## Some sites require login (or login for some rated stories) The
+## program can prompt you, or you can save it in config.  In
+## commandline version, this should go in your personal.ini, not
+## defaults.ini.
+#username:YourName
+#password:yourpassword
+
 [samandjack.net]
 ## Some sites require login (or login for some rated stories) The
 ## program can prompt you, or you can save it in config.  In