Move CLI up a level to match appengine, change module packaging to suit.

Change mechanism for loading adapters to avoid problems with module init.
Move adapter exceptions to own file.

fanficdownloader/adapters/__init__.py

@@ -5,47 +5,6 @@ from os.path import dirname, basename, normpath
 import logging
 import urlparse as up
 
-## A few exceptions for different things for adapters
-
-class FailedToDownload(Exception):
-    def __init__(self,error):
-        self.error=error
-
-    def __str__(self):
-        return self.error
-
-class InvalidStoryURL(Exception):
-    def __init__(self,url,domain,example):
-        self.url=url
-        self.domain=domain
-        self.example=example
-        
-    def __str__(self):
-        return "Bad Story URL: %s\nFor site: %s\nExample: %s" % (self.url, self.domain, self.example)
-
-class FailedToLogin(Exception):
-    def __init__(self,url,username):
-        self.url=url
-        self.username=username
-        
-    def __str__(self):
-        return "Failed to Login for URL: %s with username: %s" % (self.url, self.username)
-
-class StoryDoesNotExist(Exception):
-    def __init__(self,url):
-        self.url=url
-        
-    def __str__(self):
-        return "Story Does Not Exit: " + self.url
-
-class UnknownSite(Exception):
-    def __init__(self,url,supported_sites_list):
-        self.url=url
-        self.supported_sites_list=supported_sites_list
-
-    def __str__(self):
-        return "Unknown Site("+self.url+").  Supported sites: "+", ".join(self.supported_sites_list)
-
 ## This bit of complexity allows adapters to be added by just adding
 ## the source file.  It eliminates the long if/else clauses we used to
 ## need to pick out the adapter.
@@ -54,9 +13,6 @@ class UnknownSite(Exception):
     
 __class_list = []
 
-def _register_handler(cls):
-    __class_list.append(cls)
-
 def getAdapter(config,url):
     parsedUrl = up.urlparse(url)
     logging.debug("site:"+parsedUrl.netloc)
@@ -76,6 +32,7 @@ sys.path.insert(0,normpath(dirname(__file__)))
 
 for file in filelist:
     #print "file: "+basename(file)[:-3]
-    __import__(basename(file)[:-3])    
+    module = __import__(basename(file)[:-3])
+    __class_list.append(module.getClass())
 
 del sys.path[0]

fanficdownloader/adapters/adapter_fanfictionnet.py

@@ -6,11 +6,9 @@ import logging
 import re
 import urllib2
 
-import BeautifulSoup as bs
+import fanficdownloader.BeautifulSoup as bs
 
-import adapters
-from adapters import _register_handler
-from adapters.base_adapter import BaseSiteAdapter, utf8FromSoup
+from base_adapter import BaseSiteAdapter, utf8FromSoup
 
 class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
@@ -179,5 +177,7 @@ class FanFictionNetSiteAdapter(BaseSiteAdapter):
 
         return utf8FromSoup(span)
 
-_register_handler(FanFictionNetSiteAdapter)
+#_register_handler(FanFictionNetSiteAdapter)
+def getClass():
+    return FanFictionNetSiteAdapter
 

fanficdownloader/adapters/adapter_test1.py

@@ -2,10 +2,9 @@
 
 import datetime
 
-import BeautifulSoup as bs
+import fanficdownloader.BeautifulSoup as bs
 
-from adapters import _register_handler
-from adapters.base_adapter import BaseSiteAdapter, utf8FromSoup
+from base_adapter import BaseSiteAdapter, utf8FromSoup
 
 class TestSiteAdapter(BaseSiteAdapter):
 
@@ -84,6 +83,6 @@ horizontal rules
 ''',selfClosingTags=('br','hr')) # otherwise soup eats the br/hr tags.
         return utf8FromSoup(soup)
 
-
-_register_handler(TestSiteAdapter)
+def getClass():
+    return TestSiteAdapter
 

fanficdownloader/adapters/adapter_twilightednet.py

@@ -7,12 +7,10 @@ import re
 import urllib
 import urllib2
 
-import BeautifulSoup as bs
+import fanficdownloader.BeautifulSoup as bs
+from fanficdownloader.htmlcleanup import stripHTML
 
-import adapters
-from adapters import _register_handler
-from adapters.base_adapter import BaseSiteAdapter, utf8FromSoup
-from htmlcleanup import stripHTML
+from base_adapter import BaseSiteAdapter, utf8FromSoup
 
 class TwilightedNetSiteAdapter(BaseSiteAdapter):
 
@@ -196,5 +194,6 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter):
     
         return utf8FromSoup(span)
 
-_register_handler(TwilightedNetSiteAdapter)
+def getClass():
+    return TwilightedNetSiteAdapter
 

fanficdownloader/adapters/adapter_whoficcom.py

@@ -6,11 +6,9 @@ import logging
 import re
 import urllib2
 
-import BeautifulSoup as bs
+import fanficdownloader.BeautifulSoup as bs
 
-import adapters
-from adapters import _register_handler
-from adapters.base_adapter import BaseSiteAdapter, utf8FromSoup
+from base_adapter import BaseSiteAdapter, utf8FromSoup
 
 class WhoficComSiteAdapter(BaseSiteAdapter):
 
@@ -179,5 +177,6 @@ class WhoficComSiteAdapter(BaseSiteAdapter):
     
         return utf8FromSoup(span)
 
-_register_handler(WhoficComSiteAdapter)
+def getClass():
+    return WhoficComSiteAdapter
 

fanficdownloader/adapters/base_adapter.py

@@ -6,10 +6,10 @@ import time
 import urllib2 as u2
 import urlparse as up
 
-from story import Story
-from configurable import Configurable
-from htmlcleanup import removeEntities, removeAllEntities, stripHTML
-from adapters import InvalidStoryURL
+from fanficdownloader.story import Story
+from fanficdownloader.configurable import Configurable
+from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
+from fanficdownloader.adapters.exceptions import InvalidStoryURL
 
 class BaseSiteAdapter(Configurable):
 

fanficdownloader/adapters/exceptions.py

@@ -0,0 +1,41 @@
+## A few exceptions for different things for adapters
+
+class FailedToDownload(Exception):
+    def __init__(self,error):
+        self.error=error
+
+    def __str__(self):
+        return self.error
+
+class InvalidStoryURL(Exception):
+    def __init__(self,url,domain,example):
+        self.url=url
+        self.domain=domain
+        self.example=example
+        
+    def __str__(self):
+        return "Bad Story URL: %s\nFor site: %s\nExample: %s" % (self.url, self.domain, self.example)
+
+class FailedToLogin(Exception):
+    def __init__(self,url,username):
+        self.url=url
+        self.username=username
+        
+    def __str__(self):
+        return "Failed to Login for URL: %s with username: %s" % (self.url, self.username)
+
+class StoryDoesNotExist(Exception):
+    def __init__(self,url):
+        self.url=url
+        
+    def __str__(self):
+        return "Story Does Not Exit: " + self.url
+
+class UnknownSite(Exception):
+    def __init__(self,url,supported_sites_list):
+        self.url=url
+        self.supported_sites_list=supported_sites_list
+
+    def __str__(self):
+        return "Unknown Site("+self.url+").  Supported sites: "+", ".join(self.supported_sites_list)
+

fanficdownloader/writers/__init__.py

@@ -3,9 +3,9 @@
 ## This could (should?) use a dynamic loader like adapters, but for
 ## now, it's static, since there's so few of them.
 
-from writers.writer_html import HTMLWriter
-from writers.writer_txt  import TextWriter
-from writers.writer_epub import EpubWriter
+from writer_html import HTMLWriter
+from writer_txt  import TextWriter
+from writer_epub import EpubWriter
 
 def getWriter(type,config,story):
     if type == "html":

fanficdownloader/writers/base_writer.py

@@ -6,12 +6,10 @@ import string
 import StringIO
 import zipfile
 from zipfile import ZipFile, ZIP_DEFLATED
+import logging
 
-from story import Story
-from configurable import Configurable
-from htmlcleanup import removeEntities, removeAllEntities, stripHTML
-
-from adapters.base_adapter import *
+from fanficdownloader.configurable import Configurable
+from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML
 
 class BaseStoryWriter(Configurable):
 
@@ -141,6 +139,8 @@ class BaseStoryWriter(Configurable):
             outfilename=filename
 
         if not outstream:
+            close=True
+            logging.debug("Save directly to file: %s" % outfilename)
             if self.getConfig('make_directories'):
                 path=""
                 dirs = os.path.dirname(outfilename).split('/')
@@ -149,6 +149,10 @@ class BaseStoryWriter(Configurable):
                     if not os.path.exists(path):
                         os.mkdir(path) ## os.makedirs() doesn't work in 2.5.2?
             outstream = open(outfilename,"wb")
+        else:
+            close=False
+            logging.debug("Save to stream")
+
 
         if self.getConfig('zip_output'):
             out = StringIO.StringIO()
@@ -160,7 +164,8 @@ class BaseStoryWriter(Configurable):
         else:
             self.writeStoryImpl(outstream)
 
-        outstream.close()
+        if close:
+            outstream.close()
 
     def writeStoryImpl(self, out):
         "Must be overriden by sub classes."

fanficdownloader/writers/writer_epub.py

@@ -10,7 +10,7 @@ from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED
 ## use DOM to generate the XML files.
 from xml.dom.minidom import parse, parseString, getDOMImplementation
 
-from writers.base_writer import *
+from base_writer import *
 
 class EpubWriter(BaseStoryWriter):
 

fanficdownloader/writers/writer_html.py

@@ -3,7 +3,7 @@
 import logging
 import string
 
-from writers.base_writer import *
+from base_writer import *
 
 class HTMLWriter(BaseStoryWriter):
 

fanficdownloader/writers/writer_txt.py

@@ -4,9 +4,9 @@ import logging
 import string
 from textwrap import wrap
 
-from writers.base_writer import *
+from base_writer import *
 
-from html2text import html2text, BODY_WIDTH
+from fanficdownloader.html2text import html2text, BODY_WIDTH
 
 ## In BaseStoryWriter, we define _write to encode <unicode> objects
 ## back into <string> for true output.  But txt needs to write the

newdownload.py

@@ -1,17 +1,15 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import sys, os
+logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
 
-import adapters
-import writers
+import sys, os
+import getpass
+
+from fanficdownloader import adapters,writers
 
 import ConfigParser
 
-from writers.writer_html import HTMLWriter
-
-logging.basicConfig(level=logging.DEBUG,format="%(levelname)s:%(filename)s(%(lineno)d):%(message)s")
-
 config = ConfigParser.ConfigParser()
 
 logging.debug('reading defaults.ini config file, if present')
@@ -27,13 +25,15 @@ def writeStory(adapter,writeformat):
 try:
     adapter = adapters.getAdapter(config,sys.argv[1])
         
-    #try:
-    print adapter.getStory()
-    #except adapters.FailedToLogin, ftl:
-        # print "Login Failed, trying with user/pass"
-        # adapter.username="BobsClue"
-        # adapter.password="XXXXXXXXX"
-        # print adapter.getStory()
+    try:
+        print adapter.getStory()
+    except adapters.FailedToLogin, ftl:
+        print "Login Failed, Need Username/Password."
+        sys.stdout.write("Username: ")
+        adapter.username = sys.stdin.readline().strip()
+        adapter.password = getpass.getpass(prompt='Password: ')
+        #print("Login: `%s`, Password: `%s`" % (adapter.username, adapter.password))
+        print adapter.getStory()
 
     writeStory(adapter,"epub")
     writeStory(adapter,"html")