diff --git a/app.yaml b/app.yaml index 852b0641..d0c0668b 100644 --- a/app.yaml +++ b/app.yaml @@ -1,5 +1,5 @@ # ffd-retief-hrd fanfictiondownloader -application: fanfictiondownloader +application: ffd-retief-hrd version: 4-1-1 runtime: python27 api_version: 1 diff --git a/fanficdownloader/adapters/__init__.py b/fanficdownloader/adapters/__init__.py index 0800ca4c..70f5006a 100644 --- a/fanficdownloader/adapters/__init__.py +++ b/fanficdownloader/adapters/__init__.py @@ -15,21 +15,51 @@ # limitations under the License. # -import os, re, sys, glob +import os, re, sys, glob, types from os.path import dirname, basename, normpath import logging import urlparse as up -import fanficdownloader.exceptions as exceptions +from .. import exceptions as exceptions + +## must import each adapter here. + +import adapter_test1 +import adapter_fanfictionnet +import adapter_fanficcastletvnet +import adapter_fanfictionnet +import adapter_fictionalleyorg +import adapter_fictionpresscom +import adapter_ficwadcom +import adapter_fimfictionnet +import adapter_harrypotterfanfictioncom +import adapter_mediaminerorg +import adapter_potionsandsnitchesnet +import adapter_tenhawkpresentscom +import adapter_adastrafanficcom +import adapter_thewriterscoffeeshopcom +import adapter_tthfanficorg +import adapter_twilightednet +import adapter_twiwritenet +import adapter_whoficcom ## This bit of complexity allows adapters to be added by just adding -## the source file. It eliminates the long if/else clauses we used to -## need to pick out the adapter. +## importing. It eliminates the long if/else clauses we used to need +## to pick out the adapter. ## List of registered site adapters. - __class_list = [] +def imports(): + for name, val in globals().items(): + if isinstance(val, types.ModuleType): + yield val.__name__ + +for x in imports(): + if "fanficdownloader.adapters.adapter_" in x: + #print x + __class_list.append(sys.modules[x].getClass()) + def getAdapter(config,url): ## fix up leading protocol. fixedurl = re.sub(r"(?i)^[htp]+[:/]+","http://",url.strip()) @@ -63,16 +93,3 @@ def getClassFor(domain): for cls in __class_list: if cls.matchesSite(domain): return cls - -## Automatically import each adapter_*.py file. -## Each implement getClass() to their class - -filelist = glob.glob(dirname(__file__)+'/adapter_*.py') -sys.path.insert(0,normpath(dirname(__file__))) - -for file in filelist: - #print "file: "+basename(file)[:-3] - module = __import__(basename(file)[:-3]) - __class_list.append(module.getClass()) - -del sys.path[0] diff --git a/fanficdownloader/adapters/adapter_adastrafanficcom.py b/fanficdownloader/adapters/adapter_adastrafanficcom.py index be600c6e..bc908a21 100644 --- a/fanficdownloader/adapters/adapter_adastrafanficcom.py +++ b/fanficdownloader/adapters/adapter_adastrafanficcom.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_fanficcastletvnet.py b/fanficdownloader/adapters/adapter_fanficcastletvnet.py index 96d5d1b2..ee31b470 100644 --- a/fanficdownloader/adapters/adapter_fanficcastletvnet.py +++ b/fanficdownloader/adapters/adapter_fanficcastletvnet.py @@ -20,9 +20,9 @@ import logging import re import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_fanfictionnet.py b/fanficdownloader/adapters/adapter_fanfictionnet.py index 42e2b88e..4afa74e1 100644 --- a/fanficdownloader/adapters/adapter_fanfictionnet.py +++ b/fanficdownloader/adapters/adapter_fanfictionnet.py @@ -21,8 +21,8 @@ import re import urllib2 import time -import fanficdownloader.BeautifulSoup as bs -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_fictionalleyorg.py b/fanficdownloader/adapters/adapter_fictionalleyorg.py index 14a03ec5..7eb9e37e 100644 --- a/fanficdownloader/adapters/adapter_fictionalleyorg.py +++ b/fanficdownloader/adapters/adapter_fictionalleyorg.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_ficwadcom.py b/fanficdownloader/adapters/adapter_ficwadcom.py index 27821994..8d63aba4 100644 --- a/fanficdownloader/adapters/adapter_ficwadcom.py +++ b/fanficdownloader/adapters/adapter_ficwadcom.py @@ -22,9 +22,9 @@ import urllib2 import time import httplib, urllib -import fanficdownloader.BeautifulSoup as bs -import fanficdownloader.exceptions as exceptions -from fanficdownloader.htmlcleanup import stripHTML +from .. import BeautifulSoup as bs +from .. import exceptions as exceptions +from ..htmlcleanup import stripHTML from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_fimfictionnet.py b/fanficdownloader/adapters/adapter_fimfictionnet.py index 83322034..cddcb8ff 100644 --- a/fanficdownloader/adapters/adapter_fimfictionnet.py +++ b/fanficdownloader/adapters/adapter_fimfictionnet.py @@ -21,9 +21,9 @@ import re import urllib2 import cookielib as cl -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate @@ -88,8 +88,7 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"}) - title = soup.find("h2").find("a").text # first a link in first h2 is title. - author = soup.find("h2").find("span",{'class':'author'}).find("a").text + title, author = [link.text for link in soup.find("h2").findAll("a")] self.story.setMetadata("title", title) self.story.setMetadata("author", author) self.story.setMetadata("authorId", author) # The author's name will be unique @@ -159,4 +158,4 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter): if soup == None: raise exceptions.FailedToDownload("Error downloading Chapter: %s! Missing required element!" % url) return utf8FromSoup(soup) - + \ No newline at end of file diff --git a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py index 155ebf7f..2d44f9b6 100644 --- a/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py +++ b/fanficdownloader/adapters/adapter_harrypotterfanfictioncom.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_mediaminerorg.py b/fanficdownloader/adapters/adapter_mediaminerorg.py index 62aa50ce..23c72e30 100644 --- a/fanficdownloader/adapters/adapter_mediaminerorg.py +++ b/fanficdownloader/adapters/adapter_mediaminerorg.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py index 6cb4b415..de9a7226 100644 --- a/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py +++ b/fanficdownloader/adapters/adapter_potionsandsnitchesnet.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py index 6ec8a24b..a4922d2d 100644 --- a/fanficdownloader/adapters/adapter_tenhawkpresentscom.py +++ b/fanficdownloader/adapters/adapter_tenhawkpresentscom.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index 1debdfc7..05f6ee5e 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -19,8 +19,8 @@ import datetime import time import logging -import fanficdownloader.BeautifulSoup as bs -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from .. import exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py index 5a399a6c..fecc2bd2 100644 --- a/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py +++ b/fanficdownloader/adapters/adapter_thewriterscoffeeshopcom.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_tthfanficorg.py b/fanficdownloader/adapters/adapter_tthfanficorg.py index 1906aaac..4e9b0a3f 100644 --- a/fanficdownloader/adapters/adapter_tthfanficorg.py +++ b/fanficdownloader/adapters/adapter_tthfanficorg.py @@ -21,9 +21,9 @@ import re import urllib2 import time -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py index 2a58adfc..675f3054 100644 --- a/fanficdownloader/adapters/adapter_twilightednet.py +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_twiwritenet.py b/fanficdownloader/adapters/adapter_twiwritenet.py index 5fe9c3da..95da354f 100644 --- a/fanficdownloader/adapters/adapter_twiwritenet.py +++ b/fanficdownloader/adapters/adapter_twiwritenet.py @@ -21,9 +21,9 @@ import re import urllib import urllib2 -import fanficdownloader.BeautifulSoup as bs -from fanficdownloader.htmlcleanup import stripHTML -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from ..htmlcleanup import stripHTML +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/adapter_whoficcom.py b/fanficdownloader/adapters/adapter_whoficcom.py index fdc3fb21..40baa6ac 100644 --- a/fanficdownloader/adapters/adapter_whoficcom.py +++ b/fanficdownloader/adapters/adapter_whoficcom.py @@ -20,8 +20,8 @@ import logging import re import urllib2 -import fanficdownloader.BeautifulSoup as bs -import fanficdownloader.exceptions as exceptions +from .. import BeautifulSoup as bs +from .. import exceptions as exceptions from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index cd76abcc..6c9d3a17 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -39,13 +39,13 @@ except: pass #logging.info("Hook to make default deadline 10.0 NOT installed--not using appengine") -from fanficdownloader.story import Story -from fanficdownloader.configurable import Configurable -from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML -from fanficdownloader.exceptions import InvalidStoryURL +from ..story import Story +from ..configurable import Configurable +from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML +from ..exceptions import InvalidStoryURL try: - import fanficdownloader.chardet as chardet + from .. import chardet as chardet except ImportError: chardet = None diff --git a/fanficdownloader/writers/__init__.py b/fanficdownloader/writers/__init__.py index 7a3f7032..7d9faf64 100644 --- a/fanficdownloader/writers/__init__.py +++ b/fanficdownloader/writers/__init__.py @@ -18,7 +18,7 @@ ## This could (should?) use a dynamic loader like adapters, but for ## now, it's static, since there's so few of them. -from fanficdownloader.exceptions import FailedToDownload +from ..exceptions import FailedToDownload from writer_html import HTMLWriter from writer_txt import TextWriter diff --git a/fanficdownloader/writers/base_writer.py b/fanficdownloader/writers/base_writer.py index 411c2e4d..ad476ced 100644 --- a/fanficdownloader/writers/base_writer.py +++ b/fanficdownloader/writers/base_writer.py @@ -24,8 +24,8 @@ import zipfile from zipfile import ZipFile, ZIP_DEFLATED import logging -from fanficdownloader.configurable import Configurable -from fanficdownloader.htmlcleanup import removeEntities, removeAllEntities, stripHTML +from ..configurable import Configurable +from ..htmlcleanup import removeEntities, removeAllEntities, stripHTML class BaseStoryWriter(Configurable): diff --git a/fanficdownloader/writers/writer_epub.py b/fanficdownloader/writers/writer_epub.py index 7b3fac2f..57424c15 100644 --- a/fanficdownloader/writers/writer_epub.py +++ b/fanficdownloader/writers/writer_epub.py @@ -26,7 +26,7 @@ from zipfile import ZipFile, ZIP_STORED, ZIP_DEFLATED from xml.dom.minidom import parse, parseString, getDOMImplementation from base_writer import * -from fanficdownloader.htmlcleanup import stripHTML +from ..htmlcleanup import stripHTML class EpubWriter(BaseStoryWriter): diff --git a/fanficdownloader/writers/writer_mobi.py b/fanficdownloader/writers/writer_mobi.py index 73b48afb..87dd30f2 100644 --- a/fanficdownloader/writers/writer_mobi.py +++ b/fanficdownloader/writers/writer_mobi.py @@ -20,8 +20,8 @@ import string import StringIO from base_writer import * -from fanficdownloader.htmlcleanup import stripHTML -from fanficdownloader.mobi import Converter +from ..htmlcleanup import stripHTML +from ..mobi import Converter class MobiWriter(BaseStoryWriter): diff --git a/fanficdownloader/writers/writer_txt.py b/fanficdownloader/writers/writer_txt.py index ccf46375..55cc7719 100644 --- a/fanficdownloader/writers/writer_txt.py +++ b/fanficdownloader/writers/writer_txt.py @@ -21,7 +21,7 @@ from textwrap import wrap from base_writer import * -from fanficdownloader.html2text import html2text, BODY_WIDTH +from ..html2text import html2text, BODY_WIDTH ## In BaseStoryWriter, we define _write to encode objects ## back into for true output. But txt needs to write the