diff --git a/defaults.ini b/defaults.ini index eea3f4ae..ec3f8c41 100644 --- a/defaults.ini +++ b/defaults.ini @@ -1,111 +1,111 @@ -[defaults] - -## [defaults] section applies to all formats and sites but may be -## overridden. - -# All available titlepage_entries: -# category -# genre -# status -# datePublished -# dateUpdated -# dateCreated -# rating -# warnings -# numChapters -# numWords -# site -# siteabbrev -# author -# authorId -# authorURL -# title -# storyId -# storyUrl -# extratags -# description -# formatname -# formatext - -## items to include in title page -titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description - -## include title page as first page. -include_titlepage: true - -## include TOC page immediately after title page. -include_tocpage: true - -## python string Template, string with ${title}, ${author} etc, same as titlepage_entries -## Can include directories. ${formatext} will be added if not in name somewhere. -output_filename: ${title}-${siteabbrev}_${storyId}${formatext} -## Make directories as needed. -make_directories: true - -## put output (with output_filename) in a zip file zip_filename. -zip_output: false -## Can include directories. .zip will be added if not in name somewhere -zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip - -## try to make the output file name 'safe'--remove invalid filename chars. -## applies to both output_filename & zip_filename -safe_filename: true - -## extra tags (comma separated) to include, primarily for epub. -extratags: FanFiction - -## number of seconds to sleep between calls to the story site. -#slow_down_sleep_time:0.5 - -## Each output format has a section that overrides [defaults] - -[html] - -[txt] -## Add URLs since there aren't links. -titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,extratags,storyUrl, author URL, description - -# use \r\n for line endings, the windows convention. txt output only. -windows_eol: true - -[epub] - -## epub is already a zip file. -zip_output: false - -# entries tags to make epub subject tags -# lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d" -include_subject_tags: extratags, genre, category, lastupdate, status -#include_tocpage: false - -# epub->mobi conversions typically don't like tables. -titlepage_use_table: true - -## When using tables, make these span both columns. -wide_titlepage_entries: description, storyUrl, author URL - - -## Each site has a section that overrides [defaults] *and* the format section -[test1.com] -#titlepage_entries: title,description,category,genre, status,dateCreated,rating,numChapters,numWords,extratags,description,storyUrl,extratags -extratags: FanFiction,Testing - -## If necessary, you can define [:] sections to customize -## the formats differently for the same site. Overrides defaults, format and site. -[test1.com:txt] -extratags: FanFiction,Testing,Text - -[test1.com:html] -extratags: FanFiction,Testing,HTML - -[www.twilighted.net] -## Some sites require login (or login for some rated stories) -## The program can prompt you, or you can save it in config. -## This should go in your personal.ini, not defaults.ini. -#username:YourName -#password:yourpassword - -[www.whofic.com] - -[www.fanfiction.net] - +[defaults] + +## [defaults] section applies to all formats and sites but may be +## overridden. + +# All available titlepage_entries: +# category +# genre +# status +# datePublished +# dateUpdated +# dateCreated +# rating +# warnings +# numChapters +# numWords +# site +# siteabbrev +# author +# authorId +# authorURL +# title +# storyId +# storyUrl +# extratags +# description +# formatname +# formatext + +## items to include in title page +titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,description + +## include title page as first page. +include_titlepage: true + +## include TOC page immediately after title page. +include_tocpage: true + +## python string Template, string with ${title}, ${author} etc, same as titlepage_entries +## Can include directories. ${formatext} will be added if not in name somewhere. +output_filename: ${title}-${siteabbrev}_${storyId}${formatext} +## Make directories as needed. +make_directories: true + +## put output (with output_filename) in a zip file zip_filename. +zip_output: false +## Can include directories. .zip will be added if not in name somewhere +zip_filename: ${title}-${siteabbrev}_${storyId}${formatext}.zip + +## try to make the output file name 'safe'--remove invalid filename chars. +## applies to both output_filename & zip_filename +safe_filename: true + +## extra tags (comma separated) to include, primarily for epub. +extratags: FanFiction + +## number of seconds to sleep between calls to the story site. +#slow_down_sleep_time:0.5 + +## Each output format has a section that overrides [defaults] + +[html] + +[txt] +## Add URLs since there aren't links. +titlepage_entries: category,genre,status,datePublished,dateUpdated,dateCreated,rating,warnings,numChapters,numWords,site,extratags,storyUrl, author URL, description + +# use \r\n for line endings, the windows convention. txt output only. +windows_eol: true + +[epub] + +## epub is already a zip file. +zip_output: false + +# entries tags to make epub subject tags +# lastupdate creates two tags: "Last Update Year/Month: %Y/%m" and "Last Update: %Y/%m/%d" +include_subject_tags: extratags, genre, category, lastupdate, status +#include_tocpage: false + +# epub->mobi conversions typically don't like tables. +titlepage_use_table: true + +## When using tables, make these span both columns. +wide_titlepage_entries: description, storyUrl, author URL + + +## Each site has a section that overrides [defaults] *and* the format section +[test1.com] +#titlepage_entries: title,description,category,genre, status,dateCreated,rating,numChapters,numWords,extratags,description,storyUrl,extratags +extratags: FanFiction,Testing + +## If necessary, you can define [:] sections to customize +## the formats differently for the same site. Overrides defaults, format and site. +[test1.com:txt] +extratags: FanFiction,Testing,Text + +[test1.com:html] +extratags: FanFiction,Testing,HTML + +[www.twilighted.net] +## Some sites require login (or login for some rated stories) +## The program can prompt you, or you can save it in config. +## This should go in your personal.ini, not defaults.ini. +#username:YourName +#password:yourpassword + +[www.whofic.com] + +[www.fanfiction.net] + diff --git a/editconfig.html b/editconfig.html new file mode 100644 index 00000000..9b59fd63 --- /dev/null +++ b/editconfig.html @@ -0,0 +1,83 @@ + + + + + Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza + + + + + +
+

+ FanFiction Downloader +

+ +
+ + +
+ +
+ +
+

Edit Config

+
+ Editing configuration for {{ nickname }}. + {% if default %} Default values are shown. {% else %} Empty this box and Save to go back to use the default values. {% endif %} +
+
+ +
+
+ +
+ +
+
+ +
+ Powered by Google App Engine +

+ FanfictionLoader is a web front-end to fanficdownloader
+ Copyright © Roman Kirillov +
+ +
+ + +
+
+ + diff --git a/fanficdownloader/adapters/adapter_test1.py b/fanficdownloader/adapters/adapter_test1.py index fdf9cb87..3f4af12e 100644 --- a/fanficdownloader/adapters/adapter_test1.py +++ b/fanficdownloader/adapters/adapter_test1.py @@ -15,6 +15,7 @@ class TestSiteAdapter(BaseSiteAdapter): self.crazystring = u" crazy tests:[bare amp(&) quote(') amp(&) gt(>) lt(<) ATnT(AT&T) pound(£)]" # get storyId from url--url validation guarantees query is only sid=1234 self.story.setMetadata('storyId',self.parsedUrl.query.split('=',)[1]) + self.username='' @staticmethod def getSiteDomain(): @@ -31,8 +32,11 @@ class TestSiteAdapter(BaseSiteAdapter): if self.story.getMetadata('storyId') == '666': raise exceptions.StoryDoesNotExist(self.url) - if self.story.getMetadata('storyId') == '668': - raise exceptions.FailedToLogin(self.url,"FakeUser") + if self.getConfig("username"): + self.username = self.getConfig("username") + + if self.story.getMetadata('storyId') == '668' and self.username != "Me" : + raise exceptions.FailedToLogin(self.url,self.username) self.story.setMetadata(u'title',"Test Story Title "+self.crazystring) self.story.setMetadata('storyUrl',self.url) diff --git a/fanficdownloader/adapters/adapter_twilightednet.py b/fanficdownloader/adapters/adapter_twilightednet.py index f3d64064..751fb9c3 100644 --- a/fanficdownloader/adapters/adapter_twilightednet.py +++ b/fanficdownloader/adapters/adapter_twilightednet.py @@ -72,7 +72,7 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): d = self._fetchUrl(loginUrl, urlvals) - if self.needToLoginCheck(d) : + if "Member Account" not in d : #Member Account logging.info("Failed to login to URL %s as %s" % (loginUrl, data['penname'])) raise exceptions.FailedToLogin(url,data['penname']) @@ -98,6 +98,9 @@ class TwilightedNetSiteAdapter(BaseSiteAdapter): self.performLogin(url) data = self._fetchUrl(url) + if "Access denied. This story has not been validated by the adminstrators of this site." in data: + raise exceptions.FailedToDownload(self.getSiteDomain() +" says: Access denied. This story has not been validated by the adminstrators of this site.") + # use BeautifulSoup HTML parser to make everything easier to find. soup = bs.BeautifulSoup(data) diff --git a/fanficdownloader/adapters/base_adapter.py b/fanficdownloader/adapters/base_adapter.py index d4233d08..67c0033a 100644 --- a/fanficdownloader/adapters/base_adapter.py +++ b/fanficdownloader/adapters/base_adapter.py @@ -82,6 +82,7 @@ class BaseSiteAdapter(Configurable): ## URL pattern validation is done *after* picking an adaptor based ## on domain instead of *as* the adaptor selector so we can offer ## the user example(s) for that particular site. + ## Override validateURL(self) instead if you need more control. def getSiteURLPattern(self): "Used to validate URL. Should be override in each adapter class." return '^http://'+re.escape(self.getSiteDomain()) diff --git a/ffstorage.py b/ffstorage.py index bb17d8bb..dae09352 100644 --- a/ffstorage.py +++ b/ffstorage.py @@ -17,3 +17,7 @@ class DownloadData(db.Model): collection_name='data_chunks') blob = db.BlobProperty() index = db.IntegerProperty() + +class UserConfig(db.Model): + user = db.UserProperty() + config = db.TextProperty() diff --git a/index.html b/index.html index 5af8067c..940a5cad 100644 --- a/index.html +++ b/index.html @@ -53,11 +53,14 @@

Experimental New Version!

- This version is a new re-org/re-write of the code + This version is a new re-org/re-write of the code.

So far, only a few sites are supported: fanfiction.net, twilighted.net and whofic.com.

+

+ Login/Password is only asked for when required now. +

Mobi support (for Kindle) is only via EPub conversion in this version.

@@ -65,10 +68,11 @@
{{ error_message }}
-
+
URL:
+
Ebook format
EPub @@ -76,30 +80,13 @@ Plain Text

For Mobi (Kindle) select EPub and use the Convert link when it's finished.

-
- -
-

Login and Password

-
- If the story requires a login and password to download, - you may need to provide your credentials to download it, - otherwise just leave it empty. Currently only needed by - twilighted.net and twiwrite.net. -
-
-
Login
-
-
- -
-
Password
-
+
+
- -
- -
+
+ Customize your User Configuration. +
{% else %}
diff --git a/login.html b/login.html new file mode 100644 index 00000000..e54141cd --- /dev/null +++ b/login.html @@ -0,0 +1,96 @@ + + + + + Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza + + + + + +
+

+ FanFiction Downloader +

+ +
+ + +
+ + {% if fic.failure %} +
+ {{ fic.failure }} +
+ {% endif %} +
+
+

Login and Password

+
+ {{ site }} requires a Login/Password for this story. + You need to provide your Login/Password for {{ site }} + to download it. +
+ + +
+
Login
+
+
+ +
+
Password
+
+
+
+ +
+ +
+
+ +
+ Powered by Google App Engine +

+ FanfictionLoader is a web front-end to fanficdownloader
+ Copyright © Roman Kirillov +
+ +
+ + +
+
+ + diff --git a/main.py b/main.py index e2625b31..08838957 100644 --- a/main.py +++ b/main.py @@ -47,371 +47,415 @@ from fanficdownloader.zipdir import * from ffstorage import * -from fanficdownloader import adapters, writers +from fanficdownloader import adapters, writers, exceptions import ConfigParser -class LoginRequired(webapp.RequestHandler): - def get(self): - user = users.get_current_user() - if user: - self.redirect('/') - return - else: - logging.debug(users.create_login_url('/')) - url = users.create_login_url(self.request.uri) - template_values = {'login_url' : url} - path = os.path.join(os.path.dirname(__file__), 'index-nonlogin.html') - self.response.out.write(template.render(path, template_values)) - class MainHandler(webapp.RequestHandler): - def get(self): - user = users.get_current_user() - if user: - error = self.request.get('error') - template_values = {'nickname' : user.nickname(), 'authorized': True} - url = self.request.get('url') - template_values['url'] = url - - if error != None and len(error) > 1: - if error == 'login_required': - template_values['error_message'] = 'This story (or one of the chapters) requires you to be logged in.' - elif error == 'bad_url': - template_values['error_message'] = 'Unsupported URL: ' + url - elif error == 'custom': - template_values['error_message'] = 'Error happened: ' + self.request.get('errtext') - - filename = self.request.get('file') - if len(filename) > 1: - template_values['yourfile'] = '''''' % (filename, self.request.get('name'), self.request.get('author')) - - self.response.headers['Content-Type'] = 'text/html' - path = os.path.join(os.path.dirname(__file__), 'index.html') + def get(self): + user = users.get_current_user() + if user: + error = self.request.get('error') + template_values = {'nickname' : user.nickname(), 'authorized': True} + url = self.request.get('url') + template_values['url'] = url + + if error: + if error == 'login_required': + template_values['error_message'] = 'This story (or one of the chapters) requires you to be logged in.' + elif error == 'bad_url': + template_values['error_message'] = 'Unsupported URL: ' + url + elif error == 'custom': + template_values['error_message'] = 'Error happened: ' + self.request.get('errtext') + elif error == 'configsaved': + template_values['error_message'] = 'Configuration Saved' + + filename = self.request.get('file') + if len(filename) > 1: + template_values['yourfile'] = '''''' % (filename, self.request.get('name'), self.request.get('author')) + + self.response.headers['Content-Type'] = 'text/html' + path = os.path.join(os.path.dirname(__file__), 'index.html') - self.response.out.write(template.render(path, template_values)) - else: - logging.debug(users.create_login_url('/')) - url = users.create_login_url(self.request.uri) - template_values = {'login_url' : url, 'authorized': False} - path = os.path.join(os.path.dirname(__file__), 'index.html') - self.response.out.write(template.render(path, template_values)) + self.response.out.write(template.render(path, template_values)) + else: + logging.debug(users.create_login_url('/')) + url = users.create_login_url(self.request.uri) + template_values = {'login_url' : url, 'authorized': False} + path = os.path.join(os.path.dirname(__file__), 'index.html') + self.response.out.write(template.render(path, template_values)) +class EditConfigServer(webapp.RequestHandler): + def get(self): + self.post() + + def post(self): + user = users.get_current_user() + if not user: + self.redirect(users.create_login_url(self.request.uri)) + return + + template_values = {'nickname' : user.nickname(), 'authorized': True} + + ## Pull user's config record. + l = UserConfig.all().filter('user =', user).fetch(1) + if l: + uconfig=l[0] + else: + uconfig=None + + if self.request.get('update'): + if uconfig is None: + uconfig = UserConfig() + uconfig.user = user + uconfig.config = self.request.get('config').encode('utf8')[:1000000] ## just in case. + uconfig.put() + self.redirect("/?error=configsaved") + else: # not update, assume display for edit + if uconfig is not None and uconfig.config: + config = uconfig.config + else: + template_values['default'] = True + configfile = open("defaults.ini","rb") + config = configfile.read() + configfile.close() + template_values['config'] = config + path = os.path.join(os.path.dirname(__file__), 'editconfig.html') + self.response.headers['Content-Type'] = 'text/html' + self.response.out.write(template.render(path, template_values)) + + class FileServer(webapp.RequestHandler): - def get(self): - fileId = self.request.get('id') - - if fileId == None or len(fileId) < 3: - self.redirect('/') - return - - key = db.Key(fileId) - fanfic = db.get(key) + def get(self): + fileId = self.request.get('id') + + if fileId == None or len(fileId) < 3: + self.redirect('/') + return + + key = db.Key(fileId) + fanfic = db.get(key) - # check for completed & failure. - - name = fanfic.name.encode('utf-8') - - #name = urllib.quote(name) - - logging.info("Serving file: %s" % name) + # check for completed & failure. + + name = fanfic.name.encode('utf-8') + + #name = urllib.quote(name) + + logging.info("Serving file: %s" % name) - if name.endswith('.epub'): - self.response.headers['Content-Type'] = 'application/epub+zip' - elif name.endswith('.html'): - self.response.headers['Content-Type'] = 'text/html' - elif name.endswith('.txt'): - self.response.headers['Content-Type'] = 'text/plain' - elif name.endswith('.zip'): - self.response.headers['Content-Type'] = 'application/zip' - else: - self.response.headers['Content-Type'] = 'application/octet-stream' - - self.response.headers['Content-disposition'] = 'attachment; filename="%s"' % name + if name.endswith('.epub'): + self.response.headers['Content-Type'] = 'application/epub+zip' + elif name.endswith('.html'): + self.response.headers['Content-Type'] = 'text/html' + elif name.endswith('.txt'): + self.response.headers['Content-Type'] = 'text/plain' + elif name.endswith('.zip'): + self.response.headers['Content-Type'] = 'application/zip' + else: + self.response.headers['Content-Type'] = 'application/octet-stream' + + self.response.headers['Content-disposition'] = 'attachment; filename="%s"' % name - data = DownloadData.all().filter("download =", fanfic).order("index") - # epubs are all already compressed. - # Each chunk is compress individually to avoid having - # to hold the whole in memory just for the - # compress/uncompress - if fanfic.format != 'epub': - def dc(data): - try: - return zlib.decompress(data) - # if error, assume it's a chunk from before we started compessing. - except zlib.error: - return data - else: - def dc(data): - return data - - for datum in data: - self.response.out.write(dc(datum.blob)) + data = DownloadData.all().filter("download =", fanfic).order("index") + # epubs are all already compressed. + # Each chunk is compress individually to avoid having + # to hold the whole in memory just for the + # compress/uncompress + if fanfic.format != 'epub': + def dc(data): + try: + return zlib.decompress(data) + # if error, assume it's a chunk from before we started compessing. + except zlib.error: + return data + else: + def dc(data): + return data + + for datum in data: + self.response.out.write(dc(datum.blob)) class FileStatusServer(webapp.RequestHandler): - def get(self): - user = users.get_current_user() - if not user: - self.redirect(users.create_login_url(self.request.uri)) - return - - fileId = self.request.get('id') - - if fileId == None or len(fileId) < 3: - self.redirect('/') - - key = db.Key(fileId) - fic = db.get(key) + def get(self): + user = users.get_current_user() + if not user: + self.redirect(users.create_login_url(self.request.uri)) + return + + fileId = self.request.get('id') + + if fileId == None or len(fileId) < 3: + self.redirect('/') + + key = db.Key(fileId) + fic = db.get(key) - logging.info("Status url: %s" % fic.url) - if fic.completed and fic.format=='epub': - escaped_url = urlEscape(self.request.host_url+"/file/"+fic.name+"."+fic.format+"?id="+fileId+"&fake=file."+fic.format) - else: - escaped_url=False - template_values = dict(fic = fic, - nickname = user.nickname(), - escaped_url = escaped_url - ) - path = os.path.join(os.path.dirname(__file__), 'status.html') - self.response.out.write(template.render(path, template_values)) - + logging.info("Status url: %s" % fic.url) + if fic.completed and fic.format=='epub': + escaped_url = urlEscape(self.request.host_url+"/file/"+fic.name+"."+fic.format+"?id="+fileId+"&fake=file."+fic.format) + else: + escaped_url=False + template_values = dict(fic = fic, + nickname = user.nickname(), + escaped_url = escaped_url + ) + path = os.path.join(os.path.dirname(__file__), 'status.html') + self.response.out.write(template.render(path, template_values)) + class RecentFilesServer(webapp.RequestHandler): - def get(self): - user = users.get_current_user() - if not user: - self.redirect(users.create_login_url(self.request.uri)) - return - - q = DownloadMeta.all() - q.filter('user =', user).order('-date') - fics = q.fetch(100) - - for fic in fics: - if fic.completed and fic.format == 'epub': - fic.escaped_url = urlEscape(self.request.host_url+"/file/"+fic.name+"."+fic.format+"?id="+str(fic.key())+"&fake=file."+fic.format) - - template_values = dict(fics = fics, nickname = user.nickname()) - path = os.path.join(os.path.dirname(__file__), 'recent.html') - self.response.out.write(template.render(path, template_values)) - -class RecentAllFilesServer(webapp.RequestHandler): - def get(self): - user = users.get_current_user() - if user.nickname() != 'sigizmund': - return - - fics = db.GqlQuery("Select * From DownloadedFanfic") - template_values = dict(fics = fics, nickname = user.nickname()) - path = os.path.join(os.path.dirname(__file__), 'recent.html') - self.response.out.write(template.render(path, template_values)) + def get(self): + user = users.get_current_user() + if not user: + self.redirect(users.create_login_url(self.request.uri)) + return + + q = DownloadMeta.all() + q.filter('user =', user).order('-date') + fics = q.fetch(100) + for fic in fics: + if fic.completed and fic.format == 'epub': + fic.escaped_url = urlEscape(self.request.host_url+"/file/"+fic.name+"."+fic.format+"?id="+str(fic.key())+"&fake=file."+fic.format) + + template_values = dict(fics = fics, nickname = user.nickname()) + path = os.path.join(os.path.dirname(__file__), 'recent.html') + self.response.out.write(template.render(path, template_values)) + class FanfictionDownloader(webapp.RequestHandler): - def get(self): - self.post() + def get(self): + self.post() - def post(self): - logging.getLogger().setLevel(logging.DEBUG) - - user = users.get_current_user() - if not user: - self.redirect(users.create_login_url(self.request.uri)) - return - - format = self.request.get('format') - url = self.request.get('url') - login = self.request.get('login') - password = self.request.get('password') - - logging.info("Queuing Download: " + url) + def post(self): + logging.getLogger().setLevel(logging.DEBUG) + user = users.get_current_user() + if not user: + self.redirect(users.create_login_url(self.request.uri)) + return + + format = self.request.get('format') + url = self.request.get('url') + login = self.request.get('login') + password = self.request.get('password') + + logging.info("Queuing Download: " + url) - # use existing record if available. - q = DownloadMeta.all().filter('user =', user).filter('url =',url).filter('format =',format).fetch(1) - if( q is None or len(q) < 1 ): - download = DownloadMeta() - else: - download = q[0] - download.completed=False - download.failure=None - for c in download.data_chunks: - c.delete() - - download.user = user - download.url = url - download.format = format + # use existing record if available. + q = DownloadMeta.all().filter('user =', user).filter('url =',url).filter('format =',format).fetch(1) + if( q is None or len(q) < 1 ): + download = DownloadMeta() + else: + download = q[0] + download.completed=False + download.failure=None + for c in download.data_chunks: + c.delete() + + download.user = user + download.url = url + download.format = format - adapter = None - - try: - config = ConfigParser.ConfigParser() - logging.debug('reading defaults.ini config file, if present') - config.read('defaults.ini') - logging.debug('reading appengine.ini config file, if present') - config.read('appengine.ini') - adapter = adapters.getAdapter(config,url) - logging.info('Created an adaper: %s' % adapter) - - if len(login) > 1: - adapter.username=login - adapter.password=password - ## This scrapes the metadata, which will be - ## duplicated in the queue task, but it - ## detects bad URLs, bad login, bad story, etc - ## without waiting for the queue. So I think - ## it's worth the double up. Could maybe save - ## it all in the download object someday. - story = adapter.getStoryMetadataOnly() - download.title = story.getMetadata('title') - download.author = story.getMetadata('author') - download.put() + adapter = None + try: + config = ConfigParser.SafeConfigParser() - taskqueue.add(url='/fdowntask', - queue_name="download", - params={'format':format, - 'url':url, - 'login':login, - 'password':password, - 'user':user.email()}) - - logging.info("enqueued download key: " + str(download.key())) + ## Pull user's config record. + l = UserConfig.all().filter('user =', user).fetch(1) + if l: + uconfig=l[0] + logging.debug('reading config from UserConfig') + config.readfp(StringIO.StringIO(uconfig.config)) + else: + logging.debug('reading defaults.ini config file') + config.read('defaults.ini') + + adapter = adapters.getAdapter(config,url) + logging.info('Created an adaper: %s' % adapter) + + if len(login) > 1: + adapter.username=login + adapter.password=password + ## This scrapes the metadata, which will be + ## duplicated in the queue task, but it + ## detects bad URLs, bad login, bad story, etc + ## without waiting for the queue. So I think + ## it's worth the double up. Could maybe save + ## it all in the download object someday. + story = adapter.getStoryMetadataOnly() + download.title = story.getMetadata('title') + download.author = story.getMetadata('author') + download.put() - except Exception, e: - logging.exception(e) - download.failure = str(e) - download.put() - - self.redirect('/status?id='+str(download.key())) + taskqueue.add(url='/fdowntask', + queue_name="download", + params={'format':format, + 'url':url, + 'login':login, + 'password':password, + 'user':user.email()}) + + logging.info("enqueued download key: " + str(download.key())) - return + except exceptions.FailedToLogin, e: + logging.exception(e) + download.failure = str(e) + download.put() + logging.debug('Need to Login, display log in page') + template_values = dict(nickname = user.nickname(), + url = url, + format = format, + site = adapter.getSiteDomain(), + fic = download + ) + path = os.path.join(os.path.dirname(__file__), 'login.html') + self.response.out.write(template.render(path, template_values)) + return + except Exception, e: + logging.exception(e) + download.failure = str(e) + download.put() + + self.redirect('/status?id='+str(download.key())) + + return class FanfictionDownloaderTask(webapp.RequestHandler): - def _printableVersion(self, text): - text = removeEntities(text) - try: - d = text.decode('utf-8') - except: - d = text - return d - + def _printableVersion(self, text): + text = removeEntities(text) + try: + d = text.decode('utf-8') + except: + d = text + return d + - def post(self): - logging.getLogger().setLevel(logging.DEBUG) - - format = self.request.get('format') - url = self.request.get('url') - login = self.request.get('login') - password = self.request.get('password') - # User object can't pass, just email address - user = users.User(self.request.get('user')) - - logging.info("Downloading: " + url + " for user: "+user.nickname()) - - adapter = None - writerClass = None + def post(self): + logging.getLogger().setLevel(logging.DEBUG) + format = self.request.get('format') + url = self.request.get('url') + login = self.request.get('login') + password = self.request.get('password') + # User object can't pass, just email address + user = users.User(self.request.get('user')) + + logging.info("Downloading: " + url + " for user: "+user.nickname()) + + adapter = None + writerClass = None - # use existing record if available. - q = DownloadMeta.all().filter('user =', user).filter('url =',url).filter('format =',format).fetch(1) - if( q is None or len(q) < 1 ): - download = DownloadMeta() - else: - download = q[0] - download.completed=False - for c in download.data_chunks: - c.delete() - - download.user = user - download.url = url - download.format = format - download.put() - logging.info('Creating adapter...') - - try: - config = ConfigParser.ConfigParser() - logging.debug('reading defaults.ini config file, if present') - config.read('defaults.ini') - logging.debug('reading appengine.ini config file, if present') - config.read('appengine.ini') - adapter = adapters.getAdapter(config,url) - except Exception, e: - logging.exception(e) - download.failure = str(e) - download.put() - return - - logging.info('Created an adaper: %s' % adapter) - - if len(login) > 1: - adapter.username=login - adapter.password=password + # use existing record if available. + q = DownloadMeta.all().filter('user =', user).filter('url =',url).filter('format =',format).fetch(1) + if( q is None or len(q) < 1 ): + download = DownloadMeta() + else: + download = q[0] + download.completed=False + for c in download.data_chunks: + c.delete() + + download.user = user + download.url = url + download.format = format + download.put() + logging.info('Creating adapter...') + + try: + config = ConfigParser.ConfigParser() + config = ConfigParser.SafeConfigParser() - try: - # adapter.getStory() is what does all the heavy lifting. - writer = writers.getWriter(format,config,adapter.getStory()) - except Exception, e: - logging.exception(e) - download.failure = str(e) - download.put() - return - - download.name = writer.getOutputFileName() - download.title = adapter.getStory().getMetadata('title') - download.author = adapter.getStory().getMetadata('author') - download.put() - index=0 + ## Pull user's config record. + l = UserConfig.all().filter('user =', user).fetch(1) + if l: + uconfig=l[0] + logging.debug('reading config from UserConfig') + config.readfp(StringIO.StringIO(uconfig.config)) + else: + logging.debug('reading defaults.ini config file') + config.read('defaults.ini') + + adapter = adapters.getAdapter(config,url) + except Exception, e: + logging.exception(e) + download.failure = str(e) + download.put() + return + + logging.info('Created an adaper: %s' % adapter) + + if len(login) > 1: + adapter.username=login + adapter.password=password - outbuffer = StringIO.StringIO() - writer.writeStory(outbuffer) - data = outbuffer.getvalue() - outbuffer.close() - del writer - del adapter + try: + # adapter.getStory() is what does all the heavy lifting. + writer = writers.getWriter(format,config,adapter.getStory()) + except Exception, e: + logging.exception(e) + download.failure = str(e) + download.put() + return + + download.name = writer.getOutputFileName() + download.title = adapter.getStory().getMetadata('title') + download.author = adapter.getStory().getMetadata('author') + download.put() + index=0 - # epubs are all already compressed. - # Each chunk is compressed individually to avoid having - # to hold the whole in memory just for the - # compress/uncompress. - if format != 'epub': - def c(data): - return zlib.compress(data) - else: - def c(data): - return data - - while( len(data) > 0 ): - DownloadData(download=download, - index=index, - blob=c(data[:1000000])).put() - index += 1 - data = data[1000000:] - download.completed=True - download.put() - - logging.info("Download finished OK") - return - + outbuffer = StringIO.StringIO() + writer.writeStory(outbuffer) + data = outbuffer.getvalue() + outbuffer.close() + del writer + del adapter + + # epubs are all already compressed. + # Each chunk is compressed individually to avoid having + # to hold the whole in memory just for the + # compress/uncompress. + if format != 'epub': + def c(data): + return zlib.compress(data) + else: + def c(data): + return data + + while( len(data) > 0 ): + DownloadData(download=download, + index=index, + blob=c(data[:1000000])).put() + index += 1 + data = data[1000000:] + download.completed=True + download.put() + + logging.info("Download finished OK") + return + def toPercentDecimal(match): - "Return the %decimal number for the character for url escaping" - s = match.group(1) - return "%%%02x" % ord(s) + "Return the %decimal number for the character for url escaping" + s = match.group(1) + return "%%%02x" % ord(s) def urlEscape(data): - "Escape text, including unicode, for use in URLs" - p = re.compile(r'([^\w])') - return p.sub(toPercentDecimal, data.encode("utf-8")) + "Escape text, including unicode, for use in URLs" + p = re.compile(r'([^\w])') + return p.sub(toPercentDecimal, data.encode("utf-8")) def main(): application = webapp.WSGIApplication([('/', MainHandler), - ('/fdowntask', FanfictionDownloaderTask), - ('/fdown', FanfictionDownloader), - (r'/file.*', FileServer), - ('/status', FileStatusServer), - ('/recent', RecentFilesServer), - ('/r2d2', RecentAllFilesServer), - ('/login', LoginRequired)], + ('/fdowntask', FanfictionDownloaderTask), + ('/fdown', FanfictionDownloader), + (r'/file.*', FileServer), + ('/status', FileStatusServer), + ('/recent', RecentFilesServer), + ('/editconfig', EditConfigServer), + ], debug=False) util.run_wsgi_app(application) if __name__ == '__main__': - logging.getLogger().setLevel(logging.DEBUG) - main() + logging.getLogger().setLevel(logging.DEBUG) + main() diff --git a/newdownload.py b/newdownload.py index 49c12697..550ad5fd 100644 --- a/newdownload.py +++ b/newdownload.py @@ -10,7 +10,7 @@ from fanficdownloader import adapters,writers,exceptions import ConfigParser -config = ConfigParser.ConfigParser() +config = ConfigParser.SafeConfigParser() logging.debug('reading defaults.ini config file, if present') config.read('defaults.ini')