From 1f303013bdcac5fa0267b195accb35ff2974c173 Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sun, 3 Apr 2011 16:47:08 -0500 Subject: [PATCH] Fix to allow retrying, compress mobi chunks, Working/Finished/Failed in status.html title, keep 3 days worth. --- main.py | 35 ++++++++++++++++++++++++++++++++--- recent.html | 2 +- status.html | 3 ++- utils/remover.py | 2 +- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 2bcc8fd0..bf8b1238 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ import os import sys +import zlib import logging import traceback import StringIO @@ -121,8 +122,19 @@ class FileServer(webapp.RequestHandler): self.response.headers['Content-disposition'] = 'attachment; filename=' + name + '.mobi' data = DownloadData.all().filter("download =", fanfic).order("index") + # epub, txt and html are all already compressed. + # Each chunk is compress individually to avoid having + # to hold the whole in memory just for the + # compress/uncompress + if fanfic.format == 'mobi': + def dc(data): + return zlib.decompress(data) + else: + def dc(data): + return data + for datum in data: - self.response.out.write(datum.blob) + self.response.out.write(dc(datum.blob)) class FileStatusServer(webapp.RequestHandler): def get(self): @@ -196,6 +208,7 @@ class FanfictionDownloader(webapp.RequestHandler): else: download = q[0] download.completed=False + download.failure=None for c in download.data_chunks: c.delete() @@ -307,7 +320,11 @@ class FanfictionDownloaderTask(webapp.RequestHandler): else: writerClass = output.TextWriter - loader = FanficLoader(adapter, writerClass, quiet = True, inmemory=True, compress=False) + loader = FanficLoader(adapter, + writerClass, + quiet = True, + inmemory=True, + compress=False) try: data = loader.download() @@ -347,10 +364,22 @@ class FanfictionDownloaderTask(webapp.RequestHandler): download.author = self._printableVersion(adapter.getAuthorName()) download.put() index=0 + + # epub, txt and html are all already compressed. + # Each chunk is compressed individually to avoid having + # to hold the whole in memory just for the + # compress/uncompress. + if format == 'mobi': + def c(data): + return zlib.compress(data) + else: + def c(data): + return data + while( len(data) > 0 ): DownloadData(download=download, index=index, - blob=data[:1000000]).put() + blob=c(data[:1000000])).put() index += 1 data = data[1000000:] download.completed=True diff --git a/recent.html b/recent.html index dbe04fab..cea52742 100644 --- a/recent.html +++ b/recent.html @@ -49,7 +49,7 @@ {% if not fic.completed and not fic.failure %} Request Processing...
{% endif %} - {{ fic.url }} ({{ fic.format }}) + {{ fic.url }}

{% endfor %} diff --git a/status.html b/status.html index 64c03c30..e3c51f08 100644 --- a/status.html +++ b/status.html @@ -2,7 +2,7 @@ - Fanfiction Downloader - read fanfiction from twilighted.net, fanfiction.net, fictionpress.com, fictionalley.org, ficwad.com, potionsandsnitches.net, harrypotterfanfiction.com, mediaminer.org on Kindle, Nook, Sony Reader, iPad, iPhone, Android, Aldiko, Stanza + {% if fic.completed %} Finished {% else %} {% if fic.failure %} Failed {% else %} Working... {% endif %} {% endif %} - Fanfiction Downloader {% if not fic.completed and not fic.failure %} @@ -48,6 +48,7 @@

Not done yet. This page will periodically poll to see if your story has finished.

{% endif %} {% endif %} +

Or see your personal list of previously downloaded fanfics.

diff --git a/utils/remover.py b/utils/remover.py index 6ce8e300..954e151b 100644 --- a/utils/remover.py +++ b/utils/remover.py @@ -21,7 +21,7 @@ class Remover(webapp.RequestHandler): logging.debug("Starting r3m0v3r") user = users.get_current_user() logging.debug("Working as user %s" % user) - theDate = datetime.date.today() - datetime.timedelta(days=3) + theDate = datetime.date.today() - datetime.timedelta(days=2) logging.debug("Will delete stuff older than %s" % theDate) fics = DownloadMeta.all()