From 0c85cf1d102c2001bf74256032e19cf09931454c Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Thu, 19 Apr 2012 12:18:40 -0500 Subject: [PATCH] Web- Add list of all downloads feature to web service. --- allrecent.html | 78 +++++++++++++++++++++++++++++++++++++++ app.yaml | 6 +-- fanficdownloader/story.py | 11 +++--- ffstorage.py | 24 ++++++++++++ index.html | 11 ++++-- main.py | 51 +++++++++++++++++++++++++ 6 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 allrecent.html diff --git a/allrecent.html b/allrecent.html new file mode 100644 index 00000000..477b17b7 --- /dev/null +++ b/allrecent.html @@ -0,0 +1,78 @@ + + + + + FanFictionDownLoader (fanfiction.net, fanficauthors, fictionalley, ficwad to epub and HTML) + + + + +
+

+ FanFictionDownLoader +

+ + + + + {{yourfile}} + + +
+ {% for fic in fics %} +

+ {{ fic.title }} + by {{ fic.author }} Download Count: {{ fic.count }}
+ Word Count: {{ fic.numWords }} Chapter Count: {{ fic.numChapters }}
+ {% if fic.category %} Categories: {{ fic.category }}
{% endif %} + {% if fic.genre %} Genres: {{ fic.genre }}
{% endif %} + {% if fic.language %} Language: {{ fic.language }}
{% endif %} + {% if fic.series %} Series: {{ fic.series }}
{% endif %} + {% if fic.characters %} Characters: {{ fic.characters }}
{% endif %} + {% if fic.status %} Status: {{ fic.status }}
{% endif %} + {% if fic.datePublished %} Published: {{ fic.datePublished }}
{% endif %} + {% if fic.dateUpdated %} Last Updated: {{ fic.dateUpdated }}
{% endif %} + {% if fic.dateCreated %} Last Downloaded: {{ fic.dateCreated }}
{% endif %} + {% if fic.rating %} Rating: {{ fic.rating }}
{% endif %} + {% if fic.warnings %} Warnings: {{ fic.warnings }}
{% endif %} + {% if fic.description %} Summary: {{ fic.description }}
{% endif %} +

+ {% endfor %} +
+ + + + +
+ + diff --git a/app.yaml b/app.yaml index 5b75406c..3e09de3b 100644 --- a/app.yaml +++ b/app.yaml @@ -1,6 +1,6 @@ # ffd-retief-hrd fanfictiondownloader application: fanfictiondownloader -version: 4-4-5 +version: 4-4-7 runtime: python27 api_version: 1 threadsafe: true @@ -35,8 +35,8 @@ handlers: - url: /.* script: main.app -builtins: -- datastore_admin: on +#builtins: +#- datastore_admin: on libraries: - name: django diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index c8f06a04..5e94164f 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -218,7 +218,7 @@ class Story: value = re.sub(p,v,value) return value - def getMetadata(self, key, removeallentities=False): + def getMetadata(self, key, removeallentities=False, doreplacements=True): value = None if self.getLists().has_key(key): value = ', '.join(self.getList(key)) @@ -232,21 +232,22 @@ class Story: if key == "datePublished" or key == "dateUpdated": value = value.strftime("%Y-%m-%d") - value=self.doReplacments(value) + if doreplacements: + value=self.doReplacments(value) if removeallentities and value != None: return removeAllEntities(value) else: return value - def getAllMetadata(self, removeallentities=False): + def getAllMetadata(self, removeallentities=False, doreplacements=True): ''' All single value *and* list value metadata as strings. ''' allmetadata = {} for k in self.metadata.keys(): - allmetadata[k] = self.getMetadata(k, removeallentities) + allmetadata[k] = self.getMetadata(k, removeallentities, doreplacements) for l in self.listables.keys(): - allmetadata[l] = self.getMetadata(l, removeallentities) + allmetadata[l] = self.getMetadata(l, removeallentities, doreplacements) return allmetadata diff --git a/ffstorage.py b/ffstorage.py index 92e29d04..bad9b4a4 100644 --- a/ffstorage.py +++ b/ffstorage.py @@ -13,8 +13,23 @@ # limitations under the License. # +import pickle, copy from google.appengine.ext import db +class ObjectProperty(db.Property): + data_type = db.Blob + + def get_value_for_datastore(self, model_instance): + value = self.__get__(model_instance, model_instance.__class__) + pickled_val = pickle.dumps(value,protocol=pickle.HIGHEST_PROTOCOL) + if value is not None: return db.Blob(pickled_val) + + def make_value_from_datastore(self, value): + if value is not None: return pickle.loads(value) + + def default_value(self): + return copy.copy(self.default) + class DownloadMeta(db.Model): user = db.UserProperty() url = db.StringProperty() @@ -37,3 +52,12 @@ class DownloadData(db.Model): class UserConfig(db.Model): user = db.UserProperty() config = db.BlobProperty() + +class SavedMeta(db.Model): + url = db.StringProperty() + title = db.StringProperty() + author = db.StringProperty() + date = db.DateTimeProperty(auto_now_add=True) + count = db.IntegerProperty() + meta = ObjectProperty() + diff --git a/index.html b/index.html index eb193e75..43ec8f2b 100644 --- a/index.html +++ b/index.html @@ -54,9 +54,11 @@ much easier.

-

New Site Added

+

New Feature Added

- Support for archive.skyehawke.com has been added. Thanks to Ida Leter for implementing this. + You can now see a list of downloaded fanfics by all + users by most popular + or most recent.

Questions? Check out our @@ -66,7 +68,7 @@ If you have any problems with this application, please report them in the FanFictionDownLoader Google Group. The - Previous Version is also available for you to use if necessary. + Previous Version is also available for you to use if necessary.

{{ error_message }} @@ -96,6 +98,9 @@

Or see your personal list of previously downloaded fanfics.

+

+ See a list of downloaded fanfics by all users by most popular or most recent. +

{% else %} diff --git a/main.py b/main.py index e2d4ab47..d6884ddc 100644 --- a/main.py +++ b/main.py @@ -303,6 +303,39 @@ class RecentFilesServer(webapp2.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'recent.html') self.response.out.write(template.render(path, template_values)) +class AllRecentFilesServer(webapp2.RequestHandler): + def get(self): + user = users.get_current_user() + if not user: + self.redirect(users.create_login_url(self.request.uri)) + return + + q = SavedMeta.all() + if self.request.get('bydate'): + q.order('-date') + else: + q.order('-count') + + fics = q.fetch(200) + logging.info("Recent fetched %d downloads for user %s."%(len(fics),user.nickname())) + + sendslugs = [] + + for fic in fics: + ficslug = FicSlug(fic) + sendslugs.append(ficslug) + + template_values = dict(fics = sendslugs, nickname = user.nickname()) + path = os.path.join(os.path.dirname(__file__), 'allrecent.html') + self.response.out.write(template.render(path, template_values)) + +class FicSlug(): + def __init__(self,savedmeta): + self.url = savedmeta.url + self.count = savedmeta.count + for k, v in savedmeta.meta.iteritems(): + setattr(self,k,v) + class FanfictionDownloader(UserConfigServer): def get(self): self.post() @@ -464,6 +497,8 @@ class FanfictionDownloaderTask(UserConfigServer): download.url = adapter.getStory().getMetadata('storyUrl') download.put() + allmeta = adapter.getStory().getAllMetadata(removeallentities=True,doreplacements=False) + outbuffer = StringIO() writer.writeStory(outbuffer) data = outbuffer.getvalue() @@ -495,6 +530,21 @@ class FanfictionDownloaderTask(UserConfigServer): download.completed=True download.put() + smetal = SavedMeta.all().filter('url =', allmeta['storyUrl'] ).fetch(1) + if smetal and smetal[0]: + smeta = smetal[0] + smeta.count += 1 + else: + smeta=SavedMeta() + smeta.count = 1 + + smeta.url = allmeta['storyUrl'] + smeta.title = allmeta['title'] + smeta.author = allmeta['author'] + smeta.meta = allmeta + smeta.date = datetime.datetime.now() + smeta.put() + logging.info("Download finished OK") del data @@ -563,6 +613,7 @@ app = webapp2.WSGIApplication([('/', MainHandler), ('/fdown', FanfictionDownloader), (r'/file.*', FileServer), ('/status', FileStatusServer), + ('/allrecent', AllRecentFilesServer), ('/recent', RecentFilesServer), ('/editconfig', EditConfigServer), ('/clearrecent', ClearRecentServer),