diff --git a/app.yaml b/app.yaml index 80d89e3d..fbddee9e 100644 --- a/app.yaml +++ b/app.yaml @@ -5,14 +5,10 @@ runtime: python api_version: 1 handlers: -- url: /r3m0v3r +- url: /r3m0v3r.* script: utils/remover.py login: admin -- url: /r3m0v3r - script: main.py - login: admin - - url: /fdownloadtask script: main.py login: admin diff --git a/cron.yaml b/cron.yaml index 325ad870..f6b65d39 100644 --- a/cron.yaml +++ b/cron.yaml @@ -2,3 +2,7 @@ cron: - description: cleanup job url: /r3m0v3r schedule: every 2 hours + +- description: orphan cleanup job + url: /r3m0v3rOrphans + schedule: every 48 hours diff --git a/fanficdownloader/adapters/adapter_mediaminerorg.py b/fanficdownloader/adapters/adapter_mediaminerorg.py index 9b7efb51..d35591ad 100644 --- a/fanficdownloader/adapters/adapter_mediaminerorg.py +++ b/fanficdownloader/adapters/adapter_mediaminerorg.py @@ -81,7 +81,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter): # [ A - All Readers ], strip '[' ']' ## Above title because we remove the smtxt font to get title. - rating = soup.find("font",{"class":"smtxt"}).string[1:-1] + smtxt = soup.find("font",{"class":"smtxt"}) + if not smtxt: + raise exceptions.StoryDoesNotExist(self.url) + rating = smtxt.string[1:-1] self.story.setMetadata('rating',rating) # Find authorid and URL from... author url. diff --git a/main.py b/main.py index 7496909b..9aa5e488 100644 --- a/main.py +++ b/main.py @@ -71,6 +71,8 @@ class MainHandler(webapp.RequestHandler): template_values['error_message'] = 'Error happened: ' + self.request.get('errtext') elif error == 'configsaved': template_values['error_message'] = 'Configuration Saved' + elif error == 'recentcleared': + template_values['error_message'] = 'Your Recent Downloads List has been Cleared' filename = self.request.get('file') if len(filename) > 1: @@ -197,7 +199,6 @@ class FileServer(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'status.html') self.response.out.write(template.render(path, template_values)) - class FileStatusServer(webapp.RequestHandler): def get(self): user = users.get_current_user() @@ -235,16 +236,42 @@ class FileStatusServer(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'status.html') self.response.out.write(template.render(path, template_values)) -class RecentFilesServer(webapp.RequestHandler): +class ClearRecentServer(webapp.RequestHandler): def get(self): user = users.get_current_user() if not user: self.redirect(users.create_login_url(self.request.uri)) return + logging.info("Clearing Recent List for user: "+user.nickname()) + q = DownloadMeta.all() + q.filter('user =', user) + num=0 + while( True ): + results = q.fetch(100) + if results: + for d in results: + d.delete() + for c in d.data_chunks: + c.delete() + num = num + 1 + logging.debug('Delete '+d.url) + else: + break + logging.info('Deleted %d instances download.' % num) + self.redirect("/?error=recentcleared") + +class RecentFilesServer(webapp.RequestHandler): + def get(self): + user = users.get_current_user() + if not user: + self.redirect(users.create_login_url(self.request.uri)) + return + q = DownloadMeta.all() q.filter('user =', user).order('-date') fics = q.fetch(100) + logging.info("Recent fetched %d downloads for user %s."%(len(fics),user.nickname())) for fic in fics: if fic.completed and fic.format == 'epub': @@ -522,6 +549,7 @@ def main(): ('/status', FileStatusServer), ('/recent', RecentFilesServer), ('/editconfig', EditConfigServer), + ('/clearrecent', ClearRecentServer), ], debug=False) util.run_wsgi_app(application) diff --git a/recent.html b/recent.html index 2bd3fdb1..6baad0f7 100644 --- a/recent.html +++ b/recent.html @@ -41,7 +41,8 @@
- Hi, {{ nickname }}! These are the fanfics you've recently requested. +

Hi, {{ nickname }}! These are the fanfics you've recently requested.

+

Clear your Recent Downloads List

diff --git a/utils/remover.py b/utils/remover.py index 5162f1b2..9311fcb6 100644 --- a/utils/remover.py +++ b/utils/remover.py @@ -32,36 +32,71 @@ from google.appengine.api import users from ffstorage import * class Remover(webapp.RequestHandler): - def get(self): - logging.debug("Starting r3m0v3r") - user = users.get_current_user() - logging.debug("Working as user %s" % user) - theDate = datetime.date.today() - datetime.timedelta(days=7) - logging.debug("Will delete stuff older than %s" % theDate) + def get(self): + logging.debug("Starting r3m0v3r") + user = users.get_current_user() + logging.debug("Working as user %s" % user) + theDate = datetime.datetime.now() - datetime.timedelta(days=7) # days=7 + logging.debug("Will delete stuff older than %s" % theDate) + + fics = DownloadMeta.all() + fics.filter("date <",theDate).order("date") + + num = 0 + while( True ) : + results = fics.fetch(100) + if not results: + self.response.out.write('Finished
') + break + logging.debug([x.name for x in results]) - fics = DownloadMeta.all() - fics.filter("date <",theDate).order("date") - results = fics.fetch(100) - logging.debug([x.name for x in results]) + for d in results: + d.delete() + for c in d.data_chunks: + c.delete() + num += 1 + logging.debug('Delete '+d.url) - num = 0 - for d in results: - d.delete() - for c in d.data_chunks: - c.delete() - num = num + 1 - logging.debug('Delete '+d.url) - - logging.info('Deleted instances: %d' % num) - self.response.out.write('Deleted instances: %d' % num) - + logging.info('Deleted instances: %d' % num) + self.response.out.write('Deleted instances: %d
' % num) + +class RemoveOrphanDataChunks(webapp.RequestHandler): + def get(self): + logging.debug("Starting RemoveOrphanDataChunks") + user = users.get_current_user() + logging.debug("Working as user %s" % user) + + chunks = DownloadData.all() + + deleted = 0 + num = 0 + step=2 + while( True ) : + results = chunks.fetch(limit=step,offset=num-deleted) + if not results: + self.response.out.write('Finished
') + break + + for d in results: + ## This is the only way to test for orphans I could find. + try: + meta = d.download + except db.ReferencePropertyResolveError: + ## delete orphan chunk. + d.delete() + deleted += 1 + num += 1 + + logging.info('Deleted %d orphan chunks from %d total.' % (deleted,num)) + self.response.out.write('Deleted %d orphan chunks from %d total.
' % (deleted,num)) def main(): - application = webapp.WSGIApplication([('/r3m0v3r', Remover)], - debug=False) - util.run_wsgi_app(application) + application = webapp.WSGIApplication([('/r3m0v3r', Remover), + ('/r3m0v3rOrphans', RemoveOrphanDataChunks)], + debug=False) + util.run_wsgi_app(application) if __name__ == '__main__': - logging.getLogger().setLevel(logging.DEBUG) - main() + logging.getLogger().setLevel(logging.DEBUG) + main()