diff --git a/cron.yaml b/cron.yaml index 325ad870..f6b65d39 100644 --- a/cron.yaml +++ b/cron.yaml @@ -2,3 +2,7 @@ cron: - description: cleanup job url: /r3m0v3r schedule: every 2 hours + +- description: orphan cleanup job + url: /r3m0v3rOrphans + schedule: every 48 hours diff --git a/utils/remover.py b/utils/remover.py index 4fd84d9d..9fb12d6e 100644 --- a/utils/remover.py +++ b/utils/remover.py @@ -28,6 +28,7 @@ import logging from google.appengine.ext.webapp import util from google.appengine.ext import webapp from google.appengine.api import users +from google.appengine.api import taskqueue from ffstorage import * @@ -43,11 +44,9 @@ class Remover(webapp.RequestHandler): fics.filter("date <",theDate).order("date") results = fics.fetch(100) - if not results: - self.response.out.write('Finished
') - break logging.debug([x.name for x in results]) + num=0 for d in results: d.delete() for c in d.data_chunks: @@ -59,7 +58,13 @@ class Remover(webapp.RequestHandler): self.response.out.write('Deleted instances: %d
' % num) class RemoveOrphanDataChunks(webapp.RequestHandler): + + ## enqueue the task because it will run longer than 30sec. def get(self): + taskqueue.add(url='/r3m0v3rOrphans') + self.response.out.write('Task Enqueued
') + + def post(self): logging.debug("Starting RemoveOrphanDataChunks") user = users.get_current_user() logging.debug("Working as user %s" % user) @@ -72,7 +77,6 @@ class RemoveOrphanDataChunks(webapp.RequestHandler): while( True ) : results = chunks.fetch(limit=step,offset=num-deleted) if not results: - self.response.out.write('Finished
') break for d in results: @@ -86,7 +90,6 @@ class RemoveOrphanDataChunks(webapp.RequestHandler): num += 1 logging.info('Deleted %d orphan chunks from %d total.' % (deleted,num)) - self.response.out.write('Deleted %d orphan chunks from %d total.
' % (deleted,num)) def main(): application = webapp.WSGIApplication([('/r3m0v3r', Remover),