Improve download cleanup, add orphan chunk cleanup, add recent list clear,

improve mediaminer no such story detect.

app.yaml

@@ -5,14 +5,10 @@ runtime: python
 api_version: 1
 
 handlers:
-- url: /r3m0v3r
+- url: /r3m0v3r.*
   script: utils/remover.py
   login: admin
 
-- url: /r3m0v3r
-  script: main.py
-  login: admin
-
 - url: /fdownloadtask
   script: main.py
   login: admin

cron.yaml

@@ -2,3 +2,7 @@ cron:
 - description: cleanup job
   url: /r3m0v3r
   schedule: every 2 hours
+
+- description: orphan cleanup job
+  url: /r3m0v3rOrphans
+  schedule: every 48 hours

fanficdownloader/adapters/adapter_mediaminerorg.py

@@ -81,7 +81,10 @@ class MediaMinerOrgSiteAdapter(BaseSiteAdapter):
 
         # [ A - All Readers ], strip '[' ']'
         ## Above title because we remove the smtxt font to get title.
-        rating = soup.find("font",{"class":"smtxt"}).string[1:-1]
+        smtxt = soup.find("font",{"class":"smtxt"})
+        if not smtxt:
+            raise exceptions.StoryDoesNotExist(self.url)
+        rating = smtxt.string[1:-1]
         self.story.setMetadata('rating',rating)
 
         # Find authorid and URL from... author url.

main.py

@@ -71,6 +71,8 @@ class MainHandler(webapp.RequestHandler):
                     template_values['error_message'] = 'Error happened: ' + self.request.get('errtext')
                 elif error == 'configsaved':
                     template_values['error_message'] = 'Configuration Saved'
+                elif error == 'recentcleared':
+                    template_values['error_message'] = 'Your Recent Downloads List has been Cleared'
             
             filename = self.request.get('file')
             if len(filename) > 1:
@@ -197,7 +199,6 @@ class FileServer(webapp.RequestHandler):
             path = os.path.join(os.path.dirname(__file__), 'status.html')
             self.response.out.write(template.render(path, template_values))
             
-
 class FileStatusServer(webapp.RequestHandler):
     def get(self):
         user = users.get_current_user()
@@ -235,16 +236,42 @@ class FileStatusServer(webapp.RequestHandler):
         path = os.path.join(os.path.dirname(__file__), 'status.html')
         self.response.out.write(template.render(path, template_values))
         
-class RecentFilesServer(webapp.RequestHandler):
+class ClearRecentServer(webapp.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
             self.redirect(users.create_login_url(self.request.uri))
             return
         
+        logging.info("Clearing Recent List for user: "+user.nickname())
+        q = DownloadMeta.all()
+        q.filter('user =', user)
+        num=0
+        while( True ):
+            results = q.fetch(100)
+            if results:
+                for d in results:
+                    d.delete()
+                    for c in d.data_chunks:
+                        c.delete()
+                    num = num + 1
+                    logging.debug('Delete '+d.url)
+            else:
+                break
+        logging.info('Deleted %d instances download.' % num)
+        self.redirect("/?error=recentcleared")
+            
+class RecentFilesServer(webapp.RequestHandler):
+    def get(self):
+        user = users.get_current_user()
+        if not user:
+            self.redirect(users.create_login_url(self.request.uri))
+            return
+
         q = DownloadMeta.all()
         q.filter('user =', user).order('-date')
         fics = q.fetch(100)
+        logging.info("Recent fetched %d downloads for user %s."%(len(fics),user.nickname()))
 
         for fic in fics:
             if fic.completed and fic.format == 'epub':
@@ -522,6 +549,7 @@ def main():
                                         ('/status', FileStatusServer),
                                         ('/recent', RecentFilesServer),
                                         ('/editconfig', EditConfigServer),
+                                        ('/clearrecent', ClearRecentServer),
                                         ],
                                        debug=False)
   util.run_wsgi_app(application)

recent.html

@@ -41,7 +41,8 @@
       
       <div id='urlbox'>
 	<div id='greeting'>
-	  Hi, {{ nickname }}! These are the fanfics you've recently requested.
+	  <p>Hi, {{ nickname }}! These are the fanfics you've recently requested.</p>
+	  <p><a href="/clearrecent">Clear your Recent Downloads List</a></p>
 	</div>
       </div>
       

utils/remover.py

@@ -32,36 +32,71 @@ from google.appengine.api import users
 from ffstorage import *
 
 class Remover(webapp.RequestHandler):
-	def get(self):
-		logging.debug("Starting r3m0v3r")
-		user = users.get_current_user()
-		logging.debug("Working as user %s" % user)
-		theDate = datetime.date.today() - datetime.timedelta(days=7)
-		logging.debug("Will delete stuff older than %s" % theDate)
+    def get(self):
+        logging.debug("Starting r3m0v3r")
+        user = users.get_current_user()
+        logging.debug("Working as user %s" % user)
+        theDate = datetime.datetime.now() - datetime.timedelta(days=7) # days=7
+        logging.debug("Will delete stuff older than %s" % theDate)
+        
+        fics = DownloadMeta.all()
+        fics.filter("date <",theDate).order("date")
+        
+        num = 0
+        while( True ) :
+            results = fics.fetch(100)
+            if not results:
+                self.response.out.write('Finished<br>')
+                break
+            logging.debug([x.name for x in results])
 
-		fics = DownloadMeta.all()
-		fics.filter("date <",theDate).order("date")
-		results = fics.fetch(100)
-		logging.debug([x.name for x in results])
+            for d in results:
+                d.delete()
+                for c in d.data_chunks:
+                    c.delete()
+                num += 1
+                logging.debug('Delete '+d.url)
 
-		num = 0
-		for d in results:
-			d.delete()
-			for c in d.data_chunks:
-				c.delete()
-			num = num + 1
-			logging.debug('Delete '+d.url)
-		
-		logging.info('Deleted instances: %d' % num)
-		self.response.out.write('Deleted instances: %d' % num)
-		
+        logging.info('Deleted instances: %d' % num)
+        self.response.out.write('Deleted instances: %d<br>' % num)
+
+class RemoveOrphanDataChunks(webapp.RequestHandler):
+    def get(self):
+        logging.debug("Starting RemoveOrphanDataChunks")
+        user = users.get_current_user()
+        logging.debug("Working as user %s" % user)
+        
+        chunks = DownloadData.all()
+
+        deleted = 0
+        num = 0
+        step=2
+        while( True ) :
+            results = chunks.fetch(limit=step,offset=num-deleted)
+            if not results:
+                self.response.out.write('Finished<br>')
+                break
+
+            for d in results:
+                ## This is the only way to test for orphans I could find.
+                try:
+                    meta = d.download
+                except db.ReferencePropertyResolveError:
+                    ## delete orphan chunk.
+                    d.delete()
+                    deleted += 1
+                num += 1
+        
+        logging.info('Deleted %d orphan chunks from %d total.' % (deleted,num))
+        self.response.out.write('Deleted %d orphan chunks from %d total.<br>' % (deleted,num))
 
 def main():
-	application = webapp.WSGIApplication([('/r3m0v3r', Remover)],
-                                               debug=False)
-	util.run_wsgi_app(application)
+    application = webapp.WSGIApplication([('/r3m0v3r', Remover),
+                                          ('/r3m0v3rOrphans', RemoveOrphanDataChunks)],
+                                         debug=False)
+    util.run_wsgi_app(application)
 
 
 if __name__ == '__main__':
-	logging.getLogger().setLevel(logging.DEBUG)
-	main()
+    logging.getLogger().setLevel(logging.DEBUG)
+    main()