Merging python27 branch back into default(trunk) branch.

app.yaml

@@ -1,20 +1,22 @@
-# fanfictionloader ffd-retief
-application: fanfictionloader
-version: 4-0-7
-runtime: python
+# ffd-retief-hrd fanfictiondownloader
+application: fanfictiondownloader
+version: 4-1-1
+runtime: python27
 api_version: 1
+threadsafe: true
 
 handlers:
+
 - url: /r3m0v3r.*
-  script: utils/remover.py
+  script: utils.remover.app
   login: admin
 
 - url: /tally.*
-  script: utils/tally.py
+  script: utils.tally.app
   login: admin
 
 - url: /fdownloadtask
-  script: main.py
+  script: main.app
   login: admin
   
 - url: /css
@@ -31,7 +33,14 @@ handlers:
   upload: static/favicon\.ico
 
 - url: /.*
-  script: main.py
+  script: main.app
 
 builtins:
 - datastore_admin: on
+
+libraries:
+- name: django
+  version: "1.2"
+
+- name: PIL
+  version: "1.1.7"

cron.yaml

@@ -3,6 +3,8 @@ cron:
   url: /r3m0v3r
   schedule: every 2 hours
 
-- description: orphan cleanup job
-  url: /r3m0v3rOrphans
-  schedule: every 4 hours
+# There's a bug in the Python 2.7 runtime that prevents this from
+# working properly.  In theory, there should never be orphans anyway.
+#- description: orphan cleanup job
+#  url: /r3m0v3rOrphans
+#  schedule: every 4 hours

defaults.ini

@@ -113,6 +113,15 @@ extratags: FanFiction
 ## Primarily for commandline.
 #slow_down_sleep_time:0.5
 
+## output background color--only used by html and epub (and ignored in
+## epub by many readers).  Must be hex code, # will be added.
+background_color: ffffff
+
+## For use only with stand-alone CLI version--run a command on the
+## generated file after it's produced.  All of the titlepage_entries
+## values are available, plus output_filename.
+#post_process_cmd: addbook -f "${output_filename}" -t "${title}"
+
 ## Each output format has a section that overrides [defaults]
 [html]
 
@@ -245,6 +254,12 @@ output_filename: ${title}-${siteabbrev}_${authorId}_${storyId}${formatext}
 ## this should go in your personal.ini, not defaults.ini.
 #is_adult:true
 
+[www.tthfanfic.org]
+## Some sites do not require a login, but do require the user to
+## confirm they are adult for adult content.  In commandline version,
+## this should go in your personal.ini, not defaults.ini.
+#is_adult:true
+
 [overrides]
 ## It may sometimes be useful to override all of the specific format,
 ## site and site:format sections in your private configuration.  For

downloader.py

@@ -24,6 +24,9 @@ from os.path import normpath, expanduser, isfile, join
 from StringIO import StringIO
 from optparse import OptionParser      
 import getpass
+import string
+
+from subprocess import call
 
 from epubmerge import doMerge
 
@@ -38,7 +41,9 @@ import ConfigParser
 def writeStory(config,adapter,writeformat,metaonly=False,outstream=None):
     writer = writers.getWriter(writeformat,config,adapter)
     writer.writeStory(outstream=outstream,metaonly=metaonly)
+    output_filename=writer.getOutputFileName()
     del writer
+    return output_filename
 
 def main():
 
@@ -65,7 +70,7 @@ def main():
                      help="Update an existing epub with new chapter, give epub filename instead of storyurl.  Not compatible with inserted TOC.",)
    parser.add_option("--force",
                      action="store_true", dest="force",
-                     help="Force update of an existing epub, download and overwrite all chapters.",)
+                     help="Force overwrite or update of an existing epub, download and overwrite all chapters.",)
    
    (options, args) = parser.parse_args()
 
@@ -97,6 +102,10 @@ def main():
        config.add_section("overrides")
    except ConfigParser.DuplicateSectionError:
        pass
+
+   if options.force:
+       config.set("overrides","always_overwrite","true")
+       
    if options.options:
        for opt in options.options:
            (var,val) = opt.split('=')
@@ -112,7 +121,7 @@ def main():
                                         striptitletoc=True,
                                         forceunique=False)
            print "Updating %s, URL: %s" % (args[0],url)
-           filename = args[0]
+           output_filename = args[0]
            config.set("overrides","output_filename",args[0])
        else:
            url = args[0]
@@ -184,16 +193,14 @@ def main():
            
            adapter.setChaptersRange(options.begin,options.end)
            
-           if options.format == "all":
-               ## For testing.  Doing all three formats actually causes
-               ## some interesting config issues with format-specific
-               ## sections.  But it should rarely be an issue.
-               writeStory(config,adapter,"epub",options.metaonly)
-               writeStory(config,adapter,"html",options.metaonly)
-               writeStory(config,adapter,"txt",options.metaonly)
-           else:
-               writeStory(config,adapter,options.format,options.metaonly)
+           output_filename=writeStory(config,adapter,options.format,options.metaonly)
        
+       if not options.metaonly and adapter.getConfig("post_process_cmd"):
+           metadata = adapter.story.metadata
+           metadata['output_filename']=output_filename
+           call(string.Template(adapter.getConfig("post_process_cmd"))
+                .substitute(metadata))
+           
        del adapter
    
    except exceptions.InvalidStoryURL, isu:

epubmerge.py

@@ -216,7 +216,7 @@ def doMerge(outputio,files,authoropts=[],titleopt=None,descopt=None,
                         try:
                             outputepub.writestr(href,
                                                 epub.read(relpath+item.getAttribute("href")))
-                            if re.match(r'.*/(file|chapter)\d+\.xhtml',href):
+                            if re.match(r'.*/(file|chapter)\d+\.x?html',href):
                                 filecount+=1
                             items.append((id,href,item.getAttribute("media-type")))
                             filelist.append(href)

fanficdownloader/adapters/adapter_fimfictionnet.py

@@ -88,7 +88,8 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         
         soup = bs.BeautifulSoup(data).find("div", {"class":"content_box post_content_box"})
         
-        title, author = [link.text for link in soup.find("h2").findAll("a")]
+        title = soup.find("h2").find("a").text # first a link in first h2 is title.
+        author = soup.find("h2").find("span",{'class':'author'}).find("a").text
         self.story.setMetadata("title", title)
         self.story.setMetadata("author", author)
         self.story.setMetadata("authorId", author) # The author's name will be unique
@@ -158,4 +159,4 @@ class FimFictionNetSiteAdapter(BaseSiteAdapter):
         if soup == None:
             raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
         return utf8FromSoup(soup)
-        
+        

fanficdownloader/adapters/adapter_tthfanficorg.py

@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+
+# Copyright 2011 Fanficdownloader team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+import logging
+import re
+import urllib2
+import time
+
+import fanficdownloader.BeautifulSoup as bs
+from fanficdownloader.htmlcleanup import stripHTML
+import fanficdownloader.exceptions as exceptions
+
+from base_adapter import BaseSiteAdapter, utf8FromSoup, makeDate
+
+class TwistingTheHellmouthSiteAdapter(BaseSiteAdapter):
+
+    def __init__(self, config, url):
+        BaseSiteAdapter.__init__(self, config, url)
+        self.story.setMetadata('siteabbrev','tth')
+        self.dateformat = "%d %b %y"
+        self.is_adult=False
+        # get storyId from url--url validation guarantees query correct
+        m = re.match(self.getSiteURLPattern(),url)
+        if m:
+            self.story.setMetadata('storyId',m.group('id'))
+            logging.debug("storyId: (%s)"%self.story.getMetadata('storyId'))
+            # normalized story URL.
+            self._setURL("http://"+self.getSiteDomain()\
+                         +"/Story-"+self.story.getMetadata('storyId'))
+        else:
+            raise exceptions.InvalidStoryURL(url,
+                                             self.getSiteDomain(),
+                                             self.getSiteExampleURLs())
+
+    @staticmethod
+    def getSiteDomain():
+        return 'www.tthfanfic.org'
+
+    def getSiteExampleURLs(self):
+        return "http://www.tthfanfic.org/Story-5583 http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm ttp://www.tthfanfic.org/T-526321777890480578489880055880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm"
+
+    # http://www.tthfanfic.org/T-526321777848988007890480555880/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm
+    # http://www.tthfanfic.org/Story-5583
+    # http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm
+    def getSiteURLPattern(self):
+        return r"http://www.tthfanfic.org/(T-\d+/)?Story-(?P<id>\d+)(-\d+)?(/.*)?$"
+
+    def extractChapterUrlsAndMetadata(self):
+        # fetch the chapter.  From that we will get almost all the
+        # metadata and chapter list
+
+        url=self.url
+        logging.debug("URL: "+url)
+
+        # use BeautifulSoup HTML parser to make everything easier to find.
+        try:
+            data = self._fetchUrl(url)
+            soup = bs.BeautifulSoup(data)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+            
+        if "<h2>Story Not Found</h2>" in data:
+            raise exceptions.StoryDoesNotExist(url)
+        
+        if "NOTE: This story is rated FR21 which is above your chosen filter level." in data:
+            if self.is_adult or self.getConfig("is_adult"):
+                form = soup.find('form', {'id':'sitemaxratingform'})
+                params={'ctkn':form.find('input', {'name':'ctkn'})['value'],
+                        'sitemaxrating':'5'}
+                logging.info("Attempting to get rating cookie for %s" % url)
+                data = self._postUrl("http://"+self.getSiteDomain()+'/setmaxrating.php',params)
+                # refetch story page.
+                data = self._fetchUrl(url)
+                soup = bs.BeautifulSoup(data)
+            else:
+                raise exceptions.AdultCheckRequired(self.url)
+
+        # http://www.tthfanfic.org/AuthorStories-3449/Greywizard.htm
+        # Find authorid and URL from... author url.
+        a = soup.find('a', href=re.compile(r"^/AuthorStories-\d+"))
+        self.story.setMetadata('authorId',a['href'].split('/')[1].split('-')[1])
+        self.story.setMetadata('authorUrl','http://'+self.host+a['href'])
+        self.story.setMetadata('author',stripHTML(a))
+
+        try:
+            # going to pull part of the meta data from author list page.
+            logging.debug("author URL: "+self.story.getMetadata('authorUrl'))
+            authordata = self._fetchUrl(self.story.getMetadata('authorUrl'))
+            authorsoup = bs.BeautifulSoup(authordata)
+            # author can have several pages, scan until we find it.
+            while( not authorsoup.find('a', href=re.compile(r"^/Story-"+self.story.getMetadata('storyId'))) ):
+                nextpage = 'http://'+self.host+authorsoup.find('a', {'class':'arrowf'})['href']
+                logging.debug("author nextpage URL: "+nextpage)
+                authordata = self._fetchUrl(nextpage)
+                authorsoup = bs.BeautifulSoup(authordata)
+        except urllib2.HTTPError, e:
+            if e.code == 404:
+                raise exceptions.StoryDoesNotExist(url)
+            else:
+                raise e
+
+        storydiv = authorsoup.find('div', {'id':'st'+self.story.getMetadata('storyId'), 'class':re.compile(r"storylistitem")})
+        self.story.setMetadata('description',stripHTML(storydiv.find('div',{'class':'storydesc'})))
+        self.story.setMetadata('title',stripHTML(storydiv.find('a',{'class':'storylink'})))
+
+        verticaltable = soup.find('table', {'class':'verticaltable'})
+
+        BtVS = True
+        for cat in verticaltable.findAll('a', href=re.compile(r"^/Category-")):
+            if cat.string not in ['General', 'Non-BtVS/AtS Stories', 'BtVS/AtS Non-Crossover']:
+                self.story.addToList('category',cat.string)
+            else:
+                if cat.string == 'Non-BtVS/AtS Stories':
+                    BtVS = False
+        if BtVS:
+            self.story.addToList('category','Buffy: The Vampire Slayer')
+
+        verticaltabletds = verticaltable.findAll('td')
+        self.story.setMetadata('rating', verticaltabletds[2].string)
+        self.story.setMetadata('numWords', verticaltabletds[4].string)
+        
+        # Complete--if completed.
+        if 'Yes' in verticaltabletds[10].string:
+            self.story.setMetadata('status', 'Completed')
+        else:
+            self.story.setMetadata('status', 'In-Progress')
+            
+        self.story.setMetadata('datePublished',makeDate(stripHTML(verticaltabletds[8].string), self.dateformat))
+        self.story.setMetadata('dateUpdated',makeDate(stripHTML(verticaltabletds[9].string), self.dateformat))
+
+        for icon in storydiv.find('span',{'class':'storyicons'}).findAll('img'):
+            if( icon['title'] not in ['Non-Crossover'] ) :
+                self.story.addToList('genre',icon['title'])
+            
+        # Find the chapter selector 
+        select = soup.find('select', { 'name' : 'chapnav' } )
+    	 
+        if select is None:
+    	   # no selector found, so it's a one-chapter story.
+    	   self.chapterUrls.append((self.story.getMetadata('title'),url))
+        else:
+            allOptions = select.findAll('option')
+            for o in allOptions:
+                url = "http://"+self.host+o['value']
+                # just in case there's tags, like <i> in chapter titles.
+                self.chapterUrls.append((stripHTML(o),url))
+
+        self.story.setMetadata('numChapters',len(self.chapterUrls))
+        
+        return
+
+
+    def getChapterText(self, url):
+        logging.debug('Getting chapter text from: %s' % url)
+        soup = bs.BeautifulSoup(self._fetchUrl(url))
+
+        div = soup.find('div', {'id' : 'storyinnerbody'})
+        
+        if None == div:
+            raise exceptions.FailedToDownload("Error downloading Chapter: %s!  Missing required element!" % url)
+
+        # strip out included chapter title, if present, to avoid doubling up.
+        try:
+            div.find('h3').extract()
+        except:
+            pass
+        return utf8FromSoup(div)
+
+def getClass():
+    return TwistingTheHellmouthSiteAdapter
+

fanficdownloader/story.py

@@ -25,7 +25,7 @@ class Story:
         try:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
-            self.metadata = {'version':'4.0'}
+            self.metadata = {'version':'4.1'}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 

fanficdownloader/writers/writer_epub.py

@@ -41,7 +41,10 @@ class EpubWriter(BaseStoryWriter):
     def __init__(self, config, story):
         BaseStoryWriter.__init__(self, config, story)
 
-        self.EPUB_CSS='''body { margin-left: 2%; margin-right: 2%; margin-top: 2%; margin-bottom: 2%; text-align: justify; }
+        self.EPUB_CSS = string.Template('''
+body { margin: 2%;
+       text-align: justify;
+       background-color: #${background_color}; }
 pre { font-size: x-small; }
 sml { font-size: small; }
 h1 { text-align: center; }
@@ -63,7 +66,7 @@ h6 { text-align: center; }
 .smcap    {font-variant: small-caps;}
 .u        {text-decoration: underline;}
 .bold     {font-weight: bold;}
-'''
+''')
         
         self.EPUB_TITLE_PAGE_START = string.Template('''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@@ -376,7 +379,7 @@ h6 { text-align: center; }
         del tocncxdom
 
         # write stylesheet.css file.
-        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS)
+        outputepub.writestr("OEBPS/stylesheet.css",self.EPUB_CSS.substitute({"background_color":self.getConfig("background_color")})) 
 
         # write title page.
         if self.getConfig("titlepage_use_table"):

fanficdownloader/writers/writer_html.py

@@ -39,6 +39,7 @@ class HTMLWriter(BaseStoryWriter):
 <head>
 <title>${title} by ${author}</title>
 <style type="text/css">
+body { background-color: #${background_color}; }
 .CI {
     text-align:center;
     margin-top:0px;
@@ -94,6 +95,9 @@ class HTMLWriter(BaseStoryWriter):
 
     def writeStoryImpl(self, out):
 
+        # minor cheat, tucking bg into metadata.
+        if self.getConfig("background_color"):
+            self.story.metadata["background_color"] = self.getConfig("background_color")
         self._write(out,self.HTML_FILE_START.substitute(self.story.metadata))
 
         self.writeTitlePage(out,

index.html

@@ -54,7 +54,7 @@
 	      much easier. </p>
 	  </div>
 	  <!-- put announcements here, h3 is a good title size. -->
-	  <h3>Please Switch to <a href="http://fanfictiondownloader.appspot.com/">New Version</a></h3>
+	  <h3>This is the Official Multithreading Version</h3>
 	  <p>
 	    We have a new, more efficient, version of the system up now.  Please start using the 
 	    <a href="http://fanfictiondownloader.appspot.com/">New
@@ -63,24 +63,25 @@
 	      the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
 	      Downloader Google Group</a>.
 	  </p>
-	  <h3>New Google Quotas</h3>
-	  <p>
-	    Google has changed their quota limits for free
-	    applications using their AppEngine system, like this one.
-	    We expect that there will be times when the
 	    system exceeds it's permitted processing quota.
+	    This version of the application uses Python 2.7 and
+	    multithreading to try and reduce our usage.  Google
+	    considers Python 2.7 Experimental still, so there may be issues.
+	    <b>Good news!</b><br />
+	    The issue that was causing problems with downloading large stories
+	    has been fixed.
 	  </p>
 	  <p>
-	    You also have the option of running the downloader on your
-	    own computer if you have Python available.
-	    <a href="http://code.google.com/p/fanficdownloader/downloads/list">Download here.</a>
+	    <b>New Feature</b><br /> You can now set a custom
+	    parameter for background_color that will be used with html
+	    and epub output.  (Note: many epub readers ignore the bg color.)
 	  </p>
 	  <p>
 	    If you have any problems with this application, please
 	    report them in
 	    the <a href="http://groups.google.com/group/fanfic-downloader">Fanfiction
 	      Downloader Google Group</a>.  The
-	    <a href="http://4-0-6.fanfictionloader.appspot.com">Previous
+	    <a href="http://4-0-7.fanfictiondownloader.appspot.com">Previous
 	      Version</a> is also available for you to use if necessary.
 	  </p>
 	  <div id='error'>
@@ -213,6 +214,13 @@
 	    <br /> or the URL of any chapter, such as
 	    <br /><a href="http://www.fimfiction.com/story/123/1/">http://www.fimfiction.com/story/123/1/</a>.
 	  </dd>
+	  <dt>tthfanfic.org</dt>
+	  <dd>
+	    Use the URL of any story, with or without chapter, title and notice, such as
+	    <br /><a href="http://www.tthfanfic.org/Story-5583">http://www.tthfanfic.org/Story-5583</a>
+	    <br /><a href="http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm">http://www.tthfanfic.org/Story-5583/Greywizard+Marked+By+Kane.htm</a>.
+	    <br /><a href="http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm">http://www.tthfanfic.org/T-99999999/Story-26448-15/batzulger+Willow+Rosenberg+and+the+Mind+Riders.htm</a>.
+	  </dd>
 	</dl>	
 
 	

main.py

@@ -41,23 +41,24 @@ import ConfigParser
 ## Console page first, you will get a django version mismatch error when you
 ## to go hit one of the application pages.  Just change a file again, and
 ## make sure to hit an app page before the SDK page to clear it.
-os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
-from google.appengine.dist import use_library
-use_library('django', '1.2')
+#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+#from google.appengine.dist import use_library
+#use_library('django', '1.2')
 
 from google.appengine.ext import db
 from google.appengine.api import taskqueue
 from google.appengine.api import users
-from google.appengine.ext import webapp
+#from google.appengine.ext import webapp
+import webapp2
 from google.appengine.ext.webapp import template
-from google.appengine.ext.webapp import util
+#from google.appengine.ext.webapp2 import util
 from google.appengine.runtime import DeadlineExceededError
 
 from ffstorage import *
 
 from fanficdownloader import adapters, writers, exceptions
 
-class UserConfigServer(webapp.RequestHandler):
+class UserConfigServer(webapp2.RequestHandler):
     def getUserConfig(self,user):
         config = ConfigParser.SafeConfigParser()
 
@@ -73,7 +74,7 @@ class UserConfigServer(webapp.RequestHandler):
 
         return config
 
-class MainHandler(webapp.RequestHandler):
+class MainHandler(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if user:
@@ -160,7 +161,7 @@ class EditConfigServer(UserConfigServer):
             self.response.out.write(template.render(path, template_values))
 
 
-class FileServer(webapp.RequestHandler):
+class FileServer(webapp2.RequestHandler):
 
     def get(self):
         fileId = self.request.get('id')
@@ -221,7 +222,7 @@ class FileServer(webapp.RequestHandler):
             path = os.path.join(os.path.dirname(__file__), 'status.html')
             self.response.out.write(template.render(path, template_values))
 
-class FileStatusServer(webapp.RequestHandler):
+class FileStatusServer(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
@@ -257,7 +258,7 @@ class FileStatusServer(webapp.RequestHandler):
         path = os.path.join(os.path.dirname(__file__), 'status.html')
         self.response.out.write(template.render(path, template_values))
 
-class ClearRecentServer(webapp.RequestHandler):
+class ClearRecentServer(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
@@ -282,7 +283,7 @@ class ClearRecentServer(webapp.RequestHandler):
         logging.info('Deleted %d instances download.' % num)
         self.redirect("/?error=recentcleared")
 
-class RecentFilesServer(webapp.RequestHandler):
+class RecentFilesServer(webapp2.RequestHandler):
     def get(self):
         user = users.get_current_user()
         if not user:
@@ -556,20 +557,14 @@ def urlEscape(data):
     p = re.compile(r'([^\w])')
     return p.sub(toPercentDecimal, data.encode("utf-8"))
 
-def main():
-  application = webapp.WSGIApplication([('/', MainHandler),
-                                        ('/fdowntask', FanfictionDownloaderTask),
-                                        ('/fdown', FanfictionDownloader),
-                                        (r'/file.*', FileServer),
-                                        ('/status', FileStatusServer),
-                                        ('/recent', RecentFilesServer),
-                                        ('/editconfig', EditConfigServer),
-                                        ('/clearrecent', ClearRecentServer),
-                                        ],
-                                       debug=False)
-  util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/', MainHandler),
+                               ('/fdowntask', FanfictionDownloaderTask),
+                               ('/fdown', FanfictionDownloader),
+                               (r'/file.*', FileServer),
+                               ('/status', FileStatusServer),
+                               ('/recent', RecentFilesServer),
+                               ('/editconfig', EditConfigServer),
+                               ('/clearrecent', ClearRecentServer),
+                               ],
+                              debug=False)

utils/remover.py

@@ -25,15 +25,16 @@ Copyright 2011 Fanficdownloader team
 import datetime
 import logging
 
-from google.appengine.ext.webapp import util
-from google.appengine.ext import webapp
+#from google.appengine.ext.webapp import util
+import webapp2
+#from google.appengine.ext import webapp
 from google.appengine.api import users
 from google.appengine.api import taskqueue
 from google.appengine.api import memcache
 
 from ffstorage import *
 
-class Remover(webapp.RequestHandler):
+class Remover(webapp2.RequestHandler):
     def get(self):
         logging.debug("Starting r3m0v3r")
         user = users.get_current_user()
@@ -56,9 +57,10 @@ class Remover(webapp.RequestHandler):
             logging.debug('Delete '+d.url)
 
         logging.info('Deleted instances: %d' % num)
+        self.response.headers['Content-Type'] = 'text/html'
         self.response.out.write('Deleted instances: %d<br>' % num)
 
-class RemoveOrphanDataChunks(webapp.RequestHandler):
+class RemoveOrphanDataChunks(webapp2.RequestHandler):
 
     def get(self):
         logging.debug("Starting RemoveOrphanDataChunks")
@@ -98,15 +100,10 @@ class RemoveOrphanDataChunks(webapp.RequestHandler):
             memcache.set('orphan_search_cursor',chunks.cursor())
         
         logging.info('Deleted %d orphan chunks from %d total.' % (deleted,num))
+        self.response.headers['Content-Type'] = 'text/html'
         self.response.out.write('Deleted %d orphan chunks from %d total.' % (deleted,num))
 
-def main():
-    application = webapp.WSGIApplication([('/r3m0v3r', Remover),
-                                          ('/r3m0v3rOrphans', RemoveOrphanDataChunks)],
-                                         debug=False)
-    util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/r3m0v3r', Remover),
+                               ('/r3m0v3rOrphans', RemoveOrphanDataChunks)],
+                              debug=False)

utils/tally.py

@@ -18,15 +18,16 @@
 import datetime
 import logging
 
-from google.appengine.ext.webapp import util
-from google.appengine.ext import webapp
+#from google.appengine.ext.webapp import util
+import webapp2
+#from google.appengine.ext import webapp
 from google.appengine.api import users
 from google.appengine.api import taskqueue
 from google.appengine.api import memcache
 
 from ffstorage import *
 
-class Tally(webapp.RequestHandler):
+class Tally(webapp2.RequestHandler):
     def get(self):
         logging.debug("Starting Tally")
         user = users.get_current_user()
@@ -57,13 +58,7 @@ class Tally(webapp.RequestHandler):
         logging.info('Tallied %d fics.' % num)
         self.response.out.write('<br/>Tallied %d fics.<br/>' % num)
 
-def main():
-    application = webapp.WSGIApplication([('/tally', Tally),
-                                          ],
-                                         debug=False)
-    util.run_wsgi_app(application)
-
-
-if __name__ == '__main__':
-    logging.getLogger().setLevel(logging.DEBUG)
-    main()
+logging.getLogger().setLevel(logging.DEBUG)
+app = webapp2.WSGIApplication([('/tally', Tally),
+                               ],
+                              debug=False)