replace_metadata feature--allow regexp replacement of story metadata from ini.

calibre-plugin/ffdl_plugin.py

@@ -318,13 +318,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
                            self._update_existing_2,
                            init_label="Collecting stories for update...",
                            win_title="Get stories for updates",
-                           status_prefix="URL retrieved")
-            
+                           status_prefix="URL retrieved")            
         
         #books = self._convert_calibre_ids_to_books(db, book_ids)
         #print("update books:%s"%books)
 
-        ## XXX split here.
     def _update_existing_2(self,book_list):
         
         d = UpdateExistingDialog(self.gui,
@@ -370,10 +368,11 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
 
         self.gui.status_bar.show_message(_('Started fetching metadata for %s stories.'%len(books)), 3000)
         
-        LoopProgressDialog(self.gui,
-                           books,
-                           partial(self.get_metadata_for_book, options = options),
-                           partial(self.start_download_list, options = options))
+        if 0 < len(filter(lambda x : x['good'], books)):
+            LoopProgressDialog(self.gui,
+                               books,
+                               partial(self.get_metadata_for_book, options = options),
+                               partial(self.start_download_list, options = options))
         # LoopProgressDialog calls get_metadata_for_book for each 'good' story,
         # get_metadata_for_book updates book for each,
         # LoopProgressDialog calls start_download_list at the end which goes
@@ -689,14 +688,15 @@ class FanFictionDownLoaderPlugin(InterfaceAction):
             total_good = len(good_list)
 
             self.gui.status_bar.show_message(_('Adding/Updating %s books.'%total_good))
-            
-            LoopProgressDialog(self.gui,
-                               good_list,
-                               partial(self._update_book, options=options, db=self.gui.current_db),
-                               partial(self._update_books_completed, options=options),
-                               init_label="Updating calibre for stories...",
-                               win_title="Update calibre for stories",
-                               status_prefix="Updated")
+
+            if total_good > 0:
+                LoopProgressDialog(self.gui,
+                                   good_list,
+                                   partial(self._update_book, options=options, db=self.gui.current_db),
+                                   partial(self._update_books_completed, options=options),
+                                   init_label="Updating calibre for stories...",
+                                   win_title="Update calibre for stories",
+                                   status_prefix="Updated")
 
     def _add_or_update_book(self,book,options,prefs,mi=None):
         db = self.gui.current_db

defaults.ini

@@ -133,6 +133,19 @@ extratags: FanFiction
 ## values are available, plus output_filename.
 #post_process_cmd: addbook -f "${output_filename}" -t "${title}"
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
 
 ## Each output format has a section that overrides [defaults]
 [html]

downloader.py

@@ -25,7 +25,6 @@ from StringIO import StringIO
 from optparse import OptionParser      
 import getpass
 import string
-import time
 
 from subprocess import call
 
@@ -215,6 +214,7 @@ def main():
        print us
    
 if __name__ == "__main__":
-    start = time.time()
+    #import time
+    #start = time.time()
     main()
-    print("Total time seconds:%f"%(time.time()-start))
+    #print("Total time seconds:%f"%(time.time()-start))

fanficdownloader/adapters/adapter_test1.py

@@ -104,7 +104,8 @@ Some more longer description.  "I suck at summaries!"  "Better than it sounds!"
         self.story.addToList('category','Harry Potter')
         self.story.addToList('category','Furbie')
         self.story.addToList('category','Crossover')
-        
+        self.story.addToList('category',u'Puella Magi Madoka Magica/魔法少女まどか★マギカ')
+        self.story.addToList('category',u'Magical Girl Lyrical Nanoha')
         self.story.addToList('genre','Fantasy')
         self.story.addToList('genre','SF')
         self.story.addToList('genre','Noir')

fanficdownloader/story.py

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-import os
+import os, re
 
 from htmlcleanup import conditionalRemoveEntities, removeAllEntities
 
@@ -26,6 +26,7 @@ class Story:
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
             self.metadata = {'version':'4.3'}
+        self.replacements = []
         self.chapters = [] # chapters will be tuples of (title,html)
         self.listables = {} # some items (extratags, category, warnings & genres) are also kept as lists.
 
@@ -36,6 +37,12 @@ class Story:
     def getMetadataRaw(self,key):
         if self.metadata.has_key(key):
             return self.metadata[key]
+
+    def doReplacments(self,value):
+        for (p,v) in self.replacements:
+            if (isinstance(value,str) or isinstance(value,unicode)) and re.match(p,value):
+                value = re.sub(p,v,value)                
+        return value;
         
     def getMetadata(self, key, removeallentities=False):
         value = None
@@ -50,7 +57,8 @@ class Story:
                     value = value.strftime("%Y-%m-%d %H:%M:%S")
                 if key == "datePublished" or key == "dateUpdated":
                  value = value.strftime("%Y-%m-%d")
-                 
+
+        value=self.doReplacments(value)
         if removeallentities and value != None:
             return removeAllEntities(value)
         else:
@@ -81,10 +89,14 @@ class Story:
     def getList(self,listname):
         if not self.listables.has_key(listname):
             return []
-        return self.listables[listname]
+        return filter( lambda x : x!=None and x!='' ,
+                       map(self.doReplacments,self.listables[listname]) )
 
     def getLists(self):
-        return self.listables
+        lsts = {}
+        for ln in self.listables.keys():
+            lsts[ln] = self.getList(ln)
+        return lsts
     
     def addChapter(self, title, html):
         self.chapters.append( (title,html) )
@@ -96,6 +108,12 @@ class Story:
     def __str__(self):
         return "Metadata: " +str(self.metadata) + "\nListables: " +str(self.listables) #+ "\nChapters: "+str(self.chapters)
 
+    def setReplace(self,replace):
+        for line in replace.splitlines():
+            if "=>" in line:
+                print("line:%s"%line)
+                self.replacements.append(map( lambda x: x.strip(), line.split("=>") ))
+    
 def commaGroups(s):
     groups = []
     while s and s[-1].isdigit():

fanficdownloader/writers/base_writer.py

@@ -46,6 +46,9 @@ class BaseStoryWriter(Configurable):
         
         self.adapter = adapter
         self.story = adapter.getStoryMetadataOnly() # only cache the metadata initially.
+        
+        self.story.setReplace(self.getConfig('replace_metadata'))
+        
         self.validEntries = [
             'category',
             'genre',

plugin-defaults.ini

@@ -111,6 +111,20 @@ extratags: FanFiction
 ## epub by many readers).  Must be hex code, # will be added.
 background_color: ffffff
 
+## Use regular expressions to find and replace (or remove) metadata.
+## For example, you could change Sci-Fi=>SF, remove *-Centered tags,
+## etc.  See http://docs.python.org/library/re.html (look for re.sub)
+## for regexp details.
+## Make sure to keep at least one space at the start of each line and
+## to escape % to %%, if used.
+#replace_metadata:
+# Sci-Fi=>SF
+# Puella Magi Madoka Magica.* => Madoka
+# Comedy=>Humor
+# Crossover: (.*)=>\1
+# (.*)Great(.*)=>\1Moderate\2
+# .*-Centered=>
+
 ## Each output format has a section that overrides [defaults]
 [html]