Merge default with branch.

calibre-plugin/basicinihighlighter.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2014, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+try:
+    from PyQt5.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush)
+except ImportError as e:
+    from PyQt4.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush)
+
+class BasicIniHighlighter(QSyntaxHighlighter):
+    '''
+    QSyntaxHighlighter class for use with QTextEdit for highlighting
+    ini config files.
+
+    I looked high and low to find a high lighter for basic ini config
+    format, so I'm leaving this in the project even though I'm not
+    using.
+    '''
+    
+    def __init__( self, parent, theme ):
+        QSyntaxHighlighter.__init__( self, parent )
+        self.parent = parent
+        
+        self.highlightingRules = []
+
+        # keyword
+        self.highlightingRules.append( HighlightingRule( r"^[^:=\s][^:=]*[:=]",
+                                                         Qt.blue,
+                                                         Qt.SolidPattern ) )
+
+        # section
+        self.highlightingRules.append( HighlightingRule( r"^\[[^\]]+\]",
+                                                         Qt.darkBlue,
+                                                         Qt.SolidPattern ) )
+
+        # comment
+        self.highlightingRules.append( HighlightingRule( r"#[^\n]*" ,
+                                                         Qt.darkYellow,
+                                                         Qt.SolidPattern ) )
+
+    def highlightBlock( self, text ):
+        for rule in self.highlightingRules:
+            for match in rule.pattern.finditer(text):
+                self.setFormat( match.start(), match.end()-match.start(), rule.highlight )
+        self.setCurrentBlockState( 0 )
+
+class HighlightingRule():
+    def __init__( self, pattern, color, style ):
+        if isinstance(pattern,basestring):
+            self.pattern = re.compile(pattern)
+        else:
+            self.pattern=pattern
+        charfmt = QTextCharFormat()
+        brush = QBrush(color, style)
+        charfmt.setForeground(brush)
+        self.highlight = charfmt
+    

calibre-plugin/config.py

@@ -40,7 +40,7 @@ else:
         return x.toPyObject()
 
 from calibre.gui2.ui import get_gui
-from calibre.gui2 import dynamic, info_dialog
+from calibre.gui2 import dynamic, info_dialog, question_dialog
 from calibre.constants import numeric_version as calibre_version
 
 # pulls in translation files for _() strings
@@ -73,7 +73,7 @@ no_trans = { 'pini':'personal.ini',
 from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs, PREFS_NAMESPACE
 from calibre_plugins.fanfictiondownloader_plugin.dialogs \
     import (UPDATE, UPDATEALWAYS, collision_order, save_collisions, RejectListDialog,
-            EditTextDialog, RejectUrlEntry)
+            EditTextDialog, IniTextDialog, RejectUrlEntry, errors_dialog)
     
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
     import getConfigSections
@@ -81,6 +81,9 @@ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters \
 from calibre_plugins.fanfictiondownloader_plugin.common_utils \
     import ( KeyboardConfigDialog, PrefsViewerDialog )
 
+from calibre_plugins.fanfictiondownloader_plugin.ffdl_util \
+    import (test_config)
+
 from calibre.gui2.complete2 import EditWithComplete  #MultiCompleteLineEdit
 
 class RejectURLList:
@@ -255,7 +258,7 @@ class ConfigWidget(QWidget):
             prefs['addtolistsonread'] = self.readinglist_tab.addtolistsonread.isChecked()
 
         # personal.ini
-        ini = unicode(self.personalini_tab.ini.toPlainText())
+        ini = self.personalini_tab.personalini
         if ini:
             prefs['personal.ini'] = ini
         else:
@@ -541,10 +544,6 @@ class BasicTab(QWidget):
         if i > -1:
             self.collision.setCurrentIndex(i)
         
-    def show_defaults(self):
-        text = get_resources('plugin-defaults.ini')
-        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
-
     def show_rejectlist(self):
         d = RejectListDialog(self,
                              rejecturllist.get_list(),
@@ -565,7 +564,8 @@ class BasicTab(QWidget):
                            icon=self.windowIcon(),
                            title=_("Reject Reasons"),
                            label=_("Customize Reject List Reasons"),
-                           tooltip=_("Customize the Reasons presented when Rejecting URLs"))
+                           tooltip=_("Customize the Reasons presented when Rejecting URLs"),
+                           save_size_name='ffdl:Reject List Reasons')
         d.exec_()
         if d.result() == d.Accepted:
             prefs['rejectreasons'] = d.get_plain_text()
@@ -578,7 +578,8 @@ class BasicTab(QWidget):
                            label=_("Add Reject URLs. Use: <b>http://...,note</b> or <b>http://...,title by author - note</b><br>Invalid story URLs will be ignored."),
                            tooltip=_("One URL per line:\n<b>http://...,note</b>\n<b>http://...,title by author - note</b>"),
                            rejectreasons=rejecturllist.get_reject_reasons(),
-                           reasonslabel=_('Add this reason to all URLs added:'))
+                           reasonslabel=_('Add this reason to all URLs added:'),
+                           save_size_name='ffdl:Add Reject List')
         d.exec_()
         if d.result() == d.Accepted:
             rejecturllist.add_text(d.get_plain_text(),d.get_reason_text())
@@ -596,62 +597,84 @@ class PersonalIniTab(QWidget):
         label = QLabel(_('These settings provide more detailed control over what metadata will be displayed inside the ebook as well as let you set %(isa)s and %(u)s/%(p)s for different sites.')%no_trans)
         label.setWordWrap(True)
         self.l.addWidget(label)
-        self.l.addSpacing(5)
+#        self.l.addSpacing(5)
+
+        label = QLabel(_("<b>New:</b> This experimental version includes find, color coding, and error checking.  Red generally indicates errors.  Not all errors can be found."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
         
-        self.label = QLabel('personal.ini:')
-        self.l.addWidget(self.label)
+        # self.label = QLabel('personal.ini:')
+        # self.l.addWidget(self.label)
 
-        self.ini = QTextEdit(self)
-        try:
-            self.ini.setFont(QFont("Courier",
-                                   self.plugin_action.gui.font().pointSize()+1))
-        except Exception as e:
-            logger.error("Couldn't get font: %s"%e)
-        self.ini.setLineWrapMode(QTextEdit.NoWrap)
-        self.ini.setText(prefs['personal.ini'])
-        self.l.addWidget(self.ini)
+        # self.ini = QTextEdit(self)
+        # try:
+        #     self.ini.setFont(QFont("Courier",
+        #                            self.plugin_action.gui.font().pointSize()+1))
+        # except Exception as e:
+        #     logger.error("Couldn't get font: %s"%e)
+        # self.ini.setLineWrapMode(QTextEdit.NoWrap)
+        # self.ini.setText(prefs['personal.ini'])
+        # self.l.addWidget(self.ini)
 
+        self.personalini = prefs['personal.ini']
+
+        self.ini_button = QPushButton(_('Edit personal.ini'), self)
+        self.ini_button.setToolTip(_("Edit personal.ini file."))
+        self.ini_button.clicked.connect(self.add_ini_button)
+        self.l.addWidget(self.ini_button)
+        
         self.defaults = QPushButton(_('View Defaults')+' (plugin-defaults.ini)', self)
         self.defaults.setToolTip(_("View all of the plugin's configurable settings\nand their default settings."))
         self.defaults.clicked.connect(self.show_defaults)
         self.l.addWidget(self.defaults)
+
+        label = QLabel(_("Changes will only be saved if you click 'OK' to leave Customize FFDL."))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
         
-        # self.l.insertStretch(-1)
+        self.l.insertStretch(-1)
         # let edit box fill the space.
         
     def show_defaults(self):
-        text = get_resources('plugin-defaults.ini')
-        ShowDefaultsIniDialog(self.windowIcon(),text,self).exec_()
+        IniTextDialog(self,
+                       get_resources('plugin-defaults.ini'),
+                       icon=self.windowIcon(),
+                       title=_('Plugin Defaults'),
+                       label=_("Plugin Defaults (%s) (Read-Only)")%'plugin-defaults.ini',
+                       tooltip=_("These are all of the plugin's configurable options\nand their default settings."),
+                       use_find=True,
+                       read_only=True,
+                       save_size_name='ffdl:defaults.ini').exec_()
         
-class ShowDefaultsIniDialog(QDialog):
+    def add_ini_button(self):
+        d = IniTextDialog(self,
+                           self.personalini,
+                           icon=self.windowIcon(),
+                           title=_("Edit personal.ini"),
+                           label=_("Edit personal.ini"),
+                           tooltip=_("Edit personal.ini"),
+                           use_find=True,
+                           save_size_name='ffdl:personal.ini')
+        retry=True
+        while retry:
+            d.exec_()
+            if d.result() == d.Accepted:
+                editini = d.get_plain_text()
+                errors = test_config(editini)
 
-    def __init__(self, icon, text, parent=None):
-        QDialog.__init__(self, parent)
-        self.resize(600, 500)
-        self.l = QVBoxLayout()
-        self.setLayout(self.l)
-        self.label = QLabel(_("Plugin Defaults (%s) (Read-Only)")%'plugin-defaults.ini')
-        self.label.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
-        self.setWindowTitle(_('Plugin Defaults'))
-        self.setWindowIcon(icon)
-        self.l.addWidget(self.label)
-        
-        self.ini = QTextEdit(self)
-        self.ini.setToolTip(_("These are all of the plugin's configurable options\nand their default settings."))
-        try:
-            self.ini.setFont(QFont("Courier",
-                                   get_gui().font().pointSize()+1))
-        except Exception as e:
-            logger.error("Couldn't get font: %s"%e)
-        self.ini.setLineWrapMode(QTextEdit.NoWrap)
-        self.ini.setText(text)
-        self.ini.setReadOnly(True)
-        self.l.addWidget(self.ini)
-        
-        self.ok_button = QPushButton(_('OK'), self)
-        self.ok_button.clicked.connect(self.hide)
-        self.l.addWidget(self.ok_button)
-        
+                if errors:
+                    retry = errors_dialog(self.plugin_action.gui,
+                                          _('Go back to fix errors?'),
+                                          '<p>'+'</p><p>'.join([ '(lineno: %s) %s'%e for e in errors ])+'</p>')
+                else:
+                    retry = False
+                    
+                if not retry:
+                    self.personalini = unicode(editini)
+            else:
+                # cancelled
+                retry = False
+                
 class ReadingListTab(QWidget):
 
     def __init__(self, parent_dialog, plugin_action):

calibre-plugin/dialogs.py

@@ -4,12 +4,9 @@ from __future__ import (unicode_literals, division,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2011, Jim Miller'
+__copyright__ = '2014, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
-import logging
-logger = logging.getLogger(__name__)
-
 import traceback, re
 from functools import partial
 
@@ -23,18 +20,20 @@ from datetime import datetime
 
 try:
     from PyQt5 import QtWidgets as QtGui
+    from PyQt5 import QtCore
     from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit,
                           QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
                           QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
-                          QGroupBox, QFrame)
+                          QGroupBox, QFrame, QTextBrowser, QSize, QAction)
 except ImportError as e:
     from PyQt4 import QtGui
+    from PyQt4 import QtCore
     from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout,
-                          QPushButton, QLabel, QCheckBox, QIcon, QLineEdit,
+                          QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit,
                           QComboBox, QProgressDialog, QTimer, QDialogButtonBox,
                           QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal,
-                          QGroupBox, QFrame)
+                          QGroupBox, QFrame, QTextBrowser, QSize, QAction)
 
 try:
     from calibre.gui2 import QVariant
@@ -68,6 +67,12 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL
 
+from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable \
+    import (get_valid_sections, get_valid_entries,
+            get_valid_keywords, get_valid_entry_keywords)
+
+from inihighlighter import IniHighlighter
+
 SKIP=_('Skip')
 ADDNEW=_('Add New Book')
 UPDATE=_('Update EPUB if New Chapters')
@@ -1111,14 +1116,15 @@ class RejectListDialog(SizePersistedDialog):
     def get_deletebooks(self):
         return self.deletebooks.isChecked()
 
-class EditTextDialog(QDialog):
+class EditTextDialog(SizePersistedDialog):
 
     def __init__(self, parent, text,
                  icon=None, title=None, label=None, tooltip=None,
-                 rejectreasons=[],reasonslabel=None
+                 rejectreasons=[],reasonslabel=None,
+                 save_size_name='ffdl:edit text dialog',
                  ):
-        QDialog.__init__(self, parent)
-        self.resize(600, 500)
+        SizePersistedDialog.__init__(self, parent, save_size_name)
+
         self.l = QVBoxLayout()
         self.setLayout(self.l)
         self.label = QLabel(label)
@@ -1160,6 +1166,9 @@ class EditTextDialog(QDialog):
         button_box.accepted.connect(self.accept)
         button_box.rejected.connect(self.reject)
         self.l.addWidget(button_box)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
 
     def get_plain_text(self):
         return unicode(self.textedit.toPlainText())
@@ -1167,3 +1176,204 @@ class EditTextDialog(QDialog):
     def get_reason_text(self):
         return unicode(self.reason_edit.currentText()).strip()
     
+class IniTextDialog(SizePersistedDialog):
+
+    def __init__(self, parent, text,
+                 icon=None, title=None, label=None, tooltip=None,
+                 use_find=False,
+                 read_only=False,
+                 save_size_name='ffdl:ini text dialog',
+                 ):
+        SizePersistedDialog.__init__(self, parent, save_size_name)
+        
+        self.keys=dict()
+        
+        self.l = QVBoxLayout()
+        self.setLayout(self.l)
+        self.label = QLabel(label)
+        if title:
+            self.setWindowTitle(title)
+        if icon:
+            self.setWindowIcon(icon)
+        self.l.addWidget(self.label)
+        
+        self.textedit = QTextEdit(self)
+
+        highlighter = IniHighlighter(self.textedit,
+                                     sections=get_valid_sections(),
+                                     keywords=get_valid_keywords(),
+                                     entries=get_valid_entries(),
+                                     entry_keywords=get_valid_entry_keywords(),
+                                     )
+        
+        self.textedit.setLineWrapMode(QTextEdit.NoWrap)
+        try:
+            self.textedit.setFont(QFont("Courier",
+                                   parent.font().pointSize()+1))
+        except Exception as e:
+            logger.error("Couldn't get font: %s"%e)
+
+        self.textedit.setReadOnly(read_only)
+
+        self.textedit.setText(text)
+        self.l.addWidget(self.textedit)
+
+        self.lastStart = 0
+
+        if use_find:
+
+            findtooltip=_('Search for string in edit box.')
+            
+            horz = QHBoxLayout()
+            label = QLabel(_('Find:'))
+
+            label.setToolTip(findtooltip)
+                
+            # Button to search the document for something
+            self.findButton = QtGui.QPushButton(_('Find'),self)
+            self.findButton.clicked.connect(self.find)
+            self.findButton.setToolTip(findtooltip)
+        
+            # The field into which to type the query
+            self.findField = QLineEdit(self)
+            self.findField.setToolTip(findtooltip)
+            self.findField.returnPressed.connect(self.findButton.setFocus)
+            
+            # Case Sensitivity option
+            self.caseSens = QtGui.QCheckBox(_('Case sensitive'),self)
+            self.caseSens.setToolTip(_("Search for case sensitive string; don't treat Harry, HARRY and harry all the same."))
+            
+            horz.addWidget(label)
+            horz.addWidget(self.findField)
+            horz.addWidget(self.findButton)
+            horz.addWidget(self.caseSens)
+            
+            self.l.addLayout(horz)
+
+            self.addCtrlKeyPress(QtCore.Qt.Key_F,self.findFocus)
+            self.addCtrlKeyPress(QtCore.Qt.Key_G,self.find)
+        
+        if tooltip:
+            self.label.setToolTip(tooltip)
+            self.textedit.setToolTip(tooltip)
+
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        button_box.accepted.connect(self.accept)
+        button_box.rejected.connect(self.reject)
+        self.l.addWidget(button_box)
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+
+    def addCtrlKeyPress(self,key,func):
+        # print("addKeyPress: key(0x%x)"%key)
+        # print("control: 0x%x"%QtCore.Qt.ControlModifier)
+        self.keys[key]=func
+
+    def keyPressEvent(self, event):
+        # print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers()))
+        if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys:
+            func = self.keys[event.key()]
+            return func()
+        else:
+            return SizePersistedDialog.keyPressEvent(self, event)
+    
+    def get_plain_text(self):
+        return unicode(self.textedit.toPlainText())
+
+    def findFocus(self):
+        # print("findFocus called")
+        self.findField.setFocus()
+        self.findField.selectAll()        
+        
+    def find(self):
+
+        #print("find self.lastStart:%s"%self.lastStart)
+        
+        # Grab the parent's text
+        text = self.textedit.toPlainText()
+
+        # And the text to find
+        query = self.findField.text()
+
+        if not self.caseSens.isChecked():
+            text = text.lower()
+            query = query.lower()
+
+        # Use normal string search to find the query from the
+        # last starting position
+        self.lastStart = text.find(query,self.lastStart + 1)
+        # If the find() method didn't return -1 (not found)
+        
+        if self.lastStart >= 0:
+            end = self.lastStart + len(query)
+            self.moveCursor(self.lastStart,end)
+        else:
+            # Make the next search start from the begining again
+            self.lastStart = 0
+            self.textedit.moveCursor(self.textedit.textCursor().Start)
+                    
+    def moveCursor(self,start,end):
+
+        # We retrieve the QTextCursor object from the parent's QTextEdit
+        cursor = self.textedit.textCursor()
+
+        # Then we set the position to the beginning of the last match
+        cursor.setPosition(start)
+
+        # Next we move the Cursor by over the match and pass the KeepAnchor parameter
+        # which will make the cursor select the the match's text
+        cursor.movePosition(cursor.Right,cursor.KeepAnchor,end - start)
+
+        # And finally we set this new cursor as the parent's
+        self.textedit.setTextCursor(cursor)
+
+def errors_dialog(parent,
+                  title,
+                  html):
+
+    d = ViewLog(title,html,parent)
+
+    return d.exec_() == d.Accepted
+        
+class ViewLog(SizePersistedDialog):
+
+    def __init__(self, title, html, parent=None,
+                 save_size_name='ffdl:view log dialog',):
+        SizePersistedDialog.__init__(self, parent,save_size_name)
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+
+        label = QLabel(_('Eventually I intend for errors to be clickable and take you to the error in the file.  For now, use copy and Find.'))
+        label.setWordWrap(True)
+        self.l.addWidget(label)
+        
+        self.tb = QTextBrowser(self)
+        self.tb.setFont(QFont("Courier",
+                              parent.font().pointSize()+1))
+        self.tb.setHtml(html)
+        l.addWidget(self.tb)
+
+        horz = QHBoxLayout()
+
+        editagain = QPushButton(_('Edit Ini Again'), self)
+        editagain.clicked.connect(self.accept)
+        horz.addWidget(editagain)
+
+        saveanyway = QPushButton(_('Save Ini Anyway'), self)
+        saveanyway.clicked.connect(self.reject)
+        horz.addWidget(saveanyway)
+        
+        l.addLayout(horz)
+        self.setModal(False)
+        self.setWindowTitle(title)
+        self.setWindowIcon(QIcon(I('debug.png')))
+        #self.show()
+        
+        # Cause our dialog size to be restored from prefs or created on first usage
+        self.resize_dialog()
+
+    def copy_to_clipboard(self):
+        txt = self.tb.toPlainText()
+        QApplication.clipboard().setText(txt)
+

calibre-plugin/ffdl_util.py

@@ -8,10 +8,11 @@ __copyright__ = '2013, Jim Miller'
 __docformat__ = 'restructuredtext en'
 
 from StringIO import StringIO
+from ConfigParser import ParsingError
 
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions
 from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration
-from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs)
+from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs
 
 def get_ffdl_personalini():
     if prefs['includeimages']:
@@ -40,4 +41,15 @@ def get_ffdl_config(url,fileform="epub",personalini=None):
 
 def get_ffdl_adapter(url,fileform="epub",personalini=None):
     return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url)
+
+def test_config(initext):
+
+    configini = get_ffdl_config("test1.com?sid=555",
+                                personalini=initext)
+    try:
+        errors = configini.test_config()
+    except ParsingError as pe:
+        errors = pe.errors
     
+    return errors
+

calibre-plugin/inihighlighter.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2014, Jim Miller'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+try:
+    from PyQt5.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush, QFont)
+except ImportError as e:
+    from PyQt4.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush, QFont)
+
+# r'add_to_+key
+
+
+    
+class IniHighlighter(QSyntaxHighlighter):
+    '''
+    QSyntaxHighlighter class for use with QTextEdit for highlighting
+    ini config files.
+    '''
+    
+    def __init__( self, parent, sections=[], keywords=[], entries=[], entry_keywords=[] ):
+        QSyntaxHighlighter.__init__( self, parent )
+        self.parent = parent
+        
+        self.highlightingRules = []
+
+        if entries:
+            # *known* entries
+            reentries = r'('+(r'|'.join(entries))+r')'
+            self.highlightingRules.append( HighlightingRule( r"\b"+reentries+r"\b", Qt.darkGreen ) )
+
+        # true/false -- just to be nice.
+        self.highlightingRules.append( HighlightingRule( r"\b(true|false)\b", Qt.darkGreen ) )
+        
+        # *all* keywords -- change known later.
+        self.errorRule = HighlightingRule( r"^[^:=\s][^:=]*[:=]", Qt.red )
+        self.highlightingRules.append( self.errorRule )
+
+        # *all* entry keywords -- change known later.
+        reentrykeywords = r'('+(r'|'.join([ e % r'[a-zA-Z0-9_]+' for e in entry_keywords ]))+r')'
+        self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+reentrykeywords+r"\s*[:=]", Qt.darkMagenta ) )
+
+        if entries: # separate from known entries so entry named keyword won't be masked.
+            # *known* entry keywords
+            reentrykeywords = r'('+(r'|'.join([ e % reentries for e in entry_keywords ]))+r')'
+            self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+reentrykeywords+r"\s*[:=]", Qt.blue ) )
+
+        # *known* keywords
+        rekeywords = r'('+(r'|'.join(keywords))+r')'
+        self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+rekeywords+r"\s*[:=]", Qt.blue ) )
+
+        # *all* sections -- change known later.
+        self.highlightingRules.append( HighlightingRule( r"^\[[^\]]+\].*?$", Qt.red, QFont.Bold, blocknum=1 ) )
+
+        if sections:
+            # *known* sections
+            resections = r'('+(r'|'.join(sections))+r')'
+            resections = resections.replace('.','\.') #escape dots.
+            self.highlightingRules.append( HighlightingRule( r"^\["+resections+r"\]\s*$", Qt.darkBlue, QFont.Bold, blocknum=2 ) )
+
+        # test story sections
+        self.teststoryRule = HighlightingRule( r"^\[teststory:([0-9]+|defaults)\]", Qt.darkCyan, blocknum=3 )
+        self.highlightingRules.append( self.teststoryRule )
+
+        # NOT comments -- but can be custom columns, so don't flag.
+        #self.highlightingRules.append( HighlightingRule( r"(?<!^)#[^\n]*" , Qt.red ) )
+        
+        # comments -- comments must start from column 0.
+        self.commentRule = HighlightingRule( r"^#[^\n]*" , Qt.darkYellow )
+        self.highlightingRules.append( self.commentRule )
+
+    def highlightBlock( self, text ):
+
+        is_comment = False
+        blocknum = self.previousBlockState()
+        for rule in self.highlightingRules:
+            for match in rule.pattern.finditer(text):
+                self.setFormat( match.start(), match.end()-match.start(), rule.highlight )
+                if rule == self.commentRule:
+                    is_comment = True
+                if rule.blocknum > 0:
+                    blocknum = rule.blocknum
+
+        if not is_comment:
+            # unknown section, error all:
+            if blocknum == 1 and blocknum == self.previousBlockState():
+                self.setFormat( 0, len(text), self.errorRule.highlight )
+                
+            # teststory section rules:
+            if blocknum == 3:
+                self.setFormat( 0, len(text), self.teststoryRule.highlight )
+            
+        self.setCurrentBlockState( blocknum )
+
+class HighlightingRule():
+    def __init__( self, pattern, color,
+                  weight=QFont.Normal,
+                  style=Qt.SolidPattern,
+                  blocknum=0):
+        if isinstance(pattern,basestring):
+            self.pattern = re.compile(pattern)
+        else:
+            self.pattern=pattern
+        charfmt = QTextCharFormat()
+        brush = QBrush(color, style)
+        charfmt.setForeground(brush)
+        charfmt.setFontWeight(weight)
+        self.highlight = charfmt
+        self.blocknum=blocknum
+    

downloader.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2014 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.

fanficdownloader/configurable.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright 2011 Fanficdownloader team
+# Copyright 2014 Fanficdownloader team
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,8 @@
 #
 
 import ConfigParser, re
+import exceptions
+from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError
 
 # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc)
 # inherit from Configurable.  The config file(s) uses ini format:
@@ -32,10 +34,194 @@ import ConfigParser, re
 # [overrides]
 # titlepage_entries: category
 
+import adapters
+
+def get_valid_sections():
+    sites = adapters.getConfigSections()
+    sitesections = ['defaults','overrides']
+    for section in sites:
+        sitesections.append(section)
+        if section.startswith('www.'):
+            # add w/o www if has www
+            sitesections.append(section[4:])
+        else:
+            # add w/ www if doesn't www
+            sitesections.append('www.%s'%section)
+            
+    allowedsections = []
+    forms=['html','txt','epub','mobi']
+    allowedsections.extend(forms)
+
+    for section in sitesections:
+        allowedsections.append(section)
+        for f in forms:
+            allowedsections.append('%s:%s'%(section,f))
+    return allowedsections
+    
+def get_valid_list_entries():
+    return list(['category',
+                 'genre',
+                 'characters',
+                 'ships',
+                 'warnings',
+                 'extratags',
+                 'author',
+                 'authorId',
+                 'authorUrl',
+                 'lastupdate',
+                 ])
+
+def get_valid_scalar_entries():
+    return list(['series',
+                 'seriesUrl',
+                 'language',
+                 'status',
+                 'datePublished',
+                 'dateUpdated',
+                 'dateCreated',
+                 'rating',
+                 'numChapters',
+                 'numWords',
+                 'site',
+                 'storyId',
+                 'title',
+                 'storyUrl',
+                 'description',
+                 'formatname',
+                 'formatext',
+                 'siteabbrev',
+                 'version',
+                 # internal stuff.
+                 'authorHTML',
+                 'seriesHTML',
+                 'langcode',
+                 'output_css',
+                 ])
+
+def get_valid_entries():
+    return get_valid_list_entries() + get_valid_scalar_entries()
+
+# *known* keywords -- or rather regexps for them.
+def get_valid_keywords():
+    return list(['(in|ex)clude_metadata_(pre|post)',
+                 'add_chapter_numbers',
+                 'add_genre_when_multi_category',
+                 'allow_unsafe_filename',
+                 'always_overwrite',
+                 'anthology_tags',
+                 'anthology_title_pattern',
+                 'background_color',
+                 'bulk_load',
+                 'chapter_end',
+                 'chapter_start',
+                 'chapter_title_add_pattern',
+                 'chapter_title_strip_pattern',
+                 'check_next_chapter',
+                 'collect_series',
+                 'connect_timeout',
+                 'convert_images_to',
+                 'cover_content',
+                 'cover_exclusion_regexp',
+                 'custom_columns_settings',
+                 'dateCreated_format',
+                 'datePublished_format',
+                 'dateUpdated_format',
+                 'default_cover_image',
+                 'do_update_hook',
+                 'exclude_notes',
+                 'extra_logpage_entries',
+                 'extra_subject_tags',
+                 'extra_titlepage_entries',
+                 'extra_valid_entries',
+                 'extratags',
+                 'extracategories',
+                 'extragenres',
+                 'extracharacters',
+                 'extraships',
+                 'extrawarnings',
+                 'fail_on_password',
+                 'file_end',
+                 'file_start',
+                 'fileformat',
+                 'find_chapters',
+                 'fix_fimf_blockquotes',
+                 'force_login',
+                 'generate_cover_settings',
+                 'grayscale_images',
+                 'image_max_size',
+                 'include_images',
+                 'include_logpage',
+                 'include_subject_tags',
+                 'include_titlepage',
+                 'include_tocpage',
+                 'is_adult',
+                 'join_string_authorHTML',
+                 'keep_style_attr',
+                 'keep_summary_html',
+                 'logpage_end',
+                 'logpage_entries',
+                 'logpage_entry',
+                 'logpage_start',
+                 'logpage_update_end',
+                 'logpage_update_start',
+                 'make_directories',
+                 'make_firstimage_cover',
+                 'make_linkhtml_entries',
+                 'max_fg_sleep',
+                 'max_fg_sleep_at_downloads',
+                 'min_fg_sleep',
+                 'never_make_cover',
+                 'no_image_processing',
+                 'non_breaking_spaces',
+                 'nook_img_fix',
+                 'output_css',
+                 'output_filename',
+                 'output_filename_safepattern',
+                 'password',
+                 'post_process_cmd',
+                 'remove_transparency',
+                 'replace_br_with_p',
+                 'replace_hr',
+                 'replace_metadata',
+                 'slow_down_sleep_time',
+                 'sort_ships',
+                 'strip_chapter_numbers',
+                 'strip_chapter_numeral',
+                 'strip_text_links',
+                 'titlepage_end',
+                 'titlepage_entries',
+                 'titlepage_entry',
+                 'titlepage_no_title_entry',
+                 'titlepage_start',
+                 'titlepage_use_table',
+                 'titlepage_wide_entry',
+                 'tocpage_end',
+                 'tocpage_entry',
+                 'tocpage_start',
+                 'tweak_fg_sleep',
+                 'universe_as_series',
+                 'user_agent',
+                 'username',
+                 'website_encodings',
+                 'wide_titlepage_entries',
+                 'windows_eol',
+                 'wrap_width',
+                 'zip_filename',
+                 'zip_output',
+                 ])
+
+# *known* entry keywords -- or rather regexps for them.
+def get_valid_entry_keywords():
+    return list(['%s_label',
+                 '(default_value|include_in|join_string|keep_in_order)_%s',])
+
 class Configuration(ConfigParser.SafeConfigParser):
 
     def __init__(self, site, fileform):
         ConfigParser.SafeConfigParser.__init__(self)
+
+        self.linenos=dict() # key by section or section,key -> lineno
+        
         self.sectionslist = ['defaults']
 
         if site.startswith("www."):
@@ -53,45 +239,9 @@ class Configuration(ConfigParser.SafeConfigParser):
             self.addConfigSection(sitewithout+":"+fileform)
         self.addConfigSection("overrides")
         
-        self.listTypeEntries = [
-            'category',
-            'genre',
-            'characters',
-            'ships',
-            'warnings',
-            'extratags',
-            'author',
-            'authorId',
-            'authorUrl',
-            'lastupdate',
-            ]
+        self.listTypeEntries = get_valid_list_entries()
         
-        self.validEntries = self.listTypeEntries + [
-            'series',
-            'seriesUrl',
-            'language',
-            'status',
-            'datePublished',
-            'dateUpdated',
-            'dateCreated',
-            'rating',
-            'numChapters',
-            'numWords',
-            'site',
-            'storyId',
-            'title',
-            'storyUrl',
-            'description',
-            'formatname',
-            'formatext',
-            'siteabbrev',
-            'version',
-            # internal stuff.
-            'authorHTML',
-            'seriesHTML',
-            'langcode',
-            'output_css',
-            ]
+        self.validEntries = get_valid_entries()
 
     def addConfigSection(self,section):
         self.sectionslist.insert(0,section)
@@ -129,7 +279,7 @@ class Configuration(ConfigParser.SafeConfigParser):
     def getConfig(self, key, default=""):
         return self.get_config(self.sectionslist,key,default)
     
-    def get_config(self, sections, key, default=""):        
+    def get_config(self, sections, key, default=""):
         val = default
         for section in sections:
             try:
@@ -162,6 +312,152 @@ class Configuration(ConfigParser.SafeConfigParser):
     def getConfigList(self, key):
         return self.get_config_list(self.sectionslist, key)
 
+
+    def get_lineno(self,section,key=None):
+        if key:
+            return self.linenos.get(section+','+key,None)
+        else:
+            return self.linenos.get(section,None)
+    
+    ## Copied from Python library so as to make it save linenos too.
+    #
+    # Regular expressions for parsing section headers and options.
+    #
+    def _read(self, fp, fpname):
+        """Parse a sectioned setup file.
+
+        The sections in setup file contains a title line at the top,
+        indicated by a name in square brackets (`[]'), plus key/value
+        options lines, indicated by `name: value' format lines.
+        Continuations are represented by an embedded newline then
+        leading whitespace.  Blank lines, lines beginning with a '#',
+        and just about everything else are ignored.
+        """
+        cursect = None                            # None, or a dictionary
+        optname = None
+        lineno = 0
+        e = None                                  # None, or an exception
+        while True:
+            line = fp.readline()
+            if not line:
+                break
+            lineno = lineno + 1
+            # comment or blank line?
+            if line.strip() == '' or line[0] in '#;':
+                continue
+            if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
+                # no leading whitespace
+                continue
+            # continuation line?
+            if line[0].isspace() and cursect is not None and optname:
+                value = line.strip()
+                if value:
+                    cursect[optname] = "%s\n%s" % (cursect[optname], value)
+            # a section header or option header?
+            else:
+                # is it a section header?
+                mo = self.SECTCRE.match(line)
+                if mo:
+                    sectname = mo.group('header')
+                    if sectname in self._sections:
+                        cursect = self._sections[sectname]
+                    elif sectname == DEFAULTSECT:
+                        cursect = self._defaults
+                    else:
+                        cursect = self._dict()
+                        cursect['__name__'] = sectname
+                        self._sections[sectname] = cursect
+                        self.linenos[sectname]=lineno
+                    # So sections can't start with a continuation line
+                    optname = None
+                # no section header in the file?
+                elif cursect is None:
+                    if not e:
+                        e = ParsingError(fpname)
+                    e.append(lineno, u'(Line outside section) '+line)
+                    #raise MissingSectionHeaderError(fpname, lineno, line)
+                # an option line?
+                else:
+                    mo = self._optcre.match(line)
+                    if mo:
+                        optname, vi, optval = mo.group('option', 'vi', 'value')
+                        # This check is fine because the OPTCRE cannot
+                        # match if it would set optval to None
+                        if optval is not None:
+                            if vi in ('=', ':') and ';' in optval:
+                                # ';' is a comment delimiter only if it follows
+                                # a spacing character
+                                pos = optval.find(';')
+                                if pos != -1 and optval[pos-1].isspace():
+                                    optval = optval[:pos]
+                            optval = optval.strip()
+                        # allow empty values
+                        if optval == '""':
+                            optval = ''
+                        optname = self.optionxform(optname.rstrip())
+                        cursect[optname] = optval
+                        self.linenos[cursect['__name__']+','+optname]=lineno                        
+                    else:
+                        # a non-fatal parsing error occurred.  set up the
+                        # exception but keep going. the exception will be
+                        # raised at the end of the file and will contain a
+                        # list of all bogus lines
+                        if not e:
+                            e = ParsingError(fpname)
+                        e.append(lineno, line)
+        # if any parsing errors occurred, raise an exception
+        if e:
+            raise e
+
+    def test_config(self):
+        errors=[]
+
+        teststory_re = re.compile(r'^teststory:(defaults|[0-9]+)$')
+        allowedsections = get_valid_sections()
+
+        clude_metadata_re = re.compile(r'(add_to_)?(in|ex)clude_metadata_(pre|post)')
+
+        replace_metadata_re = re.compile(r'(add_to_)?replace_metadata')
+        from story import set_in_ex_clude, make_replacements
+
+        custom_columns_settings_re = re.compile(r'(add_to_)?custom_columns_settings')
+        
+        for section in self.sections():
+            if section not in allowedsections and not teststory_re.match(section):
+                errors.append((self.get_lineno(section),"Bad Section Name: %s"%section))
+            else:
+                ## check each keyword in section.  Due to precedence
+                ## order of sections, it's possible for bad lines to
+                ## never be used.
+                for keyword,value in self.items(section):
+                    try:
+                        
+                        ## check regex bearing keywords first.  Each
+                        ## will raise exceptions if flawed.
+                        if clude_metadata_re.match(keyword):
+                            set_in_ex_clude(value)    
+
+                        if replace_metadata_re.match(keyword):
+                            make_replacements(value)
+
+                        # if custom_columns_settings_re.match(keyword):
+                        #custom_columns_settings:
+                        # cliches=>#acolumn
+                        # themes=>#bcolumn,a
+                        # timeline=>#ccolumn,n
+                        # "FanFiction"=>#collection
+
+                            
+                        ## skipping output_filename_safepattern
+                        ## regex--not used with plugin and this isn't
+                        ## used with CLI/web yet.
+
+                    except Exception as e:
+                        errors.append((self.get_lineno(section,keyword),"Error:%s in (%s:%s)"%(e,keyword,value)))
+                
+        
+        return errors
+
 # extended by adapter, writer and story for ease of calling configuration.
 class Configurable(object):
 

fanficdownloader/story.py

@@ -288,6 +288,59 @@ class InExMatch:
         else:
             s='='
         return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match)
+    
+## metakey[,metakey]=~pattern
+## metakey[,metakey]==string
+## *for* part lines.  Effect only when trailing conditional key=~regexp matches
+## metakey[,metakey]=~pattern[&&metakey=~regexp]
+## metakey[,metakey]==string[&&metakey=~regexp]
+## metakey[,metakey]=~pattern[&&metakey==string]
+## metakey[,metakey]==string[&&metakey==string]
+def set_in_ex_clude(setting):
+    dest = []
+    # print("set_in_ex_clude:"+setting)
+    for line in setting.splitlines():
+        if line:
+            (match,condmatch)=(None,None)
+            if "&&" in line:
+                (line,conditional) = line.split("&&")
+                condmatch = InExMatch(conditional)
+            match = InExMatch(line)
+            dest.append([match,condmatch])
+    return dest
+              
+## Two or three part lines.  Two part effect everything.
+## Three part effect only those key(s) lists.
+## pattern=>replacement
+## metakey,metakey=>pattern=>replacement
+## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
+## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
+def make_replacements(replace):
+    retval=[]
+    for line in replace.splitlines():
+        # print("replacement line:%s"%line)
+        (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
+        if "&&" in line:
+            (line,conditional) = line.split("&&")
+            (condkey,condregexp) = conditional.split("=>")
+        if "=>" in line:
+            parts = line.split("=>")
+            if len(parts) > 2:
+                metakeys = map( lambda x: x.strip(), parts[0].split(",") )
+                (regexp,replacement)=parts[1:]
+            else:
+                (regexp,replacement)=parts
+
+        if regexp:
+            regexp = re_compile(regexp,line)
+            if condregexp:
+                condregexp = re_compile(condregexp,line)
+            # A way to explicitly include spaces in the
+            # replacement string.  The .ini parser eats any
+            # trailing spaces.
+            replacement=replacement.replace(SPACE_REPLACE,' ')
+            retval.append([metakeys,regexp,replacement,condkey,condregexp])
+    return retval
 
 class Story(Configurable):
 
@@ -298,7 +351,6 @@ class Story(Configurable):
             self.metadata = {'version':os.environ['CURRENT_VERSION_ID']}
         except:
             self.metadata = {'version':'4.4'}
-        self.replacements = []
         self.in_ex_cludes = {}
         self.chapters = [] # chapters will be tuples of (title,html)
         self.imgurls = []
@@ -318,7 +370,7 @@ class Story(Configurable):
             for val in self.getConfigList(config):
                 self.addToList(metadata,val)
 
-        self.setReplace(self.getConfig('replace_metadata'))
+        self.replacements = make_replacements(self.getConfig('replace_metadata'))
 
         in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre',
                             'include_metadata_post','exclude_metadata_post']
@@ -327,7 +379,7 @@ class Story(Configurable):
             # print("%s %s"%(ie,ies))
             if ies:
                 iel = []
-                self.in_ex_cludes[ie] = self.set_in_ex_clude(ies)
+                self.in_ex_cludes[ie] = set_in_ex_clude(ies)
 
     def join_list(self, key, vallist):
         return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist))
@@ -357,26 +409,6 @@ class Story(Configurable):
             self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d"))
 
         
-    ## metakey[,metakey]=~pattern
-    ## metakey[,metakey]==string
-    ## *for* part lines.  Effect only when trailing conditional key=~regexp matches
-    ## metakey[,metakey]=~pattern[&&metakey=~regexp]
-    ## metakey[,metakey]==string[&&metakey=~regexp]
-    ## metakey[,metakey]=~pattern[&&metakey==string]
-    ## metakey[,metakey]==string[&&metakey==string]
-    def set_in_ex_clude(self,setting):
-        dest = []
-        # print("set_in_ex_clude:"+setting)
-        for line in setting.splitlines():
-            if line:
-                (match,condmatch)=(None,None)
-                if "&&" in line:
-                    (line,conditional) = line.split("&&")
-                    condmatch = InExMatch(conditional)
-                match = InExMatch(line)
-                dest.append([match,condmatch])
-        return dest
-              
     def do_in_ex_clude(self,which,value,key):
         if value and which in self.in_ex_cludes:
             include = 'include' in which
@@ -407,37 +439,6 @@ class Story(Configurable):
         return value
         
 
-    ## Two or three part lines.  Two part effect everything.
-    ## Three part effect only those key(s) lists.
-    ## pattern=>replacement
-    ## metakey,metakey=>pattern=>replacement
-    ## *Five* part lines.  Effect only when trailing conditional key=>regexp matches
-    ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp]
-    def setReplace(self,replace):
-        for line in replace.splitlines():
-            # print("replacement line:%s"%line)
-            (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None)
-            if "&&" in line:
-                (line,conditional) = line.split("&&")
-                (condkey,condregexp) = conditional.split("=>")
-            if "=>" in line:
-                parts = line.split("=>")
-                if len(parts) > 2:
-                    metakeys = map( lambda x: x.strip(), parts[0].split(",") )
-                    (regexp,replacement)=parts[1:]
-                else:
-                    (regexp,replacement)=parts
-
-            if regexp:
-                regexp = re_compile(regexp,line)
-                if condregexp:
-                    condregexp = re_compile(condregexp,line)
-                # A way to explicitly include spaces in the
-                # replacement string.  The .ini parser eats any
-                # trailing spaces.
-                replacement=replacement.replace(SPACE_REPLACE,' ')
-                self.replacements.append([metakeys,regexp,replacement,condkey,condregexp])
-
     def doReplacements(self,value,key,return_list=False,seen_list=[]):
         value = self.do_in_ex_clude('include_metadata_pre',value,key)
         value = self.do_in_ex_clude('exclude_metadata_pre',value,key)