'+'
'.join([ '(lineno: %s) %s'%e for e in errors ])+'
') + else: + retry = False + + if not retry: + self.personalini = unicode(editini) + else: + # cancelled + retry = False + class ReadingListTab(QWidget): def __init__(self, parent_dialog, plugin_action): diff --git a/calibre-plugin/dialogs.py b/calibre-plugin/dialogs.py index 218a780e..a066ac2f 100644 --- a/calibre-plugin/dialogs.py +++ b/calibre-plugin/dialogs.py @@ -4,12 +4,9 @@ from __future__ import (unicode_literals, division, print_function) __license__ = 'GPL v3' -__copyright__ = '2011, Jim Miller' +__copyright__ = '2014, Jim Miller' __docformat__ = 'restructuredtext en' -import logging -logger = logging.getLogger(__name__) - import traceback, re from functools import partial @@ -23,18 +20,20 @@ from datetime import datetime try: from PyQt5 import QtWidgets as QtGui + from PyQt5 import QtCore from PyQt5.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout, - QPushButton, QLabel, QCheckBox, QIcon, QLineEdit, + QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit, QComboBox, QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal, - QGroupBox, QFrame) + QGroupBox, QFrame, QTextBrowser, QSize, QAction) except ImportError as e: from PyQt4 import QtGui + from PyQt4 import QtCore from PyQt4.Qt import (QDialog, QTableWidget, QVBoxLayout, QHBoxLayout, QGridLayout, - QPushButton, QLabel, QCheckBox, QIcon, QLineEdit, + QPushButton, QFont, QLabel, QCheckBox, QIcon, QLineEdit, QComboBox, QProgressDialog, QTimer, QDialogButtonBox, QPixmap, Qt, QAbstractItemView, QTextEdit, pyqtSignal, - QGroupBox, QFrame) + QGroupBox, QFrame, QTextBrowser, QSize, QAction) try: from calibre.gui2 import QVariant @@ -68,6 +67,12 @@ from calibre_plugins.fanfictiondownloader_plugin.common_utils \ from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.geturls import get_urls_from_html, get_urls_from_text from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.adapters import getNormalStoryURL +from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable \ + import (get_valid_sections, get_valid_entries, + get_valid_keywords, get_valid_entry_keywords) + +from inihighlighter import IniHighlighter + SKIP=_('Skip') ADDNEW=_('Add New Book') UPDATE=_('Update EPUB if New Chapters') @@ -1111,14 +1116,15 @@ class RejectListDialog(SizePersistedDialog): def get_deletebooks(self): return self.deletebooks.isChecked() -class EditTextDialog(QDialog): +class EditTextDialog(SizePersistedDialog): def __init__(self, parent, text, icon=None, title=None, label=None, tooltip=None, - rejectreasons=[],reasonslabel=None + rejectreasons=[],reasonslabel=None, + save_size_name='ffdl:edit text dialog', ): - QDialog.__init__(self, parent) - self.resize(600, 500) + SizePersistedDialog.__init__(self, parent, save_size_name) + self.l = QVBoxLayout() self.setLayout(self.l) self.label = QLabel(label) @@ -1160,6 +1166,9 @@ class EditTextDialog(QDialog): button_box.accepted.connect(self.accept) button_box.rejected.connect(self.reject) self.l.addWidget(button_box) + + # Cause our dialog size to be restored from prefs or created on first usage + self.resize_dialog() def get_plain_text(self): return unicode(self.textedit.toPlainText()) @@ -1167,3 +1176,204 @@ class EditTextDialog(QDialog): def get_reason_text(self): return unicode(self.reason_edit.currentText()).strip() +class IniTextDialog(SizePersistedDialog): + + def __init__(self, parent, text, + icon=None, title=None, label=None, tooltip=None, + use_find=False, + read_only=False, + save_size_name='ffdl:ini text dialog', + ): + SizePersistedDialog.__init__(self, parent, save_size_name) + + self.keys=dict() + + self.l = QVBoxLayout() + self.setLayout(self.l) + self.label = QLabel(label) + if title: + self.setWindowTitle(title) + if icon: + self.setWindowIcon(icon) + self.l.addWidget(self.label) + + self.textedit = QTextEdit(self) + + highlighter = IniHighlighter(self.textedit, + sections=get_valid_sections(), + keywords=get_valid_keywords(), + entries=get_valid_entries(), + entry_keywords=get_valid_entry_keywords(), + ) + + self.textedit.setLineWrapMode(QTextEdit.NoWrap) + try: + self.textedit.setFont(QFont("Courier", + parent.font().pointSize()+1)) + except Exception as e: + logger.error("Couldn't get font: %s"%e) + + self.textedit.setReadOnly(read_only) + + self.textedit.setText(text) + self.l.addWidget(self.textedit) + + self.lastStart = 0 + + if use_find: + + findtooltip=_('Search for string in edit box.') + + horz = QHBoxLayout() + label = QLabel(_('Find:')) + + label.setToolTip(findtooltip) + + # Button to search the document for something + self.findButton = QtGui.QPushButton(_('Find'),self) + self.findButton.clicked.connect(self.find) + self.findButton.setToolTip(findtooltip) + + # The field into which to type the query + self.findField = QLineEdit(self) + self.findField.setToolTip(findtooltip) + self.findField.returnPressed.connect(self.findButton.setFocus) + + # Case Sensitivity option + self.caseSens = QtGui.QCheckBox(_('Case sensitive'),self) + self.caseSens.setToolTip(_("Search for case sensitive string; don't treat Harry, HARRY and harry all the same.")) + + horz.addWidget(label) + horz.addWidget(self.findField) + horz.addWidget(self.findButton) + horz.addWidget(self.caseSens) + + self.l.addLayout(horz) + + self.addCtrlKeyPress(QtCore.Qt.Key_F,self.findFocus) + self.addCtrlKeyPress(QtCore.Qt.Key_G,self.find) + + if tooltip: + self.label.setToolTip(tooltip) + self.textedit.setToolTip(tooltip) + + button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + button_box.accepted.connect(self.accept) + button_box.rejected.connect(self.reject) + self.l.addWidget(button_box) + + # Cause our dialog size to be restored from prefs or created on first usage + self.resize_dialog() + + def addCtrlKeyPress(self,key,func): + # print("addKeyPress: key(0x%x)"%key) + # print("control: 0x%x"%QtCore.Qt.ControlModifier) + self.keys[key]=func + + def keyPressEvent(self, event): + # print("event: key(0x%x) modifiers(0x%x)"%(event.key(),event.modifiers())) + if (event.modifiers() & QtCore.Qt.ControlModifier) and event.key() in self.keys: + func = self.keys[event.key()] + return func() + else: + return SizePersistedDialog.keyPressEvent(self, event) + + def get_plain_text(self): + return unicode(self.textedit.toPlainText()) + + def findFocus(self): + # print("findFocus called") + self.findField.setFocus() + self.findField.selectAll() + + def find(self): + + #print("find self.lastStart:%s"%self.lastStart) + + # Grab the parent's text + text = self.textedit.toPlainText() + + # And the text to find + query = self.findField.text() + + if not self.caseSens.isChecked(): + text = text.lower() + query = query.lower() + + # Use normal string search to find the query from the + # last starting position + self.lastStart = text.find(query,self.lastStart + 1) + # If the find() method didn't return -1 (not found) + + if self.lastStart >= 0: + end = self.lastStart + len(query) + self.moveCursor(self.lastStart,end) + else: + # Make the next search start from the begining again + self.lastStart = 0 + self.textedit.moveCursor(self.textedit.textCursor().Start) + + def moveCursor(self,start,end): + + # We retrieve the QTextCursor object from the parent's QTextEdit + cursor = self.textedit.textCursor() + + # Then we set the position to the beginning of the last match + cursor.setPosition(start) + + # Next we move the Cursor by over the match and pass the KeepAnchor parameter + # which will make the cursor select the the match's text + cursor.movePosition(cursor.Right,cursor.KeepAnchor,end - start) + + # And finally we set this new cursor as the parent's + self.textedit.setTextCursor(cursor) + +def errors_dialog(parent, + title, + html): + + d = ViewLog(title,html,parent) + + return d.exec_() == d.Accepted + +class ViewLog(SizePersistedDialog): + + def __init__(self, title, html, parent=None, + save_size_name='ffdl:view log dialog',): + SizePersistedDialog.__init__(self, parent,save_size_name) + self.l = l = QVBoxLayout() + self.setLayout(l) + + label = QLabel(_('Eventually I intend for errors to be clickable and take you to the error in the file. For now, use copy and Find.')) + label.setWordWrap(True) + self.l.addWidget(label) + + self.tb = QTextBrowser(self) + self.tb.setFont(QFont("Courier", + parent.font().pointSize()+1)) + self.tb.setHtml(html) + l.addWidget(self.tb) + + horz = QHBoxLayout() + + editagain = QPushButton(_('Edit Ini Again'), self) + editagain.clicked.connect(self.accept) + horz.addWidget(editagain) + + saveanyway = QPushButton(_('Save Ini Anyway'), self) + saveanyway.clicked.connect(self.reject) + horz.addWidget(saveanyway) + + l.addLayout(horz) + self.setModal(False) + self.setWindowTitle(title) + self.setWindowIcon(QIcon(I('debug.png'))) + #self.show() + + # Cause our dialog size to be restored from prefs or created on first usage + self.resize_dialog() + + def copy_to_clipboard(self): + txt = self.tb.toPlainText() + QApplication.clipboard().setText(txt) + diff --git a/calibre-plugin/ffdl_util.py b/calibre-plugin/ffdl_util.py index c169d80c..6f28eb30 100644 --- a/calibre-plugin/ffdl_util.py +++ b/calibre-plugin/ffdl_util.py @@ -8,10 +8,11 @@ __copyright__ = '2013, Jim Miller' __docformat__ = 'restructuredtext en' from StringIO import StringIO +from ConfigParser import ParsingError from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader import adapters, exceptions from calibre_plugins.fanfictiondownloader_plugin.fanficdownloader.configurable import Configuration -from calibre_plugins.fanfictiondownloader_plugin.prefs import (prefs) +from calibre_plugins.fanfictiondownloader_plugin.prefs import prefs def get_ffdl_personalini(): if prefs['includeimages']: @@ -40,4 +41,15 @@ def get_ffdl_config(url,fileform="epub",personalini=None): def get_ffdl_adapter(url,fileform="epub",personalini=None): return adapters.getAdapter(get_ffdl_config(url,fileform,personalini),url) + +def test_config(initext): + + configini = get_ffdl_config("test1.com?sid=555", + personalini=initext) + try: + errors = configini.test_config() + except ParsingError as pe: + errors = pe.errors + return errors + diff --git a/calibre-plugin/inihighlighter.py b/calibre-plugin/inihighlighter.py new file mode 100644 index 00000000..8be745c2 --- /dev/null +++ b/calibre-plugin/inihighlighter.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2014, Jim Miller' +__docformat__ = 'restructuredtext en' + +import re + +try: + from PyQt5.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush, QFont) +except ImportError as e: + from PyQt4.Qt import (Qt, QSyntaxHighlighter, QTextCharFormat, QBrush, QFont) + +# r'add_to_+key + + + +class IniHighlighter(QSyntaxHighlighter): + ''' + QSyntaxHighlighter class for use with QTextEdit for highlighting + ini config files. + ''' + + def __init__( self, parent, sections=[], keywords=[], entries=[], entry_keywords=[] ): + QSyntaxHighlighter.__init__( self, parent ) + self.parent = parent + + self.highlightingRules = [] + + if entries: + # *known* entries + reentries = r'('+(r'|'.join(entries))+r')' + self.highlightingRules.append( HighlightingRule( r"\b"+reentries+r"\b", Qt.darkGreen ) ) + + # true/false -- just to be nice. + self.highlightingRules.append( HighlightingRule( r"\b(true|false)\b", Qt.darkGreen ) ) + + # *all* keywords -- change known later. + self.errorRule = HighlightingRule( r"^[^:=\s][^:=]*[:=]", Qt.red ) + self.highlightingRules.append( self.errorRule ) + + # *all* entry keywords -- change known later. + reentrykeywords = r'('+(r'|'.join([ e % r'[a-zA-Z0-9_]+' for e in entry_keywords ]))+r')' + self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+reentrykeywords+r"\s*[:=]", Qt.darkMagenta ) ) + + if entries: # separate from known entries so entry named keyword won't be masked. + # *known* entry keywords + reentrykeywords = r'('+(r'|'.join([ e % reentries for e in entry_keywords ]))+r')' + self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+reentrykeywords+r"\s*[:=]", Qt.blue ) ) + + # *known* keywords + rekeywords = r'('+(r'|'.join(keywords))+r')' + self.highlightingRules.append( HighlightingRule( r"^(add_to_)?"+rekeywords+r"\s*[:=]", Qt.blue ) ) + + # *all* sections -- change known later. + self.highlightingRules.append( HighlightingRule( r"^\[[^\]]+\].*?$", Qt.red, QFont.Bold, blocknum=1 ) ) + + if sections: + # *known* sections + resections = r'('+(r'|'.join(sections))+r')' + resections = resections.replace('.','\.') #escape dots. + self.highlightingRules.append( HighlightingRule( r"^\["+resections+r"\]\s*$", Qt.darkBlue, QFont.Bold, blocknum=2 ) ) + + # test story sections + self.teststoryRule = HighlightingRule( r"^\[teststory:([0-9]+|defaults)\]", Qt.darkCyan, blocknum=3 ) + self.highlightingRules.append( self.teststoryRule ) + + # NOT comments -- but can be custom columns, so don't flag. + #self.highlightingRules.append( HighlightingRule( r"(? 0: + blocknum = rule.blocknum + + if not is_comment: + # unknown section, error all: + if blocknum == 1 and blocknum == self.previousBlockState(): + self.setFormat( 0, len(text), self.errorRule.highlight ) + + # teststory section rules: + if blocknum == 3: + self.setFormat( 0, len(text), self.teststoryRule.highlight ) + + self.setCurrentBlockState( blocknum ) + +class HighlightingRule(): + def __init__( self, pattern, color, + weight=QFont.Normal, + style=Qt.SolidPattern, + blocknum=0): + if isinstance(pattern,basestring): + self.pattern = re.compile(pattern) + else: + self.pattern=pattern + charfmt = QTextCharFormat() + brush = QBrush(color, style) + charfmt.setForeground(brush) + charfmt.setFontWeight(weight) + self.highlight = charfmt + self.blocknum=blocknum + diff --git a/downloader.py b/downloader.py index 62a92dea..f0a994a9 100644 --- a/downloader.py +++ b/downloader.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2011 Fanficdownloader team +# Copyright 2014 Fanficdownloader team # # Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. diff --git a/fanficdownloader/configurable.py b/fanficdownloader/configurable.py index 9b6d35f4..1aa48325 100644 --- a/fanficdownloader/configurable.py +++ b/fanficdownloader/configurable.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Copyright 2011 Fanficdownloader team +# Copyright 2014 Fanficdownloader team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ # import ConfigParser, re +import exceptions +from ConfigParser import DEFAULTSECT, MissingSectionHeaderError, ParsingError # All of the writers(epub,html,txt) and adapters(ffnet,twlt,etc) # inherit from Configurable. The config file(s) uses ini format: @@ -32,10 +34,194 @@ import ConfigParser, re # [overrides] # titlepage_entries: category +import adapters + +def get_valid_sections(): + sites = adapters.getConfigSections() + sitesections = ['defaults','overrides'] + for section in sites: + sitesections.append(section) + if section.startswith('www.'): + # add w/o www if has www + sitesections.append(section[4:]) + else: + # add w/ www if doesn't www + sitesections.append('www.%s'%section) + + allowedsections = [] + forms=['html','txt','epub','mobi'] + allowedsections.extend(forms) + + for section in sitesections: + allowedsections.append(section) + for f in forms: + allowedsections.append('%s:%s'%(section,f)) + return allowedsections + +def get_valid_list_entries(): + return list(['category', + 'genre', + 'characters', + 'ships', + 'warnings', + 'extratags', + 'author', + 'authorId', + 'authorUrl', + 'lastupdate', + ]) + +def get_valid_scalar_entries(): + return list(['series', + 'seriesUrl', + 'language', + 'status', + 'datePublished', + 'dateUpdated', + 'dateCreated', + 'rating', + 'numChapters', + 'numWords', + 'site', + 'storyId', + 'title', + 'storyUrl', + 'description', + 'formatname', + 'formatext', + 'siteabbrev', + 'version', + # internal stuff. + 'authorHTML', + 'seriesHTML', + 'langcode', + 'output_css', + ]) + +def get_valid_entries(): + return get_valid_list_entries() + get_valid_scalar_entries() + +# *known* keywords -- or rather regexps for them. +def get_valid_keywords(): + return list(['(in|ex)clude_metadata_(pre|post)', + 'add_chapter_numbers', + 'add_genre_when_multi_category', + 'allow_unsafe_filename', + 'always_overwrite', + 'anthology_tags', + 'anthology_title_pattern', + 'background_color', + 'bulk_load', + 'chapter_end', + 'chapter_start', + 'chapter_title_add_pattern', + 'chapter_title_strip_pattern', + 'check_next_chapter', + 'collect_series', + 'connect_timeout', + 'convert_images_to', + 'cover_content', + 'cover_exclusion_regexp', + 'custom_columns_settings', + 'dateCreated_format', + 'datePublished_format', + 'dateUpdated_format', + 'default_cover_image', + 'do_update_hook', + 'exclude_notes', + 'extra_logpage_entries', + 'extra_subject_tags', + 'extra_titlepage_entries', + 'extra_valid_entries', + 'extratags', + 'extracategories', + 'extragenres', + 'extracharacters', + 'extraships', + 'extrawarnings', + 'fail_on_password', + 'file_end', + 'file_start', + 'fileformat', + 'find_chapters', + 'fix_fimf_blockquotes', + 'force_login', + 'generate_cover_settings', + 'grayscale_images', + 'image_max_size', + 'include_images', + 'include_logpage', + 'include_subject_tags', + 'include_titlepage', + 'include_tocpage', + 'is_adult', + 'join_string_authorHTML', + 'keep_style_attr', + 'keep_summary_html', + 'logpage_end', + 'logpage_entries', + 'logpage_entry', + 'logpage_start', + 'logpage_update_end', + 'logpage_update_start', + 'make_directories', + 'make_firstimage_cover', + 'make_linkhtml_entries', + 'max_fg_sleep', + 'max_fg_sleep_at_downloads', + 'min_fg_sleep', + 'never_make_cover', + 'no_image_processing', + 'non_breaking_spaces', + 'nook_img_fix', + 'output_css', + 'output_filename', + 'output_filename_safepattern', + 'password', + 'post_process_cmd', + 'remove_transparency', + 'replace_br_with_p', + 'replace_hr', + 'replace_metadata', + 'slow_down_sleep_time', + 'sort_ships', + 'strip_chapter_numbers', + 'strip_chapter_numeral', + 'strip_text_links', + 'titlepage_end', + 'titlepage_entries', + 'titlepage_entry', + 'titlepage_no_title_entry', + 'titlepage_start', + 'titlepage_use_table', + 'titlepage_wide_entry', + 'tocpage_end', + 'tocpage_entry', + 'tocpage_start', + 'tweak_fg_sleep', + 'universe_as_series', + 'user_agent', + 'username', + 'website_encodings', + 'wide_titlepage_entries', + 'windows_eol', + 'wrap_width', + 'zip_filename', + 'zip_output', + ]) + +# *known* entry keywords -- or rather regexps for them. +def get_valid_entry_keywords(): + return list(['%s_label', + '(default_value|include_in|join_string|keep_in_order)_%s',]) + class Configuration(ConfigParser.SafeConfigParser): def __init__(self, site, fileform): ConfigParser.SafeConfigParser.__init__(self) + + self.linenos=dict() # key by section or section,key -> lineno + self.sectionslist = ['defaults'] if site.startswith("www."): @@ -53,45 +239,9 @@ class Configuration(ConfigParser.SafeConfigParser): self.addConfigSection(sitewithout+":"+fileform) self.addConfigSection("overrides") - self.listTypeEntries = [ - 'category', - 'genre', - 'characters', - 'ships', - 'warnings', - 'extratags', - 'author', - 'authorId', - 'authorUrl', - 'lastupdate', - ] + self.listTypeEntries = get_valid_list_entries() - self.validEntries = self.listTypeEntries + [ - 'series', - 'seriesUrl', - 'language', - 'status', - 'datePublished', - 'dateUpdated', - 'dateCreated', - 'rating', - 'numChapters', - 'numWords', - 'site', - 'storyId', - 'title', - 'storyUrl', - 'description', - 'formatname', - 'formatext', - 'siteabbrev', - 'version', - # internal stuff. - 'authorHTML', - 'seriesHTML', - 'langcode', - 'output_css', - ] + self.validEntries = get_valid_entries() def addConfigSection(self,section): self.sectionslist.insert(0,section) @@ -129,7 +279,7 @@ class Configuration(ConfigParser.SafeConfigParser): def getConfig(self, key, default=""): return self.get_config(self.sectionslist,key,default) - def get_config(self, sections, key, default=""): + def get_config(self, sections, key, default=""): val = default for section in sections: try: @@ -162,6 +312,152 @@ class Configuration(ConfigParser.SafeConfigParser): def getConfigList(self, key): return self.get_config_list(self.sectionslist, key) + + def get_lineno(self,section,key=None): + if key: + return self.linenos.get(section+','+key,None) + else: + return self.linenos.get(section,None) + + ## Copied from Python library so as to make it save linenos too. + # + # Regular expressions for parsing section headers and options. + # + def _read(self, fp, fpname): + """Parse a sectioned setup file. + + The sections in setup file contains a title line at the top, + indicated by a name in square brackets (`[]'), plus key/value + options lines, indicated by `name: value' format lines. + Continuations are represented by an embedded newline then + leading whitespace. Blank lines, lines beginning with a '#', + and just about everything else are ignored. + """ + cursect = None # None, or a dictionary + optname = None + lineno = 0 + e = None # None, or an exception + while True: + line = fp.readline() + if not line: + break + lineno = lineno + 1 + # comment or blank line? + if line.strip() == '' or line[0] in '#;': + continue + if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR": + # no leading whitespace + continue + # continuation line? + if line[0].isspace() and cursect is not None and optname: + value = line.strip() + if value: + cursect[optname] = "%s\n%s" % (cursect[optname], value) + # a section header or option header? + else: + # is it a section header? + mo = self.SECTCRE.match(line) + if mo: + sectname = mo.group('header') + if sectname in self._sections: + cursect = self._sections[sectname] + elif sectname == DEFAULTSECT: + cursect = self._defaults + else: + cursect = self._dict() + cursect['__name__'] = sectname + self._sections[sectname] = cursect + self.linenos[sectname]=lineno + # So sections can't start with a continuation line + optname = None + # no section header in the file? + elif cursect is None: + if not e: + e = ParsingError(fpname) + e.append(lineno, u'(Line outside section) '+line) + #raise MissingSectionHeaderError(fpname, lineno, line) + # an option line? + else: + mo = self._optcre.match(line) + if mo: + optname, vi, optval = mo.group('option', 'vi', 'value') + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + if vi in ('=', ':') and ';' in optval: + # ';' is a comment delimiter only if it follows + # a spacing character + pos = optval.find(';') + if pos != -1 and optval[pos-1].isspace(): + optval = optval[:pos] + optval = optval.strip() + # allow empty values + if optval == '""': + optval = '' + optname = self.optionxform(optname.rstrip()) + cursect[optname] = optval + self.linenos[cursect['__name__']+','+optname]=lineno + else: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + if not e: + e = ParsingError(fpname) + e.append(lineno, line) + # if any parsing errors occurred, raise an exception + if e: + raise e + + def test_config(self): + errors=[] + + teststory_re = re.compile(r'^teststory:(defaults|[0-9]+)$') + allowedsections = get_valid_sections() + + clude_metadata_re = re.compile(r'(add_to_)?(in|ex)clude_metadata_(pre|post)') + + replace_metadata_re = re.compile(r'(add_to_)?replace_metadata') + from story import set_in_ex_clude, make_replacements + + custom_columns_settings_re = re.compile(r'(add_to_)?custom_columns_settings') + + for section in self.sections(): + if section not in allowedsections and not teststory_re.match(section): + errors.append((self.get_lineno(section),"Bad Section Name: %s"%section)) + else: + ## check each keyword in section. Due to precedence + ## order of sections, it's possible for bad lines to + ## never be used. + for keyword,value in self.items(section): + try: + + ## check regex bearing keywords first. Each + ## will raise exceptions if flawed. + if clude_metadata_re.match(keyword): + set_in_ex_clude(value) + + if replace_metadata_re.match(keyword): + make_replacements(value) + + # if custom_columns_settings_re.match(keyword): + #custom_columns_settings: + # cliches=>#acolumn + # themes=>#bcolumn,a + # timeline=>#ccolumn,n + # "FanFiction"=>#collection + + + ## skipping output_filename_safepattern + ## regex--not used with plugin and this isn't + ## used with CLI/web yet. + + except Exception as e: + errors.append((self.get_lineno(section,keyword),"Error:%s in (%s:%s)"%(e,keyword,value))) + + + return errors + # extended by adapter, writer and story for ease of calling configuration. class Configurable(object): diff --git a/fanficdownloader/story.py b/fanficdownloader/story.py index d3bad278..a98938a3 100644 --- a/fanficdownloader/story.py +++ b/fanficdownloader/story.py @@ -288,6 +288,59 @@ class InExMatch: else: s='=' return u'InExMatch(%s %s%s %s)'%(self.keys,f,s,self.match) + +## metakey[,metakey]=~pattern +## metakey[,metakey]==string +## *for* part lines. Effect only when trailing conditional key=~regexp matches +## metakey[,metakey]=~pattern[&&metakey=~regexp] +## metakey[,metakey]==string[&&metakey=~regexp] +## metakey[,metakey]=~pattern[&&metakey==string] +## metakey[,metakey]==string[&&metakey==string] +def set_in_ex_clude(setting): + dest = [] + # print("set_in_ex_clude:"+setting) + for line in setting.splitlines(): + if line: + (match,condmatch)=(None,None) + if "&&" in line: + (line,conditional) = line.split("&&") + condmatch = InExMatch(conditional) + match = InExMatch(line) + dest.append([match,condmatch]) + return dest + +## Two or three part lines. Two part effect everything. +## Three part effect only those key(s) lists. +## pattern=>replacement +## metakey,metakey=>pattern=>replacement +## *Five* part lines. Effect only when trailing conditional key=>regexp matches +## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp] +def make_replacements(replace): + retval=[] + for line in replace.splitlines(): + # print("replacement line:%s"%line) + (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None) + if "&&" in line: + (line,conditional) = line.split("&&") + (condkey,condregexp) = conditional.split("=>") + if "=>" in line: + parts = line.split("=>") + if len(parts) > 2: + metakeys = map( lambda x: x.strip(), parts[0].split(",") ) + (regexp,replacement)=parts[1:] + else: + (regexp,replacement)=parts + + if regexp: + regexp = re_compile(regexp,line) + if condregexp: + condregexp = re_compile(condregexp,line) + # A way to explicitly include spaces in the + # replacement string. The .ini parser eats any + # trailing spaces. + replacement=replacement.replace(SPACE_REPLACE,' ') + retval.append([metakeys,regexp,replacement,condkey,condregexp]) + return retval class Story(Configurable): @@ -298,7 +351,6 @@ class Story(Configurable): self.metadata = {'version':os.environ['CURRENT_VERSION_ID']} except: self.metadata = {'version':'4.4'} - self.replacements = [] self.in_ex_cludes = {} self.chapters = [] # chapters will be tuples of (title,html) self.imgurls = [] @@ -318,7 +370,7 @@ class Story(Configurable): for val in self.getConfigList(config): self.addToList(metadata,val) - self.setReplace(self.getConfig('replace_metadata')) + self.replacements = make_replacements(self.getConfig('replace_metadata')) in_ex_clude_list = ['include_metadata_pre','exclude_metadata_pre', 'include_metadata_post','exclude_metadata_post'] @@ -327,7 +379,7 @@ class Story(Configurable): # print("%s %s"%(ie,ies)) if ies: iel = [] - self.in_ex_cludes[ie] = self.set_in_ex_clude(ies) + self.in_ex_cludes[ie] = set_in_ex_clude(ies) def join_list(self, key, vallist): return self.getConfig("join_string_"+key,u", ").replace(SPACE_REPLACE,' ').join(map(unicode, vallist)) @@ -357,26 +409,6 @@ class Story(Configurable): self.addToList('lastupdate',value.strftime("Last Update: %Y/%m/%d")) - ## metakey[,metakey]=~pattern - ## metakey[,metakey]==string - ## *for* part lines. Effect only when trailing conditional key=~regexp matches - ## metakey[,metakey]=~pattern[&&metakey=~regexp] - ## metakey[,metakey]==string[&&metakey=~regexp] - ## metakey[,metakey]=~pattern[&&metakey==string] - ## metakey[,metakey]==string[&&metakey==string] - def set_in_ex_clude(self,setting): - dest = [] - # print("set_in_ex_clude:"+setting) - for line in setting.splitlines(): - if line: - (match,condmatch)=(None,None) - if "&&" in line: - (line,conditional) = line.split("&&") - condmatch = InExMatch(conditional) - match = InExMatch(line) - dest.append([match,condmatch]) - return dest - def do_in_ex_clude(self,which,value,key): if value and which in self.in_ex_cludes: include = 'include' in which @@ -407,37 +439,6 @@ class Story(Configurable): return value - ## Two or three part lines. Two part effect everything. - ## Three part effect only those key(s) lists. - ## pattern=>replacement - ## metakey,metakey=>pattern=>replacement - ## *Five* part lines. Effect only when trailing conditional key=>regexp matches - ## metakey[,metakey]=>pattern=>replacement[&&metakey=>regexp] - def setReplace(self,replace): - for line in replace.splitlines(): - # print("replacement line:%s"%line) - (metakeys,regexp,replacement,condkey,condregexp)=(None,None,None,None,None) - if "&&" in line: - (line,conditional) = line.split("&&") - (condkey,condregexp) = conditional.split("=>") - if "=>" in line: - parts = line.split("=>") - if len(parts) > 2: - metakeys = map( lambda x: x.strip(), parts[0].split(",") ) - (regexp,replacement)=parts[1:] - else: - (regexp,replacement)=parts - - if regexp: - regexp = re_compile(regexp,line) - if condregexp: - condregexp = re_compile(condregexp,line) - # A way to explicitly include spaces in the - # replacement string. The .ini parser eats any - # trailing spaces. - replacement=replacement.replace(SPACE_REPLACE,' ') - self.replacements.append([metakeys,regexp,replacement,condkey,condregexp]) - def doReplacements(self,value,key,return_list=False,seen_list=[]): value = self.do_in_ex_clude('include_metadata_pre',value,key) value = self.do_in_ex_clude('exclude_metadata_pre',value,key)