Sync to trunk.

Changelog.yaml

@@ -19,6 +19,62 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.8.44
+  date: 2012-03-23
+
+  new features:
+    - title: "E-book viewer: A whole new full screen mode."
+      description: "The new mode has no toolbars to distract from the text and the ability to set the width of the column of text via Preferences in the ebook viewer. Click the Fullscreen button on the toolbar in the viewer to enter fullscreen mode (or press the F11 or Ctrl+Shit+F keys)"
+      type: major
+      tickets: [959830]
+
+    - title: "Copy to Library: If books were auto merged by the copy to library process, popup a message telling the user about it, as otherwise some people forget they have turned on auto merge and accuse calibre of losing their books."
+
+    - title: "Unix driver for Ectaco JetBook color"
+      tickets: [958442] 
+
+    - title: "Add a link to the 'Adding Books Preferences' in the drop down menu of the Add Books button for easier access and more prominence"
+      tickets: [958145] 
+
+    - title: "Smarten punctuation: Add a few more cases for detecting opening and closing quotes"
+ 
+  bug fixes:
+    - title: "Get Books: Updates to various store plugins to deal with website changes: Amazon Europe, Waterstones, Foyles, B&N, Kobo, Woblink and Empik"
+
+    - title: "Catalog generation: Do not error out when generating csv/xml catalogs if the catalog title contains filename invalid characters."
+      tickets: [960154]
+
+    - title: "RTF Output: Ignore corrupted images in the input document, instead of erroring out."
+      tickets: [959600]
+
+    - title: "E-book viewer: Try to preserve page position when the window is resized"
+
+    - title: "Fix bug that caused wrong series to be shown when clicking on the first letter of a series group in the Tag Browser"
+
+    - title: "Fix calibre not supporting different http and https proxies."
+      tickets: [960173]
+
+    - title: "MOBI Input: Fix regression caused by KF8 support that broke reading of ancient non-Amazon PRC files"
+
+    - title: "Fix EPUB to EPUB conversion of an EPUB with obfuscated fonts resulting in the fonts not being readable in Adobe Digital Editions"
+      tickets: [957527]
+
+    - title: "RTF Output: Fix bug that broke conversion to RTF when the input document contains <img> tags with no src attribute."
+
+    - title: "Fix regression in 0.8.43 that broke use of general mode templates that ended in a semi-colon."
+      tickets: [957295]
+
+  improved recipes:
+    - b92
+    - Various Polish news sources
+    - Le Monde
+    - FHM UK
+
+  new recipes:
+    - title: Ivana Milakovic and Klub knjige 
+      author: Darko Miletic
+
+
 - version: 0.8.43
   date: 2012-03-16
 

recipes/b92.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008-2011, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 b92.net
 '''
@@ -20,13 +20,13 @@ class B92(BasicNewsRecipe):
     encoding              = 'cp1250'
     language              = 'sr'
     publication_type      = 'newsportal'
-    masthead_url          = 'http://www.b92.net/images/fp/logo.gif'
+    masthead_url          = 'http://b92s.net/v4/img/new-logo.png'
     extra_css             = """ 
-                                @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
                                 @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
                                 body{font-family: Arial,Helvetica,sans1,sans-serif} 
-                                .articledescription{font-family: serif1, serif}
                                 .article-info2,.article-info1{text-transform: uppercase; font-size: small}
+                                img{display: block}
+                                .sms{font-weight: bold}
                             """
     
     conversion_options = {
@@ -37,11 +37,17 @@ class B92(BasicNewsRecipe):
                         , 'linearize_tables' : True
                         }
     
-    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    preprocess_regexps = [
+                           (re.compile(u'\u0110'), lambda match: u'\u00D0'),
+                           (re.compile(r'<html.*?<body>', re.DOTALL|re.IGNORECASE), lambda match: '<html><head><title>something</title></head><body>')
+                         ]
     
     keep_only_tags    = [dict(attrs={'class':['article-info1','article-text']})]
-    remove_attributes = ['width','height','align','hspace','vspace','border']
-    remove_tags       = [dict(name=['embed','link','base','meta'])]
+    remove_attributes = ['width','height','align','hspace','vspace','border','lang','xmlns:fb']
+    remove_tags       = [
+                           dict(name=['embed','link','base','meta','iframe'])
+                          ,dict(attrs={'id':'social'})
+                        ]
 
     feeds          = [
                         (u'Vesti'      , u'http://www.b92.net/info/rss/vesti.xml'     )

recipes/marketing_magazine.recipe

@@ -1,4 +1,6 @@
 __license__   = 'GPL v3'
+__author__    = 'faber1971'
+description   = 'Collection of Italian marketing websites - v1.04 (17, March 2012)'
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
@@ -9,12 +11,9 @@ class AdvancedUserRecipe1327062445(BasicNewsRecipe):
     auto_cleanup = True
     remove_javascript = True
     no_stylesheets = True
+    conversion_options = {'linearize_tables': True}
     remove_tags = [
                      dict(name='ul', attrs={'id':'ads0'})
                   ]
     masthead_url            = 'http://www.simrendeogun.com/wp-content/uploads/2011/06/New-Marketing-Magazine-Logo.jpg'
-    __author__    = 'faber1971'
-    description   = 'Collection of Italian marketing websites - v1.03 (20, February 2012)'
-    language = 'it'
-
-    feeds          = [(u'My Marketing', u'http://feed43.com/0537744466058428.xml'), (u'My Marketing_', u'http://feed43.com/8126723074604845.xml'), (u'Venturini', u'http://robertoventurini.blogspot.com/feeds/posts/default?alt=rss'), (u'Ninja Marketing', u'http://feeds.feedburner.com/NinjaMarketing'), (u'Comunitàzione', u'http://www.comunitazione.it/feed/novita.asp'), (u'Brandforum news', u'http://www.brandforum.it/rss/news'), (u'Brandforum papers', u'http://www.brandforum.it/rss/papers'), (u'MarketingArena', u'http://feeds.feedburner.com/marketingarena'), (u'minimarketing', u'http://feeds.feedburner.com/minimarketingit'), (u'Disambiguando', u'http://giovannacosenza.wordpress.com/feed/')]
+    feeds          = [(u'My Marketing', u'http://feed43.com/0537744466058428.xml'), (u'My Marketing_', u'http://feed43.com/8126723074604845.xml'), (u'Venturini', u'http://robertoventurini.blogspot.com/feeds/posts/default?alt=rss'), (u'Ninja Marketing', u'http://feeds.feedburner.com/NinjaMarketing'), (u'Comunitàzione', u'http://www.comunitazione.it/feed/novita.asp'), (u'Brandforum news', u'http://www.brandforum.it/rss/news'), (u'Brandforum papers', u'http://www.brandforum.it/rss/papers'), (u'MarketingArena', u'http://feeds.feedburner.com/marketingarena'), (u'minimarketing', u'http://feeds.feedburner.com/minimarketingit'), (u'Marketing Journal', u'http://feeds.feedburner.com/marketingjournal/jPwA'), (u'Disambiguando', u'http://giovannacosenza.wordpress.com/feed/')]

setup/iso_639/hi.po

@@ -10,19 +10,19 @@ msgstr ""
 "Report-Msgid-Bugs-To: Debian iso-codes team <pkg-isocodes-"
 "devel@lists.alioth.debian.org>\n"
 "POT-Creation-Date: 2011-11-25 14:01+0000\n"
-"PO-Revision-Date: 2011-09-27 16:03+0000\n"
-"Last-Translator: Kovid Goyal <Unknown>\n"
+"PO-Revision-Date: 2012-03-18 12:56+0000\n"
+"Last-Translator: Vibhav Pant <vibhavp@gmail.com>\n"
 "Language-Team: Hindi\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
-"X-Launchpad-Export-Date: 2011-11-26 05:19+0000\n"
-"X-Generator: Launchpad (build 14381)\n"
+"X-Launchpad-Export-Date: 2012-03-19 04:40+0000\n"
+"X-Generator: Launchpad (build 14969)\n"
 "Language: \n"
 
 #. name for aaa
 msgid "Ghotuo"
-msgstr ""
+msgstr "घोटुओ"
 
 #. name for aab
 msgid "Alumu-Tesu"
@@ -30,7 +30,7 @@ msgstr ""
 
 #. name for aac
 msgid "Ari"
-msgstr ""
+msgstr "अरी"
 
 #. name for aad
 msgid "Amal"
@@ -58,11 +58,11 @@ msgstr ""
 
 #. name for aak
 msgid "Ankave"
-msgstr ""
+msgstr "अनकावे"
 
 #. name for aal
 msgid "Afade"
-msgstr ""
+msgstr "अफ़ाडे"
 
 #. name for aam
 msgid "Aramanik"
@@ -74,7 +74,7 @@ msgstr ""
 
 #. name for aao
 msgid "Arabic; Algerian Saharan"
-msgstr ""
+msgstr "अरबी भाषा; अल्जीरियाई सहारा"
 
 #. name for aap
 msgid "Arára; Pará"
@@ -94,11 +94,11 @@ msgstr ""
 
 #. name for aat
 msgid "Albanian; Arvanitika"
-msgstr ""
+msgstr "अल्बानियन भाषा; अरवनितिका"
 
 #. name for aau
 msgid "Abau"
-msgstr ""
+msgstr "अबाऊ"
 
 #. name for aaw
 msgid "Solong"
@@ -110,7 +110,7 @@ msgstr ""
 
 #. name for aaz
 msgid "Amarasi"
-msgstr ""
+msgstr "अमारासि"
 
 #. name for aba
 msgid "Abé"
@@ -142,7 +142,7 @@ msgstr ""
 
 #. name for abh
 msgid "Arabic; Tajiki"
-msgstr ""
+msgstr "अरबी; ताजिकि"
 
 #. name for abi
 msgid "Abidji"
@@ -150,7 +150,7 @@ msgstr ""
 
 #. name for abj
 msgid "Aka-Bea"
-msgstr ""
+msgstr "अका-बीआ"
 
 #. name for abk
 msgid "Abkhazian"
@@ -166,11 +166,11 @@ msgstr ""
 
 #. name for abn
 msgid "Abua"
-msgstr ""
+msgstr "अबुआ"
 
 #. name for abo
 msgid "Abon"
-msgstr ""
+msgstr "अबोन"
 
 #. name for abp
 msgid "Ayta; Abellen"
@@ -178,7 +178,7 @@ msgstr ""
 
 #. name for abq
 msgid "Abaza"
-msgstr ""
+msgstr "अबाज़ा"
 
 #. name for abr
 msgid "Abron"
@@ -186,7 +186,7 @@ msgstr ""
 
 #. name for abs
 msgid "Malay; Ambonese"
-msgstr ""
+msgstr "मलय; अम्बोनीसी"
 
 #. name for abt
 msgid "Ambulas"

src/calibre/constants.py

@@ -4,7 +4,7 @@
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 43)
+numeric_version = (0, 8, 44)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/ebooks/mobi/debug/headers.py

@@ -328,7 +328,7 @@ def __init__(self, record0, offset):
             (self.sect_idx, self.skel_idx, self.datp_idx, self.oth_idx
                     ) = struct.unpack_from(b'>4L', self.raw, 248)
             self.unknown9 = self.raw[264:self.length]
-            if self.meta_orth_indx != self.sect_idx:
+            if self.meta_orth_indx not in {NULL_INDEX, self.sect_idx}:
                 raise ValueError('KF8 header has different Meta orth and '
                         'section indices')
 

src/calibre/ebooks/rtf/rtfml.py

@@ -109,6 +109,7 @@ def mlize_spine(self):
             if item.spine_position is None:
                 stylizer = Stylizer(item.data, item.href, self.oeb_book,
                         self.opts, self.opts.output_profile)
+                self.currently_dumping_item = item
                 output += self.dump_text(item.data.find(XHTML('body')), stylizer)
                 output += '{\\page }'
         for item in self.oeb_book.spine:
@@ -118,6 +119,7 @@ def mlize_spine(self):
             content = self.remove_tabs(content)
             content = etree.fromstring(content)
             stylizer = Stylizer(content, item.href, self.oeb_book, self.opts, self.opts.output_profile)
+            self.currently_dumping_item = item
             output += self.dump_text(content.find(XHTML('body')), stylizer)
             output += '{\\page }'
         output += self.footer()
@@ -160,9 +162,15 @@ def insert_images(self, text):
 
         for item in self.oeb_book.manifest:
             if item.media_type in OEB_RASTER_IMAGES:
-                src = os.path.basename(item.href)
-                data, width, height = self.image_to_hexstring(item.data)
-                text = text.replace('SPECIAL_IMAGE-%s-REPLACE_ME' % src, '\n\n{\\*\\shppict{\\pict\\picw%i\\pich%i\\jpegblip \n%s\n}}\n\n' % (width, height, data))
+                src = item.href
+                try:
+                    data, width, height = self.image_to_hexstring(item.data)
+                except:
+                    self.log.warn('Image %s is corrupted, ignoring'%item.href)
+                    repl = '\n\n'
+                else:
+                    repl = '\n\n{\\*\\shppict{\\pict\\jpegblip\\picw%i\\pich%i \n%s\n}}\n\n' % (width, height, data)
+                text = text.replace('SPECIAL_IMAGE-%s-REPLACE_ME' % src, repl)
         return text
 
     def image_to_hexstring(self, data):
@@ -205,7 +213,8 @@ def clean_text(self, text):
         return text
 
     def dump_text(self, elem, stylizer, tag_stack=[]):
-        from calibre.ebooks.oeb.base import XHTML_NS, namespace, barename
+        from calibre.ebooks.oeb.base import (XHTML_NS, namespace, barename,
+                urlnormalize)
 
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:
@@ -236,7 +245,7 @@ def dump_text(self, elem, stylizer, tag_stack=[]):
         if tag == 'img':
             src = elem.get('src')
             if src:
-                src = os.path.basename(elem.get('src'))
+                src = urlnormalize(self.currently_dumping_item.abshref(src))
                 block_start = ''
                 block_end = ''
                 if 'block' not in tag_stack:

src/calibre/gui2/actions/catalog.py

@@ -13,6 +13,7 @@
 from calibre.gui2.tools import generate_catalog
 from calibre.utils.config import dynamic
 from calibre.gui2.actions import InterfaceAction
+from calibre import sanitize_file_name_unicode
 
 class GenerateCatalogAction(InterfaceAction):
 
@@ -89,7 +90,8 @@ def catalog_generated(self, job):
                     _('Select destination for %(title)s.%(fmt)s') % dict(
                         title=job.catalog_title, fmt=job.fmt.lower()))
             if export_dir:
-                destination = os.path.join(export_dir, '%s.%s' % (job.catalog_title, job.fmt.lower()))
+                destination = os.path.join(export_dir, '%s.%s' % (
+                    sanitize_file_name_unicode(job.catalog_title), job.fmt.lower()))
                 shutil.copyfile(job.catalog_file_path, destination)
 
 

src/calibre/gui2/actions/copy_to_library.py

@@ -13,7 +13,8 @@
 from PyQt4.Qt import QToolButton
 
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2 import error_dialog, Dispatcher, warning_dialog, gprefs
+from calibre.gui2 import (error_dialog, Dispatcher, warning_dialog, gprefs,
+        info_dialog)
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import now
@@ -30,6 +31,7 @@ def __init__(self, ids, db, loc, progress, done, delete_after):
         self.progress = progress
         self.done = done
         self.delete_after = delete_after
+        self.auto_merged_ids = {}
 
     def run(self):
         try:
@@ -79,6 +81,8 @@ def _doit(self, newdb):
             if prefs['add_formats_to_existing']:
                 identical_book_list = newdb.find_identical_books(mi)
                 if identical_book_list: # books with same author and nearly same title exist in newdb
+                    self.auto_merged_ids[x] = _('%(title)s by %(author)s')%\
+                    dict(title=mi.title, author=mi.format_field('authors')[1])
                     automerged = True
                     seen_fmts = set()
                     for identical_book in identical_book_list:
@@ -196,6 +200,15 @@ def progress(idx, title):
             self.gui.status_bar.show_message(
                     _('Copied %(num)d books to %(loc)s') %
                     dict(num=len(ids), loc=loc), 2000)
+            if self.worker.auto_merged_ids:
+                books = '\n'.join(self.worker.auto_merged_ids.itervalues())
+                info_dialog(self.gui, _('Auto merged'),
+                        _('Some books were automatically merged into existing '
+                            'records in the target library. Click Show '
+                            'details to see which ones. This behavior is '
+                            'controlled by the Auto merge option in '
+                            'Preferences->Adding books.'), det_msg=books,
+                        show=True)
             if delete_after and self.worker.processed:
                 v = self.gui.library_view
                 ci = v.currentIndex()

src/calibre/gui2/comments_editor.py

@@ -9,10 +9,10 @@
 
 from lxml import html
 
-from PyQt4.Qt import QApplication, QFontInfo, QSize, QWidget, QPlainTextEdit, \
-    QToolBar, QVBoxLayout, QAction, QIcon, Qt, QTabWidget, QUrl, \
-    QSyntaxHighlighter, QColor, QChar, QColorDialog, QMenu, QInputDialog, \
-    QHBoxLayout
+from PyQt4.Qt import (QApplication, QFontInfo, QSize, QWidget, QPlainTextEdit,
+    QToolBar, QVBoxLayout, QAction, QIcon, Qt, QTabWidget, QUrl,
+    QSyntaxHighlighter, QColor, QChar, QColorDialog, QMenu, QInputDialog,
+    QHBoxLayout, QKeySequence)
 from PyQt4.QtWebKit import QWebView, QWebPage
 
 from calibre.ebooks.chardet import xml_to_unicode
@@ -32,6 +32,7 @@ def __init__(self, wac, icon, text, checkable, view):
                 type=Qt.QueuedConnection)
         self.page_action.changed.connect(self.update_state,
                 type=Qt.QueuedConnection)
+        self.update_state()
 
     @property
     def page_action(self):
@@ -66,6 +67,12 @@ def __init__(self, parent=None):
 
         self.comments_pat = re.compile(r'<!--.*?-->', re.DOTALL)
 
+        extra_shortcuts = {
+                'ToggleBold': 'Bold',
+                'ToggleItalic': 'Italic',
+                'ToggleUnderline': 'Underline',
+        }
+
         for wac, name, icon, text, checkable in [
                 ('ToggleBold', 'bold', 'format-text-bold', _('Bold'), True),
                 ('ToggleItalic', 'italic', 'format-text-italic', _('Italic'),
@@ -106,6 +113,9 @@ def __init__(self, parent=None):
             ]:
             ac = PageAction(wac, icon, text, checkable, self)
             setattr(self, 'action_'+name, ac)
+            ss = extra_shortcuts.get(wac, None)
+            if ss:
+                ac.setShortcut(QKeySequence(getattr(QKeySequence, ss)))
 
         self.action_color = QAction(QIcon(I('format-text-color')), _('Foreground color'),
                 self)

src/calibre/gui2/complete.py

@@ -6,8 +6,8 @@
 __docformat__ = 'restructuredtext en'
 
 
-from PyQt4.Qt import QLineEdit, QAbstractListModel, Qt, \
-        QApplication, QCompleter
+from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt,
+        QApplication, QCompleter, QMetaObject)
 
 from calibre.utils.icu import sort_key, lower
 from calibre.gui2 import NONE
@@ -182,14 +182,27 @@ def set_space_before_sep(self, space_before):
     def set_add_separator(self, what):
         self.lineEdit().set_add_separator(what)
 
-
+    def show_initial_value(self, what):
+        '''
+        Show an initial value. Handle the case of the initial value being blank
+        correctly (on Qt 4.8.0 having a blank value causes the first value from
+        the completer to be shown, when the event loop runs).
+        '''
+        what = unicode(what)
+        le = self.lineEdit()
+        if not what.strip():
+            QMetaObject.invokeMethod(self, 'clearEditText',
+                    Qt.QueuedConnection)
+        else:
+            self.setEditText(what)
+            le.selectAll()
 
 if __name__ == '__main__':
     from PyQt4.Qt import QDialog, QVBoxLayout
     app = QApplication([])
     d = QDialog()
     d.setLayout(QVBoxLayout())
-    le = MultiCompleteLineEdit(d)
+    le = MultiCompleteComboBox(d)
     d.layout().addWidget(le)
     le.all_items = ['one', 'otwo', 'othree', 'ooone', 'ootwo', 'oothree']
     d.exec_()

src/calibre/gui2/library/delegates.py

@@ -128,8 +128,7 @@ def createEditor(self, parent, option, index):
             for item in sorted(complete_items, key=sort_key):
                 editor.addItem(item)
             ct = index.data(Qt.DisplayRole).toString()
-            editor.setEditText(ct)
-            editor.lineEdit().selectAll()
+            editor.show_initial_value(ct)
         else:
             editor = EnLineEdit(parent)
         return editor
@@ -170,8 +169,7 @@ def createEditor(self, parent, option, index):
             for item in sorted(all_items, key=sort_key):
                 editor.addItem(item)
             ct = index.data(Qt.DisplayRole).toString()
-            editor.setEditText(ct)
-            editor.lineEdit().selectAll()
+            editor.show_initial_value(ct)
         else:
             editor = EnLineEdit(parent)
         return editor
@@ -190,8 +188,7 @@ def createEditor(self, parent, option, index):
         editor = LanguagesEdit(parent=parent)
         editor.init_langs(index.model().db)
         ct = index.data(Qt.DisplayRole).toString()
-        editor.setEditText(ct)
-        editor.lineEdit().selectAll()
+        editor.show_initial_value(ct)
         return editor
 
     def setModelData(self, editor, model, index):

src/calibre/gui2/metadata/single_download.py

@@ -882,6 +882,11 @@ def __init__(self, current_cover=None, parent=None):
         self.covers_widget.chosen.connect(self.ok_clicked)
         self.stack.addWidget(self.covers_widget)
 
+        # Workaround for Qt 4.8.0 bug that causes the frame of the window to go
+        # off the top of the screen if a max height is not set for the
+        # QWebView. Seems to only happen on windows, but keep it for all
+        # platforms just in case.
+        self.identify_widget.comments_view.setMaximumHeight(500)
         self.resize(850, 550)
 
         self.finished.connect(self.cleanup)

src/calibre/gui2/tag_browser/model.py

@@ -1170,6 +1170,8 @@ def tokens(self):
                         charclass = ''.join(letters_seen)
                         if k == 'author_sort':
                             expr = r'%s:"~(^[%s])|(&\s*[%s])"'%(k, charclass, charclass)
+                        elif k == 'series':
+                            expr = r'series_sort:"~^[%s]"'%(charclass)
                         else:
                             expr = r'%s:"~^[%s]"'%(k, charclass)
                         if node_searches[tag_item.tag.state] == 'true':

src/calibre/gui2/viewer/documentview.py

@@ -8,7 +8,7 @@
 from base64 import b64encode
 from functools import partial
 
-from PyQt4.Qt import (QSize, QSizePolicy, QUrl, SIGNAL, Qt, QTimer,
+from PyQt4.Qt import (QSize, QSizePolicy, QUrl, SIGNAL, Qt,
                      QPainter, QPalette, QBrush, QFontDatabase, QDialog,
                      QColor, QPoint, QImage, QRegion, QVariant, QIcon,
                      QFont, pyqtSignature, QAction, QByteArray, QMenu,
@@ -184,12 +184,10 @@ def do_config(self, parent=None):
                 self.misc_config()
                 self.after_load()
 
-    def __init__(self, shortcuts, parent=None, resize_callback=lambda: None,
-            debug_javascript=False):
+    def __init__(self, shortcuts, parent=None, debug_javascript=False):
         QWebPage.__init__(self, parent)
         self.setObjectName("py_bridge")
         self.debug_javascript = debug_javascript
-        self.resize_callback = resize_callback
         self.current_language = None
         self.loaded_javascript = False
         self.js_loader = JavaScriptLoader(
@@ -259,12 +257,6 @@ def load_javascript_libraries(self):
         if self.loaded_javascript:
             return
         self.loaded_javascript = True
-        self.javascript(
-            '''
-            window.onresize = function(event) {
-                window.py_bridge.window_resized();
-            }
-            ''')
         self.loaded_lang = self.js_loader(self.mainFrame().evaluateJavaScript,
                 self.current_language, self.hyphenate_default_lang)
 
@@ -310,10 +302,6 @@ def switch_to_window_mode(self):
     def debug(self, msg):
         prints(msg)
 
-    @pyqtSignature('')
-    def window_resized(self):
-        self.resize_callback()
-
     def reference_mode(self, enable):
         self.javascript(('enter' if enable else 'leave')+'_reference_mode()')
 
@@ -444,7 +432,7 @@ def xpos(self):
     def scroll_fraction(self):
         def fget(self):
             try:
-                return float(self.ypos)/(self.height-self.window_height)
+                return abs(float(self.ypos)/(self.height-self.window_height))
             except ZeroDivisionError:
                 return 0.
         def fset(self, val):
@@ -516,7 +504,6 @@ def initialize_view(self, debug_javascript=False):
         self.initial_pos = 0.0
         self.to_bottom = False
         self.document = Document(self.shortcuts, parent=self,
-                resize_callback=self.viewport_resized,
                 debug_javascript=debug_javascript)
         self.setPage(self.document)
         self.manager = None
@@ -722,6 +709,7 @@ def load_path(self, path, pos=0.0):
         if self.manager is not None:
             self.manager.load_started()
         self.loading_url = QUrl.fromLocalFile(path)
+        html = re.sub(ur'<\s*title\s*/\s*>', u'', html, flags=re.IGNORECASE)
         if has_svg:
             self.setContent(QByteArray(html.encode(path.encoding)), mt, QUrl.fromLocalFile(path))
         else:
@@ -1035,13 +1023,9 @@ def handle_key_press(self, event):
         return handled
 
     def resizeEvent(self, event):
-        ret = QWebView.resizeEvent(self, event)
-        QTimer.singleShot(10, self.initialize_scrollbar)
-        return ret
-
-    def viewport_resized(self):
         if self.manager is not None:
-            self.manager.viewport_resized(self.scroll_fraction)
+            self.manager.viewport_resize_started(event)
+        return QWebView.resizeEvent(self, event)
 
     def event(self, ev):
         if ev.type() == ev.Gesture:

src/calibre/gui2/viewer/main.py

@@ -224,6 +224,10 @@ def __init__(self, pathtoebook=None, debug_javascript=False, open_at=None):
         self.toc.setVisible(False)
         self.action_quit = QAction(self)
         self.addAction(self.action_quit)
+        self.view_resized_timer = QTimer(self)
+        self.view_resized_timer.timeout.connect(self.viewport_resize_finished)
+        self.view_resized_timer.setSingleShot(True)
+        self.resize_in_progress = False
         qs = [Qt.CTRL+Qt.Key_Q]
         if isosx:
             qs += [Qt.CTRL+Qt.Key_W]
@@ -264,6 +268,9 @@ def __init__(self, pathtoebook=None, debug_javascript=False, open_at=None):
         self.connect(self.action_full_screen, SIGNAL('triggered(bool)'),
                      self.toggle_fullscreen)
         self.action_full_screen.setShortcuts([Qt.Key_F11, Qt.CTRL+Qt.SHIFT+Qt.Key_F])
+        self.action_full_screen.setToolTip(_('Toggle full screen (%s)') %
+                _(' or ').join([unicode(x.toString(x.NativeText)) for x in
+                    self.action_full_screen.shortcuts()]))
         self.connect(self.action_back, SIGNAL('triggered(bool)'), self.back)
         self.connect(self.action_bookmark, SIGNAL('triggered(bool)'), self.bookmark)
         self.connect(self.action_forward, SIGNAL('triggered(bool)'), self.forward)
@@ -311,6 +318,7 @@ def __init__(self, pathtoebook=None, debug_javascript=False, open_at=None):
             border-radius: 20px;
         }
         ''')
+        self.window_mode_changed = None
         self.toggle_toolbar_action = QAction(_('Show/hide controls'), self)
         self.toggle_toolbar_action.triggered.connect(self.toggle_toolbars)
         self.addAction(self.toggle_toolbar_action)
@@ -441,6 +449,8 @@ def toggle_fullscreen(self, x):
             self.showFullScreen()
 
     def showFullScreen(self):
+        self.view.document.page_position.save()
+        self.window_mode_changed = 'fullscreen'
         self.tool_bar.setVisible(False)
         self.tool_bar2.setVisible(False)
         self._original_frame_margins = (
@@ -450,7 +460,6 @@ def showFullScreen(self):
         self.centralwidget.layout().setContentsMargins(0, 0, 0, 0)
 
         super(EbookViewer, self).showFullScreen()
-        QTimer.singleShot(10, self.show_full_screen_label)
 
     def show_full_screen_label(self):
         f = self.full_screen_label
@@ -469,6 +478,8 @@ def show_full_screen_label(self):
         self.view.document.switch_to_fullscreen_mode()
 
     def showNormal(self):
+        self.view.document.page_position.save()
+        self.window_mode_changed = 'normal'
         self.esc_full_screen_action.setEnabled(False)
         self.tool_bar.setVisible(True)
         self.tool_bar2.setVisible(True)
@@ -478,7 +489,16 @@ def showNormal(self):
             self.centralwidget.layout().setContentsMargins(om[0])
             self.frame.layout().setContentsMargins(om[1])
         super(EbookViewer, self).showNormal()
-        self.view.document.switch_to_window_mode()
+
+    def handle_window_mode_toggle(self):
+        if self.window_mode_changed:
+            fs = self.window_mode_changed == 'fullscreen'
+            self.window_mode_changed = None
+            if fs:
+                self.show_full_screen_label()
+            else:
+                self.view.document.switch_to_window_mode()
+            self.view.document.page_position.restore()
 
     def goto(self, ref):
         if ref:
@@ -507,6 +527,10 @@ def goto_bookmark(self, bm):
     def toc_clicked(self, index):
         item = self.toc_model.itemFromIndex(index)
         if item.abspath is not None:
+            if not os.path.exists(item.abspath):
+                return error_dialog(self, _('No such location'),
+                        _('The location pointed to by this item'
+                            ' does not exist.'), show=True)
             url = QUrl.fromLocalFile(item.abspath)
             if item.fragment:
                 url.setFragment(item.fragment)
@@ -674,16 +698,28 @@ def load_path(self, path, pos=0.0):
         self.open_progress_indicator(_('Laying out %s')%self.current_title)
         self.view.load_path(path, pos=pos)
 
-    def viewport_resized(self, frac):
-        new_page = self.pos.value()
-        if self.current_page is not None:
-            try:
-                frac = float(new_page-self.current_page.start_page)/(self.current_page.pages-1)
-            except ZeroDivisionError:
-                frac = 0
-            self.view.scroll_to(frac, notify=False)
+    def viewport_resize_started(self, event):
+        if not self.resize_in_progress:
+            # First resize, so save the current page position
+            self.resize_in_progress = True
+            if not self.window_mode_changed:
+                # The special handling for window mode changed will already
+                # have saved page position, so only save it if this is not a
+                # mode change
+                self.view.document.page_position.save()
+
+        if self.resize_in_progress:
+            self.view_resized_timer.start(75)
+
+    def viewport_resize_finished(self):
+        # There hasn't been a resize event for some time
+        # restore the current page position.
+        self.resize_in_progress = False
+        if self.window_mode_changed:
+            # This resize is part of a window mode change, special case it
+            self.handle_window_mode_toggle()
         else:
-            self.set_page_number(frac)
+            self.view.document.page_position.restore()
 
     def close_progress_indicator(self):
         self.pi.stop()

src/calibre/gui2/viewer/main.ui

@@ -284,9 +284,6 @@
    <property name="text">
     <string>Toggle full screen</string>
    </property>
-   <property name="toolTip">
-    <string>Toggle full screen (F11)</string>
-   </property>
   </action>
   <action name="action_print">
    <property name="icon">

src/calibre/gui2/viewer/position.py

@@ -57,12 +57,20 @@ def current_pos(self):
         return ans
 
     def __enter__(self):
-        self._cpos = self.current_pos
+        self.save()
 
     def __exit__(self, *args):
+        self.restore()
+
+    def save(self):
+        self._cpos = self.current_pos
+
+    def restore(self):
+        if self._cpos is None: return
         if isinstance(self._cpos, (int, float)):
             self.document.scroll_fraction = self._cpos
         else:
             self.scroll_to_cfi(self._cpos)
         self._cpos = None
 
+

src/calibre/library/caches.py

@@ -172,11 +172,14 @@ def force_to_bool(val):
 
 class CacheRow(list): # {{{
 
-    def __init__(self, db, composites, val):
+    def __init__(self, db, composites, val, series_col, series_sort_col):
         self.db = db
         self._composites = composites
         list.__init__(self, val)
         self._must_do = len(composites) > 0
+        self._series_col = series_col
+        self._series_sort_col = series_sort_col
+        self._series_sort = None
 
     def __getitem__(self, col):
         if self._must_do:
@@ -191,12 +194,19 @@ def __getitem__(self, col):
             elif col in self._composites:
                 is_comp = True
             if is_comp:
-                id = list.__getitem__(self, 0)
+                id_ = list.__getitem__(self, 0)
                 self._must_do = False
-                mi = self.db.get_metadata(id, index_is_id=True,
+                mi = self.db.get_metadata(id_, index_is_id=True,
                                           get_user_categories=False)
                 for c in self._composites:
                     self[c] =  mi.get(self._composites[c])
+        if col == self._series_sort_col and self._series_sort is None:
+            if self[self._series_col]:
+                self._series_sort = title_sort(self[self._series_col])
+                self[self._series_sort_col] = self._series_sort
+            else:
+                self._series_sort = ''
+                self[self._series_sort_col] = ''
         return list.__getitem__(self, col)
 
     def __getslice__(self, i, j):
@@ -226,6 +236,8 @@ def __init__(self, FIELD_MAP, field_metadata, db_prefs=None):
         for key in field_metadata:
             if field_metadata[key]['datatype'] == 'composite':
                 self.composites[field_metadata[key]['rec_index']] = key
+        self.series_col = field_metadata['series']['rec_index']
+        self.series_sort_col = field_metadata['series_sort']['rec_index']
         self._data = []
         self._map = self._map_filtered = []
         self.first_sort = True
@@ -918,9 +930,11 @@ def refresh_ids(self, db, ids):
         for id in ids:
             try:
                 self._data[id] = CacheRow(db, self.composites,
-                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
+                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0],
+                        self.series_col, self.series_sort_col)
                 self._data[id].append(db.book_on_device_string(id))
                 self._data[id].append(self.marked_ids_dict.get(id, None))
+                self._data[id].append(None)
             except IndexError:
                 return None
         try:
@@ -935,9 +949,11 @@ def books_added(self, ids, db):
         self._data.extend(repeat(None, max(ids)-len(self._data)+2))
         for id in ids:
             self._data[id] = CacheRow(db, self.composites,
-                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
+                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0],
+                        self.series_col, self.series_sort_col)
             self._data[id].append(db.book_on_device_string(id))
             self._data[id].append(self.marked_ids_dict.get(id, None))
+            self._data[id].append(None) # Series sort column
         self._map[0:0] = ids
         self._map_filtered[0:0] = ids
 
@@ -962,11 +978,13 @@ def refresh(self, db, field=None, ascending=True):
         temp = db.conn.get('SELECT * FROM meta2')
         self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else []
         for r in temp:
-            self._data[r[0]] = CacheRow(db, self.composites, r)
+            self._data[r[0]] = CacheRow(db, self.composites, r,
+                                        self.series_col, self.series_sort_col)
         for item in self._data:
             if item is not None:
                 item.append(db.book_on_device_string(item[0]))
-                item.append(None)
+                # Temp mark and series_sort columns
+                item.extend((None, None))
 
         marked_col = self.FIELD_MAP['marked']
         for id_,val in self.marked_ids_dict.iteritems():

src/calibre/library/check_library.py

@@ -47,8 +47,8 @@ def __init__(self, library_path, db):
         self.is_case_sensitive = db.is_case_sensitive
 
         self.all_authors = frozenset([x[1] for x in db.all_authors()])
-        self.all_ids = frozenset([id for id in db.all_ids()])
-        self.all_dbpaths = frozenset(self.dbpath(id) for id in self.all_ids)
+        self.all_ids = frozenset([id_ for id_ in db.all_ids()])
+        self.all_dbpaths = frozenset(self.dbpath(id_) for id_ in self.all_ids)
         self.all_lc_dbpaths = frozenset([f.lower() for f in self.all_dbpaths])
 
         self.db_id_regexp = re.compile(r'^.* \((\d+)\)$')
@@ -73,8 +73,8 @@ def __init__(self, library_path, db):
 
         self.failed_folders = []
 
-    def dbpath(self, id):
-        return self.db.path(id, index_is_id=True)
+    def dbpath(self, id_):
+        return self.db.path(id_, index_is_id=True)
 
     @property
     def errors_occurred(self):
@@ -116,21 +116,21 @@ def scan_library(self, name_ignores, extension_ignores):
                     self.invalid_titles.append((auth_dir, db_path, 0))
                     continue
 
-                id = m.group(1)
-                # Third check: the id must be in the DB and the paths must match
+                id_ = m.group(1)
+                # Third check: the id_ must be in the DB and the paths must match
                 if self.is_case_sensitive:
-                    if int(id) not in self.all_ids or \
+                    if int(id_) not in self.all_ids or \
                             db_path not in self.all_dbpaths:
                         self.extra_titles.append((title_dir, db_path, 0))
                         continue
                 else:
-                    if int(id) not in self.all_ids or \
+                    if int(id_) not in self.all_ids or \
                             db_path.lower() not in self.all_lc_dbpaths:
                         self.extra_titles.append((title_dir, db_path, 0))
                         continue
 
                 # Record the book to check its formats
-                self.book_dirs.append((db_path, title_dir, id))
+                self.book_dirs.append((db_path, title_dir, id_))
                 found_titles = True
 
             # Fourth check: author directories that contain no titles
@@ -145,6 +145,21 @@ def scan_library(self, name_ignores, extension_ignores):
                 # Sort-of check: exception processing directory
                 self.failed_folders.append((title_path, traceback.format_exc(), []))
 
+        # Check for formats and covers in db for book dirs that are gone
+        for id_ in self.all_ids:
+            path = self.dbpath(id_)
+            if not os.path.exists(os.path.join(lib, path)):
+                title_dir = os.path.basename(path)
+                book_formats = frozenset([x for x in
+                            self.db.format_files(id_, index_is_id=True)])
+                for fmt in book_formats:
+                    self.missing_formats.append((title_dir,
+                            os.path.join(path, fmt[0]+'.'+fmt[1].lower()), id_))
+                if self.db.has_cover(id_):
+                    self.missing_covers.append((title_dir,
+                            os.path.join(path, 'cover.jpg'), id_))
+
+
     def is_ebook_file(self, filename):
         ext = os.path.splitext(filename)[1]
         if not ext:
@@ -226,8 +241,8 @@ def lc_map(fnames, fset):
         if self.db.has_cover(book_id):
             if 'cover.jpg' not in filenames:
                 self.missing_covers.append((title_dir,
-                        os.path.join(db_path, title_dir, 'cover.jpg'), book_id))
+                        os.path.join(db_path, 'cover.jpg'), book_id))
         else:
             if 'cover.jpg' in filenames:
                 self.extra_covers.append((title_dir,
-                        os.path.join(db_path, title_dir, 'cover.jpg'), book_id))
+                        os.path.join(db_path, 'cover.jpg'), book_id))

src/calibre/library/cli.py

@@ -204,7 +204,8 @@ def write(self, msg):
         pass
 NULL = DevNull()
 
-def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
+def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
+        oauthors, oisbn, otags, oseries, oseries_index):
     orig = sys.stdout
     #sys.stdout = NULL
     try:
@@ -231,6 +232,11 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
                 mi.title = os.path.splitext(os.path.basename(book))[0]
             if not mi.authors:
                 mi.authors = [_('Unknown')]
+            for x in ('title', 'authors', 'isbn', 'tags', 'series'):
+                val = locals()[x]
+                if val: setattr(mi, x[1:], val)
+            if oseries:
+                mi.series_index = oseries_index
 
             formats.append(format)
             metadata.append(mi)
@@ -302,39 +308,56 @@ def add_option_parser():
     parser.add_option('-e', '--empty', action='store_true', default=False,
                     help=_('Add an empty book (a book with no formats)'))
     parser.add_option('-t', '--title', default=None,
-            help=_('Set the title of the added empty book'))
+            help=_('Set the title of the added book(s)'))
     parser.add_option('-a', '--authors', default=None,
-            help=_('Set the authors of the added empty book'))
+            help=_('Set the authors of the added book(s)'))
     parser.add_option('-i', '--isbn', default=None,
-            help=_('Set the ISBN of the added empty book'))
+            help=_('Set the ISBN of the added book(s)'))
+    parser.add_option('-T', '--tags', default=None,
+            help=_('Set the tags of the added book(s)'))
+    parser.add_option('-s', '--series', default=None,
+            help=_('Set the series of the added book(s)'))
+    parser.add_option('-S', '--series-index', default=1.0, type=float,
+            help=_('Set the series number of the added book(s)'))
+
 
     return parser
 
-def do_add_empty(db, title, authors, isbn):
-    from calibre.ebooks.metadata import MetaInformation, string_to_authors
+def do_add_empty(db, title, authors, isbn, tags, series, series_index):
+    from calibre.ebooks.metadata import MetaInformation
     mi = MetaInformation(None)
     if title is not None:
         mi.title = title
     if authors:
-        mi.authors = string_to_authors(authors)
+        mi.authors = authors
     if isbn:
         mi.isbn = isbn
+    if tags:
+        mi.tags = tags
+    if series:
+        mi.series, mi.series_index = series, series_index
     db.import_book(mi, [])
     write_dirtied(db)
     send_message()
 
 def command_add(args, dbpath):
+    from calibre.ebooks.metadata import string_to_authors
     parser = add_option_parser()
     opts, args = parser.parse_args(sys.argv[:1] + args)
+    aut = string_to_authors(opts.authors) if opts.authors else []
+    tags = [x.strip() for x in opts.tags.split(',')] if opts.tags else []
     if opts.empty:
-        do_add_empty(get_db(dbpath, opts), opts.title, opts.authors, opts.isbn)
+        do_add_empty(get_db(dbpath, opts), opts.title, aut, opts.isbn, tags,
+                opts.series, opts.series_index)
         return 0
     if len(args) < 2:
         parser.print_help()
         print
         print >>sys.stderr, _('You must specify at least one file to add')
         return 1
-    do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory, opts.recurse, opts.duplicates)
+    do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory,
+            opts.recurse, opts.duplicates, opts.title, opts.author, opts.isbn,
+            tags, opts.series, opts.series_index)
     return 0
 
 def do_remove(db, ids):

src/calibre/library/database2.py

@@ -434,6 +434,8 @@ def migrate_preference(key, default):
         self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False)
         self.FIELD_MAP['marked'] = base = base+1
         self.field_metadata.set_field_record_index('marked', base, prefer_custom=False)
+        self.FIELD_MAP['series_sort'] = base = base+1
+        self.field_metadata.set_field_record_index('series_sort', base, prefer_custom=False)
 
         script = '''
         DROP VIEW IF EXISTS meta2;

src/calibre/library/field_metadata.py

@@ -327,6 +327,16 @@ class FieldMetadata(dict):
                              'is_custom':False,
                              'is_category':False,
                            'is_csp': False}),
+            ('series_sort',  {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':{},
+                           'kind':'field',
+                           'name':_('Series Sort'),
+                           'search_terms':['series_sort'],
+                           'is_custom':False,
+                           'is_category':False,
+                           'is_csp': False}),
             ('sort',      {'table':None,
                            'column':None,
                            'datatype':'text',

src/calibre/manual/template_lang.rst

@@ -298,6 +298,7 @@ The following functions are available in addition to those described in single-f
     * ``or(value, value, ...)`` -- returns the string "1" if any value is not empty, otherwise returns the empty string. This function works well with test or first_non_empty. You can have as many values as you want.
     * ``print(a, b, ...)`` -- prints the arguments to standard output. Unless you start calibre from the command line (``calibre-debug -g``), the output will go to a black hole.
     * ``raw_field(name)`` -- returns the metadata field named by name without applying any formatting.
+    * ``series_sort()`` -- returns the series sort value.
     * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
     * ``strcat_max(max, string1, prefix2, string2, ...)`` -- Returns a string formed by concatenating the arguments. The returned value is initialized to string1. `Prefix, string` pairs are added to the end of the value as long as the resulting string length is less than `max`. String1 is returned even if string1 is longer than max. You can pass as many `prefix, string` pairs as you wish.
     * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.