Sync to trunk.

recipes/den_of_geek.recipe

@@ -0,0 +1,21 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1316944753(BasicNewsRecipe):
+    title          = u'Den of Geek'
+    __author__            = 'Jaded'
+    language = 'en'
+    description = 'From science fiction enthusiasts through to gaming fanatics, Den of Geek has become the one-stop UK website for people genuinely passionate about their entertainment media. Den of Geek covers popular culture but always with an edgy, UK centric slant that sets it apart from the crowd.'
+    category               = 'Movies, TV, Games, Comics, Cult, News, Reviews'
+    language              = 'en'
+
+    oldest_article = 7
+    max_articles_per_feed = 100
+    auto_cleanup = True
+
+    no_stylesheets        = True
+    use_embedded_content  = True
+    publication_type      = 'newsportal'
+    masthead_url ='http://www.denofgeek.com/siteimage/scale/0/0/logo.gif'
+    cover_url ='http://a5.sphotos.ak.fbcdn.net/hphotos-ak-snc6/166479_180131695357862_139191826118516_354818_4993703_n.jpg'
+
+    feeds          = [(u'Movies', u'http://www.denofgeek.com/movies/rss/'), (u'TV', u'http://www.denofgeek.com/television/rss/'), (u'Comics & Books', u'http://www.denofgeek.com/comics/rss/'), (u'Games', u'http://www.denofgeek.com/games/rss/'), (u'DVD/Blu-ray', u'http://www.denofgeek.com/Reviews/rss/')]

src/calibre/db/__init__.py

@@ -7,6 +7,7 @@
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+SPOOL_SIZE = 30*1024*1024
 
 '''
 Rewrite of the calibre database backend.

src/calibre/db/backend.py

@@ -8,7 +8,7 @@
 __docformat__ = 'restructuredtext en'
 
 # Imports {{{
-import os, shutil, uuid, json, glob, time, tempfile
+import os, shutil, uuid, json, glob, time
 from functools import partial
 
 import apsw
@@ -16,7 +16,8 @@
 from calibre import isbytestring, force_unicode, prints
 from calibre.constants import (iswindows, filesystem_encoding,
         preferred_encoding)
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile
+from calibre.db import SPOOL_SIZE
 from calibre.db.schema_upgrades import SchemaUpgrade
 from calibre.library.field_metadata import FieldMetadata
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
@@ -38,7 +39,7 @@
 
 '''
 
-SPOOL_SIZE = 30*1024*1024
+
 
 class DynamicFilter(object): # {{{
 
@@ -805,7 +806,7 @@ def cover(self, path, as_file=False, as_image=False,
                         shutil.copyfileobj(f, pt)
                     return pt.name
                 if as_file:
-                    ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
+                    ret = SpooledTemporaryFile(SPOOL_SIZE)
                     shutil.copyfileobj(f, ret)
                     ret.seek(0)
                 else:

src/calibre/devices/kobo/driver.py

@@ -51,11 +51,11 @@ class KOBO(USBMS):
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
             _('The Kobo supports several collections including ')+\
-                    'Read, Closed, Im_Reading ' +\
+                    'Read, Closed, Im_Reading. ' +\
             _('Create tags for automatic management'),
     ]
 
-    EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['tags'])
+    EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(['tags'])]
 
     OPT_COLLECTIONS = 0
 
@@ -659,7 +659,7 @@ def update_device_database_collections(self, booklists, collections_attributes,
             "Read":2,
             "Closed":3,
             "Shortlist":4,
-            # "Preview":99, # Unsupported as we don't want to change it 
+            # "Preview":99, # Unsupported as we don't want to change it
         }
 
         # Define lists for the ReadStatus

src/calibre/ebooks/metadata/sources/amazon.py

@@ -73,6 +73,20 @@ def __init__(self, url, result_queue, browser, log, relevance, domain, plugin, t
             8: ['août'],
             9: ['sept'],
             12: ['déc'],
+            },
+            'es': {
+                1: ['enero'],
+                2: ['febrero'],
+                3: ['marzo'],
+                4: ['abril'],
+                5: ['mayo'],
+                6: ['junio'],
+                7: ['julio'],
+                8: ['agosto'],
+                9: ['septiembre', 'setiembre'],
+                10: ['octubre'],
+                11: ['noviembre'],
+                12: ['diciembre'],
             },
                 'jp': {
             1: [u'1月'],
@@ -101,13 +115,16 @@ def __init__(self, url, result_queue, browser, log, relevance, domain, plugin, t
                  text()="Dettagli prodotto" or \
                  text()="Product details" or \
                  text()="Détails sur le produit" or \
+                 text()="Detalles del producto" or \
                  text()="登録情報"]/../div[@class="content"]
             '''
+        # Editor: is for Spanish
         self.publisher_xpath = '''
             descendant::*[starts-with(text(), "Publisher:") or \
                     starts-with(text(), "Verlag:") or \
                     starts-with(text(), "Editore:") or \
                     starts-with(text(), "Editeur") or \
+                    starts-with(text(), "Editor:") or \
                     starts-with(text(), "出版社:")]
             '''
         self.language_xpath =    '''
@@ -116,12 +133,14 @@ def __init__(self, url, result_queue, browser, log, relevance, domain, plugin, t
                 or text() = "Language" \
                 or text() = "Sprache:" \
                 or text() = "Lingua:" \
+                or text() = "Idioma:" \
                 or starts-with(text(), "Langue") \
                 or starts-with(text(), "言語") \
                 ]
             '''
+
         self.ratings_pat = re.compile(
-            r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}')
+            r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de) ([\d\.]+)( (stars|Sternen|stelle|estrellas)){0,1}')
 
         lm = {
                 'eng': ('English', 'Englisch'),
@@ -143,6 +162,7 @@ def delocalize_datestr(self, raw):
         for i, vals in self.months.iteritems():
             for x in vals:
                 ans = ans.replace(x, self.english_months[i])
+        ans = ans.replace(' de ', ' ')
         return ans
 
     def run(self):
@@ -422,6 +442,7 @@ class Amazon(Source):
             'uk' : _('UK'),
             'it' : _('Italy'),
             'jp' : _('Japan'),
+            'es' : _('Spain'),
     }
 
     options = (
@@ -789,6 +810,16 @@ def download_cover(self, log, result_queue, abort, # {{{
             ),
     ] # }}}
 
+    es_tests = [ # {{{
+            (
+                {'identifiers':{'isbn': '8483460831'}},
+                [title_test('Tiempos Interesantes',
+                    exact=True), authors_test(['Terry Pratchett'])
+                 ]
+
+            ),
+    ] # }}}
+
     jp_tests = [ # {{{
             ( # isbn -> title, authors
                 {'identifiers':{'isbn': '9784101302720' }},
@@ -804,6 +835,6 @@ def download_cover(self, log, result_queue, abort, # {{{
     ] # }}}
 
     test_identify_plugin(Amazon.name, com_tests)
-    #test_identify_plugin(Amazon.name, jp_tests)
+    #test_identify_plugin(Amazon.name, es_tests)
 # }}}
 

src/calibre/ebooks/metadata/worker.py

@@ -249,6 +249,7 @@ def _run(self, tdir):
             recs[pref.name] = getattr(self.opts, pref.name)
 
         plugboards = self.db.prefs.get('plugboards', {})
+        template_functions = self.db.prefs.get('user_template_functions', [])
 
         for i, task in enumerate(tasks):
             tids = [x[-1] for x in task]
@@ -260,7 +261,7 @@ def _run(self, tdir):
             job = ParallelJob('save_book',
                     'Save books (%d of %d)'%(i, len(tasks)),
                     lambda x,y:x,
-                    args=[tids, dpath, plugboards, self.path, recs])
+                    args=[tids, dpath, plugboards, template_functions, self.path, recs])
             jobs.add(job)
             server.add_job(job)
 
@@ -312,9 +313,12 @@ def get_notifications(self, job, ids):
                 break
 
 
-def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x):
+def save_book(ids, dpath, plugboards, template_functions, path, recs,
+              notification=lambda x,y:x):
     from calibre.library.save_to_disk import config, save_serialized_to_disk
     from calibre.customize.ui import apply_null_metadata
+    from calibre.utils.formatter_functions import load_user_template_functions
+    load_user_template_functions(template_functions)
     opts = config().parse()
     for name in recs:
         setattr(opts, name, recs[name])

src/calibre/ebooks/pdf/images.cpp

@@ -126,7 +126,7 @@ void XMLImages::add(GfxState *state, Object *ref, Stream *str,
 
     if (img->type == jpeg) {
         int c;
-        str = ((DCTStream *)str)->getRawStream();
+        str = str->getNextStream();
         str->reset();
 
         // copy the stream

src/calibre/ebooks/pdf/reflow.cpp

@@ -625,7 +625,12 @@ static string get_link_dest(LinkAction *link, PDFDoc *doc) {
   return oss.str();
 }
 
+#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17)
 void XMLOutputDev::process_link(Link* link){
+#else
+void XMLOutputDev::process_link(AnnotLink* link){
+#endif
+
   double _x1, _y1, _x2, _y2;
   int x1, y1, x2, y2;
   

src/calibre/ebooks/pdf/reflow.h

@@ -244,6 +244,11 @@ class XMLOutputDev : public OutputDev {
     XMLImages *images;
     PDFDoc *doc;
 
+#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17)
     void process_link(Link* link);
+#else
+    void process_link(AnnotLink* link);
+#endif
+
 };
 }

src/calibre/gui2/metadata/basic_widgets.py

@@ -7,7 +7,7 @@
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import textwrap, re, os, errno
+import textwrap, re, os, errno, shutil
 
 from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal, QMessageBox,
     QIcon, QToolButton, QWidget, QLabel, QGridLayout, QApplication,
@@ -33,8 +33,9 @@
 from calibre.library.comments import comments_to_html
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.utils.icu import strcmp
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile
 from calibre.gui2.languages import LanguagesEdit as LE
+from calibre.db import SPOOL_SIZE
 
 def save_dialog(parent, title, msg, det_msg=''):
     d = QMessageBox(parent)
@@ -43,8 +44,6 @@ def save_dialog(parent, title, msg, det_msg=''):
     d.setStandardButtons(QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel)
     return d.exec_()
 
-
-
 '''
 The interface common to all widgets used to set basic metadata
 class BasicMetadataWidget(object):
@@ -731,8 +730,12 @@ def commit(self, db, id_):
             else:
                 old_extensions.add(ext)
         for ext in new_extensions:
-            db.add_format(id_, ext, open(paths[ext], 'rb'), notify=False,
-                    index_is_id=True)
+            with SpooledTemporaryFile(SPOOL_SIZE) as spool:
+                with open(paths[ext], 'rb') as f:
+                    shutil.copyfileobj(f, spool)
+                spool.seek(0)
+                db.add_format(id_, ext, spool, notify=False,
+                        index_is_id=True)
         dbfmts = db.formats(id_, index_is_id=True)
         db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts
             else [])])

src/calibre/library/database2.py

@@ -7,7 +7,7 @@
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
-        json, uuid, tempfile, hashlib, copy
+        json, uuid, hashlib, copy
 from collections import defaultdict
 import threading, random
 from itertools import repeat
@@ -26,7 +26,8 @@
 from calibre.library.prefs import DBPrefs
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
-from calibre.ptempfile import PersistentTemporaryFile, base_dir
+from calibre.ptempfile import (PersistentTemporaryFile,
+        base_dir, SpooledTemporaryFile)
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
@@ -610,7 +611,7 @@ def set_path(self, index, index_is_id=False):
                 with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
                     f.write(cdata)
             for format in formats:
-                with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream:
+                with SpooledTemporaryFile(SPOOL_SIZE) as stream:
                     try:
                         self.copy_format_to(id, format, stream, index_is_id=True)
                         stream.seek(0)
@@ -694,7 +695,7 @@ def cover(self, index, index_is_id=False, as_file=False, as_image=False,
                         shutil.copyfileobj(f, pt)
                     return pt.name
                 if as_file:
-                    ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
+                    ret = SpooledTemporaryFile(SPOOL_SIZE)
                     shutil.copyfileobj(f, ret)
                     ret.seek(0)
                 else:
@@ -1282,7 +1283,7 @@ def format(self, index, format, index_is_id=False, as_file=False,
                             shutil.copyfileobj(f, pt)
                             ret = pt.name
                 elif as_file:
-                    ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE)
+                    ret = SpooledTemporaryFile(SPOOL_SIZE)
                     shutil.copyfileobj(f, ret)
                     ret.seek(0)
                     # Various bits of code try to use the name as the default

src/calibre/ptempfile.py

@@ -181,4 +181,17 @@ def __exit__(self, *args):
 
 
 
+class SpooledTemporaryFile(tempfile.SpooledTemporaryFile):
+
+    def __init__(self, max_size=0, suffix="", prefix="", dir=None, mode='w+b',
+            bufsize=-1):
+        if prefix == None:
+            prefix = ''
+        if suffix is None:
+            suffix = ''
+        if dir is None:
+            dir = base_dir()
+        tempfile.SpooledTemporaryFile.__init__(self, max_size=max_size, suffix=suffix,
+                prefix=prefix, dir=dir, mode=mode, bufsize=bufsize)
+
 

src/calibre/utils/date.py

@@ -170,32 +170,40 @@ def format_date(dt, format, assume_utc=False, as_utc=False):
     if format == 'iso':
         return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc)
 
+    if dt == UNDEFINED_DATE:
+        return ''
+
     strf = partial(strftime, t=dt.timetuple())
 
-    def format_day(mo):
-        l = len(mo.group(0))
+    def format_day(dy):
+        l = len(dy)
         if l == 1: return '%d'%dt.day
         if l == 2: return '%02d'%dt.day
         if l == 3: return strf('%a')
         return strf('%A')
 
     def format_month(mo):
-        l = len(mo.group(0))
+        l = len(mo)
         if l == 1: return '%d'%dt.month
         if l == 2: return '%02d'%dt.month
         if l == 3: return strf('%b')
         return strf('%B')
 
-    def format_year(mo):
-        if len(mo.group(0)) == 2: return '%02d'%(dt.year % 100)
+    def format_year(yr):
+        if len(yr) == 2: return '%02d'%(dt.year % 100)
         return '%04d'%dt.year
 
-    if dt == UNDEFINED_DATE:
-        return ''
+    def repl_func(mo):
+        s = mo.group(0)
+        if s is None:
+            return ''
+        if s[0] == 'd':
+            return format_day(s)
+        if s[0] == 'M':
+            return format_month(s)
+        return format_year(s)
 
-    format = re.sub('d{1,4}', format_day, format)
-    format = re.sub('M{1,4}', format_month, format)
-    return re.sub('yyyy|yy', format_year, format)
+    return re.sub('(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format)
 
 def replace_months(datestr, clang):
     # Replace months by english equivalent for parse_date

src/calibre/web/feeds/news.py

@@ -13,7 +13,7 @@
 from contextlib import nested, closing
 
 
-from calibre import (browser, __appname__, iswindows,
+from calibre import (browser, __appname__, iswindows, force_unicode,
                     strftime, preferred_encoding, as_unicode)
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
 from calibre.ebooks.metadata.opf2 import OPFCreator
@@ -49,7 +49,7 @@ class BasicNewsRecipe(Recipe):
 
     #: A couple of lines that describe the content this recipe downloads.
     #: This will be used primarily in a GUI that presents a list of recipes.
-    description = ''
+    description = u''
 
     #: The author of this recipe
     __author__             = __appname__
@@ -112,8 +112,6 @@ class BasicNewsRecipe(Recipe):
     #: If set to "optional" the use of a username and password becomes optional
     needs_subscription     = False
 
-    #:
-
     #: If True the navigation bar is center aligned, otherwise it is left aligned
     center_navbar = True
 
@@ -1205,12 +1203,22 @@ def create_opf(self, feeds, dir=None):
         mi.author_sort = __appname__
         mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
         mi.timestamp = nowf()
+        article_titles, aseen = [], set()
+        for f in feeds:
+            for a in f:
+                if a.title and a.title not in aseen:
+                    aseen.add(a.title)
+                    article_titles.append(force_unicode(a.title, 'utf-8'))
+
         mi.comments = self.description
+        if not isinstance(mi.comments, unicode):
+            mi.comments = mi.comments.decode('utf-8', 'replace')
+        mi.comments += ('\n\n' + _('Articles in this issue: ') + '\n' +
+                '\n\n'.join(article_titles))
+
         language = canonicalize_lang(self.language)
         if language is not None:
             mi.language = language
-        if not isinstance(mi.comments, unicode):
-            mi.comments = mi.comments.decode('utf-8', 'replace')
         mi.pubdate = nowf()
         opf_path = os.path.join(dir, 'index.opf')
         ncx_path = os.path.join(dir, 'index.ncx')
@@ -1256,6 +1264,7 @@ def create_opf(self, feeds, dir=None):
         self.play_order_counter = 0
         self.play_order_map = {}
 
+
         def feed_index(num, parent):
             f = feeds[num]
             for j, a in enumerate(f):