'
__docformat__ = 'restructuredtext en'
-import os, textwrap, sys, operator
+import textwrap, operator
from copy import deepcopy, copy
from lxml import etree
-from calibre.customize.conversion import InputFormatPlugin
from calibre import guess_type
class Canvas(etree.XSLTExtension):
@@ -406,76 +405,4 @@ def get_text_styles(self, node):
-class LRFInput(InputFormatPlugin):
- name = 'LRF Input'
- author = 'Kovid Goyal'
- description = 'Convert LRF files to HTML'
- file_types = set(['lrf'])
-
- def convert(self, stream, options, file_ext, log,
- accelerators):
- self.log = log
- self.log('Generating XML')
- from calibre.ebooks.lrf.lrfparser import LRFDocument
- d = LRFDocument(stream)
- d.parse()
- xml = d.to_xml(write_files=True)
- if options.verbose > 2:
- open('lrs.xml', 'wb').write(xml.encode('utf-8'))
- parser = etree.XMLParser(no_network=True, huge_tree=True)
- try:
- doc = etree.fromstring(xml, parser=parser)
- except:
- self.log.warn('Failed to parse XML. Trying to recover')
- parser = etree.XMLParser(no_network=True, huge_tree=True,
- recover=True)
- doc = etree.fromstring(xml, parser=parser)
-
-
- char_button_map = {}
- for x in doc.xpath('//CharButton[@refobj]'):
- ro = x.get('refobj')
- jump_button = doc.xpath('//*[@objid="%s"]'%ro)
- if jump_button:
- jump_to = jump_button[0].xpath('descendant::JumpTo[@refpage and @refobj]')
- if jump_to:
- char_button_map[ro] = '%s.xhtml#%s'%(jump_to[0].get('refpage'),
- jump_to[0].get('refobj'))
- plot_map = {}
- for x in doc.xpath('//Plot[@refobj]'):
- ro = x.get('refobj')
- image = doc.xpath('//Image[@objid="%s" and @refstream]'%ro)
- if image:
- imgstr = doc.xpath('//ImageStream[@objid="%s" and @file]'%
- image[0].get('refstream'))
- if imgstr:
- plot_map[ro] = imgstr[0].get('file')
-
- self.log('Converting XML to HTML...')
- styledoc = etree.fromstring(P('templates/lrf.xsl', data=True))
- media_type = MediaType()
- styles = Styles()
- text_block = TextBlock(styles, char_button_map, plot_map, log)
- canvas = Canvas(doc, styles, text_block, log)
- image_block = ImageBlock(canvas)
- ruled_line = RuledLine()
- extensions = {
- ('calibre', 'media-type') : media_type,
- ('calibre', 'text-block') : text_block,
- ('calibre', 'ruled-line') : ruled_line,
- ('calibre', 'styles') : styles,
- ('calibre', 'canvas') : canvas,
- ('calibre', 'image-block'): image_block,
- }
- transform = etree.XSLT(styledoc, extensions=extensions)
- try:
- result = transform(doc)
- except RuntimeError:
- sys.setrecursionlimit(5000)
- result = transform(doc)
-
- with open('content.opf', 'wb') as f:
- f.write(result)
- styles.write()
- return os.path.abspath('content.opf')
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 286bcee9d0..0312a7db6a 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -710,7 +710,7 @@ def fmt(x, y):
fmt('Title sort', self.title_sort)
if self.authors:
fmt('Author(s)', authors_to_string(self.authors) + \
- ((' [' + self.author_sort + ']')
+ ((' [' + self.author_sort + ']')
if self.author_sort and self.author_sort != _('Unknown') else ''))
if self.publisher:
fmt('Publisher', self.publisher)
diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py
index a14e18569a..c0c3900a5d 100644
--- a/src/calibre/ebooks/metadata/book/json_codec.py
+++ b/src/calibre/ebooks/metadata/book/json_codec.py
@@ -12,7 +12,6 @@
from calibre.constants import filesystem_encoding, preferred_encoding
from calibre.library.field_metadata import FieldMetadata
from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE, local_tz
-from calibre.utils.magick import Image
from calibre import isbytestring
# Translate datetimes to and from strings. The string form is the datetime in
@@ -37,6 +36,8 @@ def encode_thumbnail(thumbnail):
'''
Encode the image part of a thumbnail, then return the 3 part tuple
'''
+ from calibre.utils.magick import Image
+
if thumbnail is None:
return None
if not isinstance(thumbnail, (tuple, list)):
diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py
index 30fe53f1a2..477b805ba0 100644
--- a/src/calibre/ebooks/metadata/epub.py
+++ b/src/calibre/ebooks/metadata/epub.py
@@ -129,9 +129,57 @@ def __init__(self, path):
def open(self, path, *args, **kwargs):
return open(os.path.join(self.root, path), *args, **kwargs)
-def get_cover(opf, opf_path, stream, reader=None):
+def render_cover(opf, opf_path, zf, reader=None):
from calibre.ebooks import render_html_svg_workaround
from calibre.utils.logging import default_log
+
+ cpage = opf.first_spine_item()
+ if not cpage:
+ return
+ if reader is not None and reader.encryption_meta.is_encrypted(cpage):
+ return
+
+ with TemporaryDirectory('_epub_meta') as tdir:
+ with CurrentDir(tdir):
+ zf.extractall()
+ opf_path = opf_path.replace('/', os.sep)
+ cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage)
+ if not os.path.exists(cpage):
+ return
+
+ if isosx:
+ # On OS X trying to render a HTML cover which uses embedded
+ # fonts more than once in the same process causes a crash in Qt
+ # so be safe and remove the fonts as well as any @font-face
+ # rules
+ for f in walk('.'):
+ if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'):
+ os.remove(f)
+ ffpat = re.compile(br'@font-face.*?{.*?}',
+ re.DOTALL|re.IGNORECASE)
+ with open(cpage, 'r+b') as f:
+ raw = f.read()
+ f.truncate(0)
+ raw = ffpat.sub(b'', raw)
+ f.write(raw)
+ from calibre.ebooks.chardet import xml_to_unicode
+ raw = xml_to_unicode(raw,
+ strip_encoding_pats=True, resolve_entities=True)[0]
+ from lxml import html
+ for link in html.fromstring(raw).xpath('//link'):
+ href = link.get('href', '')
+ if href:
+ path = os.path.join(os.path.dirname(cpage), href)
+ if os.path.exists(path):
+ with open(path, 'r+b') as f:
+ raw = f.read()
+ f.truncate(0)
+ raw = ffpat.sub(b'', raw)
+ f.write(raw)
+
+ return render_html_svg_workaround(cpage, default_log)
+
+def get_cover(opf, opf_path, stream, reader=None):
raster_cover = opf.raster_cover
stream.seek(0)
zf = ZipFile(stream)
@@ -152,27 +200,7 @@ def get_cover(opf, opf_path, stream, reader=None):
zf.close()
return data
- cpage = opf.first_spine_item()
- if not cpage:
- return
- if reader is not None and reader.encryption_meta.is_encrypted(cpage):
- return
-
- with TemporaryDirectory('_epub_meta') as tdir:
- with CurrentDir(tdir):
- zf.extractall()
- if isosx:
- # On OS X trying to render an HTML cover which uses embedded
- # fonts more than once in the same process causes a crash in Qt
- # so be safe and remove the fonts.
- for f in walk('.'):
- if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'):
- os.remove(f)
- opf_path = opf_path.replace('/', os.sep)
- cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage)
- if not os.path.exists(cpage):
- return
- return render_html_svg_workaround(cpage, default_log)
+ return render_cover(opf, opf_path, zf, reader=reader)
def get_metadata(stream, extract_cover=True):
""" Return metadata as a :class:`Metadata` object """
diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py
index 26b0b6c59b..73ba7e77f4 100644
--- a/src/calibre/ebooks/metadata/meta.py
+++ b/src/calibre/ebooks/metadata/meta.py
@@ -222,6 +222,11 @@ def forked_read_metadata(path, tdir):
from calibre.ebooks.metadata.opf2 import metadata_to_opf
with open(path, 'rb') as f:
fmt = os.path.splitext(path)[1][1:].lower()
+ f.seek(0, 2)
+ sz = f.tell()
+ with open(os.path.join(tdir, 'size.txt'), 'wb') as s:
+ s.write(str(sz).encode('ascii'))
+ f.seek(0)
mi = get_metadata(f, fmt)
if mi.cover_data and mi.cover_data[1]:
with open(os.path.join(tdir, 'cover.jpg'), 'wb') as f:
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index 9b8ae12b10..8d37e95dc4 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -1019,6 +1019,11 @@ def raster_cover(self):
mt = item.get('media-type', '')
if 'xml' not in mt:
return item.get('href', None)
+ for item in self.itermanifest():
+ if item.get('href', None) == cover_id:
+ mt = item.get('media-type', '')
+ if mt.startswith('image/'):
+ return item.get('href', None)
@dynamic_property
def cover(self):
diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py
index cae31abe09..3d08b96c5f 100644
--- a/src/calibre/ebooks/metadata/sources/amazon.py
+++ b/src/calibre/ebooks/metadata/sources/amazon.py
@@ -12,19 +12,14 @@
from threading import Thread
from Queue import Queue, Empty
-from lxml.html import tostring
from calibre import as_unicode
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import (Source, Option, fixcase,
fixauthors)
-from calibre.utils.cleantext import clean_ascii_chars
-from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata.book.base import Metadata
-from calibre.library.comments import sanitize_comments_html
from calibre.utils.date import parse_date
from calibre.utils.localization import canonicalize_lang
-from calibre.utils.soupparser import fromstring
class Worker(Thread): # Get details {{{
@@ -43,6 +38,8 @@ def __init__(self, url, result_queue, browser, log, relevance, domain,
self.browser = browser.clone_browser()
self.cover_url = self.amazon_id = self.isbn = None
self.domain = domain
+ from lxml.html import tostring
+ self.tostring = tostring
months = {
'de': {
@@ -176,6 +173,10 @@ def run(self):
self.log.exception('get_details failed for url: %r'%self.url)
def get_details(self):
+ from calibre.utils.cleantext import clean_ascii_chars
+ from calibre.utils.soupparser import fromstring
+ from calibre.ebooks.chardet import xml_to_unicode
+
try:
raw = self.browser.open_novisit(self.url, timeout=self.timeout).read().strip()
except Exception as e:
@@ -210,7 +211,7 @@ def get_details(self):
errmsg = root.xpath('//*[@id="errorMessage"]')
if errmsg:
msg = 'Failed to parse amazon details page: %r'%self.url
- msg += tostring(errmsg, method='text', encoding=unicode).strip()
+ msg += self.tostring(errmsg, method='text', encoding=unicode).strip()
self.log.error(msg)
return
@@ -322,10 +323,10 @@ def parse_title(self, root):
tdiv = root.xpath('//h1[contains(@class, "parseasinTitle")]')[0]
actual_title = tdiv.xpath('descendant::*[@id="btAsinTitle"]')
if actual_title:
- title = tostring(actual_title[0], encoding=unicode,
+ title = self.tostring(actual_title[0], encoding=unicode,
method='text').strip()
else:
- title = tostring(tdiv, encoding=unicode, method='text').strip()
+ title = self.tostring(tdiv, encoding=unicode, method='text').strip()
return re.sub(r'[(\[].*[)\]]', '', title).strip()
def parse_authors(self, root):
@@ -337,7 +338,7 @@ def parse_authors(self, root):
''')
for x in aname:
x.tail = ''
- authors = [tostring(x, encoding=unicode, method='text').strip() for x
+ authors = [self.tostring(x, encoding=unicode, method='text').strip() for x
in aname]
authors = [a for a in authors if a]
return authors
@@ -356,6 +357,8 @@ def parse_rating(self, root):
return float(m.group(1))/float(m.group(3)) * 5
def parse_comments(self, root):
+ from calibre.library.comments import sanitize_comments_html
+
desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]')
if desc:
desc = desc[0]
@@ -365,7 +368,7 @@ def parse_comments(self, root):
for a in desc.xpath('descendant::a[@href]'):
del a.attrib['href']
a.tag = 'span'
- desc = tostring(desc, method='html', encoding=unicode).strip()
+ desc = self.tostring(desc, method='html', encoding=unicode).strip()
# Encoding bug in Amazon data U+fffd (replacement char)
# in some examples it is present in place of '
@@ -602,6 +605,11 @@ def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
Note this method will retry without identifiers automatically if no
match is found with identifiers.
'''
+ from lxml.html import tostring
+ from calibre.utils.cleantext import clean_ascii_chars
+ from calibre.utils.soupparser import fromstring
+ from calibre.ebooks.chardet import xml_to_unicode
+
query, domain = self.create_query(log, title=title, authors=authors,
identifiers=identifiers)
if query is None:
diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py
index 9ae8902671..4c334f4e46 100644
--- a/src/calibre/ebooks/metadata/sources/base.py
+++ b/src/calibre/ebooks/metadata/sources/base.py
@@ -12,7 +12,6 @@
from calibre import browser, random_user_agent
from calibre.customize import Plugin
-from calibre.utils.logging import ThreadSafeLog, FileStream
from calibre.utils.config import JSONConfig
from calibre.utils.titlecase import titlecase
from calibre.utils.icu import capitalize, lower, upper
@@ -34,6 +33,7 @@
msprefs.defaults['cover_priorities'] = {'Google':2}
def create_log(ostream=None):
+ from calibre.utils.logging import ThreadSafeLog, FileStream
log = ThreadSafeLog(level=ThreadSafeLog.DEBUG)
log.outputs = [FileStream(ostream)]
return log
diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py
index 06e874e8ca..6857d62d4d 100644
--- a/src/calibre/ebooks/metadata/sources/douban.py
+++ b/src/calibre/ebooks/metadata/sources/douban.py
@@ -12,14 +12,10 @@
from functools import partial
from Queue import Queue, Empty
-from lxml import etree
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source
from calibre.ebooks.metadata.book.base import Metadata
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.utils.date import parse_date, utcnow
-from calibre.utils.cleantext import clean_ascii_chars
from calibre import as_unicode
NAMESPACES = {
@@ -28,22 +24,6 @@
'db': 'http://www.douban.com/xmlns/',
'gd': 'http://schemas.google.com/g/2005'
}
-XPath = partial(etree.XPath, namespaces=NAMESPACES)
-total_results = XPath('//openSearch:totalResults')
-start_index = XPath('//openSearch:startIndex')
-items_per_page = XPath('//openSearch:itemsPerPage')
-entry = XPath('//atom:entry')
-entry_id = XPath('descendant::atom:id')
-title = XPath('descendant::atom:title')
-description = XPath('descendant::atom:summary')
-publisher = XPath("descendant::db:attribute[@name='publisher']")
-isbn = XPath("descendant::db:attribute[@name='isbn13']")
-date = XPath("descendant::db:attribute[@name='pubdate']")
-creator = XPath("descendant::db:attribute[@name='author']")
-booktag = XPath("descendant::db:tag/attribute::name")
-rating = XPath("descendant::gd:rating/attribute::average")
-cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
-
def get_details(browser, url, timeout): # {{{
try:
if Douban.DOUBAN_API_KEY and Douban.DOUBAN_API_KEY != '':
@@ -61,6 +41,25 @@ def get_details(browser, url, timeout): # {{{
# }}}
def to_metadata(browser, log, entry_, timeout): # {{{
+ from lxml import etree
+ from calibre.ebooks.chardet import xml_to_unicode
+ from calibre.utils.date import parse_date, utcnow
+ from calibre.utils.cleantext import clean_ascii_chars
+
+ XPath = partial(etree.XPath, namespaces=NAMESPACES)
+ entry = XPath('//atom:entry')
+ entry_id = XPath('descendant::atom:id')
+ title = XPath('descendant::atom:title')
+ description = XPath('descendant::atom:summary')
+ publisher = XPath("descendant::db:attribute[@name='publisher']")
+ isbn = XPath("descendant::db:attribute[@name='isbn13']")
+ date = XPath("descendant::db:attribute[@name='pubdate']")
+ creator = XPath("descendant::db:attribute[@name='author']")
+ booktag = XPath("descendant::db:tag/attribute::name")
+ rating = XPath("descendant::gd:rating/attribute::average")
+ cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
+
+
def get_text(extra, x):
try:
ans = x(extra)
@@ -275,6 +274,7 @@ def get_cached_cover_url(self, identifiers): # {{{
def get_all_details(self, br, log, entries, abort, # {{{
result_queue, timeout):
+ from lxml import etree
for relevance, i in enumerate(entries):
try:
ans = to_metadata(br, log, i, timeout)
@@ -298,6 +298,13 @@ def get_all_details(self, br, log, entries, abort, # {{{
def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
identifiers={}, timeout=30):
+ from lxml import etree
+ from calibre.ebooks.chardet import xml_to_unicode
+ from calibre.utils.cleantext import clean_ascii_chars
+
+ XPath = partial(etree.XPath, namespaces=NAMESPACES)
+ entry = XPath('//atom:entry')
+
query = self.create_query(log, title=title, authors=authors,
identifiers=identifiers)
if not query:
diff --git a/src/calibre/ebooks/metadata/sources/google.py b/src/calibre/ebooks/metadata/sources/google.py
index f9c43d86cc..3962afcb5e 100644
--- a/src/calibre/ebooks/metadata/sources/google.py
+++ b/src/calibre/ebooks/metadata/sources/google.py
@@ -12,8 +12,6 @@
from functools import partial
from Queue import Queue, Empty
-from lxml import etree
-
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source
from calibre.ebooks.metadata.book.base import Metadata
@@ -29,23 +27,6 @@
'dc' : 'http://purl.org/dc/terms',
'gd' : 'http://schemas.google.com/g/2005'
}
-XPath = partial(etree.XPath, namespaces=NAMESPACES)
-
-total_results = XPath('//openSearch:totalResults')
-start_index = XPath('//openSearch:startIndex')
-items_per_page = XPath('//openSearch:itemsPerPage')
-entry = XPath('//atom:entry')
-entry_id = XPath('descendant::atom:id')
-creator = XPath('descendant::dc:creator')
-identifier = XPath('descendant::dc:identifier')
-title = XPath('descendant::dc:title')
-date = XPath('descendant::dc:date')
-publisher = XPath('descendant::dc:publisher')
-subject = XPath('descendant::dc:subject')
-description = XPath('descendant::dc:description')
-language = XPath('descendant::dc:language')
-rating = XPath('descendant::gd:rating[@average]')
-
def get_details(browser, url, timeout): # {{{
try:
raw = browser.open_novisit(url, timeout=timeout).read()
@@ -61,6 +42,24 @@ def get_details(browser, url, timeout): # {{{
# }}}
def to_metadata(browser, log, entry_, timeout): # {{{
+ from lxml import etree
+ XPath = partial(etree.XPath, namespaces=NAMESPACES)
+
+ # total_results = XPath('//openSearch:totalResults')
+ # start_index = XPath('//openSearch:startIndex')
+ # items_per_page = XPath('//openSearch:itemsPerPage')
+ entry = XPath('//atom:entry')
+ entry_id = XPath('descendant::atom:id')
+ creator = XPath('descendant::dc:creator')
+ identifier = XPath('descendant::dc:identifier')
+ title = XPath('descendant::dc:title')
+ date = XPath('descendant::dc:date')
+ publisher = XPath('descendant::dc:publisher')
+ subject = XPath('descendant::dc:subject')
+ description = XPath('descendant::dc:description')
+ language = XPath('descendant::dc:language')
+ rating = XPath('descendant::gd:rating[@average]')
+
def get_text(extra, x):
try:
@@ -266,6 +265,7 @@ def get_cached_cover_url(self, identifiers): # {{{
def get_all_details(self, br, log, entries, abort, # {{{
result_queue, timeout):
+ from lxml import etree
for relevance, i in enumerate(entries):
try:
ans = to_metadata(br, log, i, timeout)
@@ -289,6 +289,10 @@ def get_all_details(self, br, log, entries, abort, # {{{
def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
identifiers={}, timeout=30):
+ from lxml import etree
+ XPath = partial(etree.XPath, namespaces=NAMESPACES)
+ entry = XPath('//atom:entry')
+
query = self.create_query(log, title=title, authors=authors,
identifiers=identifiers)
if not query:
diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py
index 1da7f906bb..7e15ad275e 100644
--- a/src/calibre/ebooks/metadata/sources/isbndb.py
+++ b/src/calibre/ebooks/metadata/sources/isbndb.py
@@ -9,12 +9,9 @@
from urllib import quote
-from lxml import etree
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source, Option
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.utils.cleantext import clean_ascii_chars
from calibre.utils.icu import lower
from calibre.ebooks.metadata.book.base import Metadata
@@ -122,6 +119,7 @@ def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
result_queue.put(result)
def parse_feed(self, feed, seen, orig_title, orig_authors, identifiers):
+ from lxml import etree
def tostring(x):
if x is None:
@@ -198,6 +196,10 @@ def ismatch(title, authors):
def make_query(self, q, abort, title=None, authors=None, identifiers={},
max_pages=10, timeout=30):
+ from lxml import etree
+ from calibre.ebooks.chardet import xml_to_unicode
+ from calibre.utils.cleantext import clean_ascii_chars
+
page_num = 1
parser = etree.XMLParser(recover=True, no_network=True)
br = self.browser
diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py
index 1164567ff5..bb1bbb9d42 100755
--- a/src/calibre/ebooks/metadata/sources/overdrive.py
+++ b/src/calibre/ebooks/metadata/sources/overdrive.py
@@ -9,18 +9,14 @@
'''
Fetch metadata using Overdrive Content Reserve
'''
-import re, random, mechanize, copy, json
+import re, random, copy, json
from threading import RLock
from Queue import Queue, Empty
-from lxml import html
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source, Option
from calibre.ebooks.metadata.book.base import Metadata
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.library.comments import sanitize_comments_html
-from calibre.utils.soupparser import fromstring
ovrdrv_data_cache = {}
cache_lock = RLock()
@@ -80,6 +76,7 @@ def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
def download_cover(self, log, result_queue, abort, # {{{
title=None, authors=None, identifiers={}, timeout=30):
+ import mechanize
cached_url = self.get_cached_cover_url(identifiers)
if cached_url is None:
log.info('No cached cover found, running identify')
@@ -170,6 +167,7 @@ def safe_query(self, br, query_url, post=''):
this page attempts to set a cookie that Mechanize doesn't like
copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar
'''
+ import mechanize
goodcookies = br._ua_handlers['_cookies'].cookiejar
clean_cj = mechanize.CookieJar()
cookies_to_copy = []
@@ -187,6 +185,7 @@ def safe_query(self, br, query_url, post=''):
br.set_cookiejar(clean_cj)
def overdrive_search(self, br, log, q, title, author):
+ import mechanize
# re-initialize the cookiejar to so that it's clean
clean_cj = mechanize.CookieJar()
br.set_cookiejar(clean_cj)
@@ -303,6 +302,7 @@ def sort_ovrdrv_results(self, raw, log, title=None, title_tokens=None, author=No
return ''
def overdrive_get_record(self, br, log, q, ovrdrv_id):
+ import mechanize
search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}'
results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc'
@@ -393,6 +393,11 @@ def parse_search_results(self, ovrdrv_data, mi):
def get_book_detail(self, br, metadata_url, mi, ovrdrv_id, log):
+ from lxml import html
+ from calibre.ebooks.chardet import xml_to_unicode
+ from calibre.utils.soupparser import fromstring
+ from calibre.library.comments import sanitize_comments_html
+
try:
raw = br.open_novisit(metadata_url).read()
except Exception, e:
diff --git a/src/calibre/ebooks/metadata/sources/ozon.py b/src/calibre/ebooks/metadata/sources/ozon.py
index de45e0b8db..d40e43d582 100644
--- a/src/calibre/ebooks/metadata/sources/ozon.py
+++ b/src/calibre/ebooks/metadata/sources/ozon.py
@@ -6,15 +6,11 @@
__docformat__ = 'restructuredtext en'
import re
-import urllib2
import datetime
from urllib import quote_plus
from Queue import Queue, Empty
-from lxml import etree, html
+
from calibre import as_unicode
-
-from calibre.ebooks.chardet import xml_to_unicode
-
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.sources.base import Source
from calibre.ebooks.metadata.book.base import Metadata
@@ -43,6 +39,7 @@ class Ozon(Source):
isbnRegex = re.compile(isbnPattern)
def get_book_url(self, identifiers): # {{{
+ import urllib2
ozon_id = identifiers.get('ozon', None)
res = None
if ozon_id:
@@ -81,6 +78,9 @@ def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
def identify(self, log, result_queue, abort, title=None, authors=None,
identifiers={}, timeout=30): # {{{
+ from lxml import etree
+ from calibre.ebooks.chardet import xml_to_unicode
+
if not self.is_configured():
return
query = self.create_query(log, title=title, authors=authors, identifiers=identifiers)
@@ -283,6 +283,9 @@ def download_cover(self, log, result_queue, abort, title=None, authors=None, ide
# }}}
def get_book_details(self, log, metadata, timeout): # {{{
+ from lxml import html, etree
+ from calibre.ebooks.chardet import xml_to_unicode
+
url = self.get_book_url(metadata.get_identifiers())[2]
raw = self.browser.open_novisit(url, timeout=timeout).read()
diff --git a/src/calibre/ebooks/odt/input.py b/src/calibre/ebooks/odt/input.py
index 214a40c29b..430d95b31f 100644
--- a/src/calibre/ebooks/odt/input.py
+++ b/src/calibre/ebooks/odt/input.py
@@ -12,7 +12,6 @@
from odf.odf2xhtml import ODF2XHTML
from calibre import CurrentDir, walk
-from calibre.customize.conversion import InputFormatPlugin
class Extract(ODF2XHTML):
@@ -178,16 +177,4 @@ def __call__(self, stream, odir, log):
return os.path.abspath('metadata.opf')
-class ODTInput(InputFormatPlugin):
-
- name = 'ODT Input'
- author = 'Kovid Goyal'
- description = 'Convert ODT (OpenOffice) files to HTML'
- file_types = set(['odt'])
-
-
- def convert(self, stream, options, file_ext, log,
- accelerators):
- return Extract()(stream, '.', log)
-
diff --git a/src/calibre/ebooks/pdb/__init__.py b/src/calibre/ebooks/pdb/__init__.py
index c8089297db..428cbe82ab 100644
--- a/src/calibre/ebooks/pdb/__init__.py
+++ b/src/calibre/ebooks/pdb/__init__.py
@@ -7,31 +7,38 @@
class PDBError(Exception):
pass
+FORMAT_READERS = None
-from calibre.ebooks.pdb.ereader.reader import Reader as ereader_reader
-from calibre.ebooks.pdb.palmdoc.reader import Reader as palmdoc_reader
-from calibre.ebooks.pdb.ztxt.reader import Reader as ztxt_reader
-from calibre.ebooks.pdb.pdf.reader import Reader as pdf_reader
-from calibre.ebooks.pdb.plucker.reader import Reader as plucker_reader
+def _import_readers():
+ global FORMAT_READERS
+ from calibre.ebooks.pdb.ereader.reader import Reader as ereader_reader
+ from calibre.ebooks.pdb.palmdoc.reader import Reader as palmdoc_reader
+ from calibre.ebooks.pdb.ztxt.reader import Reader as ztxt_reader
+ from calibre.ebooks.pdb.pdf.reader import Reader as pdf_reader
+ from calibre.ebooks.pdb.plucker.reader import Reader as plucker_reader
-FORMAT_READERS = {
- 'PNPdPPrs': ereader_reader,
- 'PNRdPPrs': ereader_reader,
- 'zTXTGPlm': ztxt_reader,
- 'TEXtREAd': palmdoc_reader,
- '.pdfADBE': pdf_reader,
- 'DataPlkr': plucker_reader,
-}
+ FORMAT_READERS = {
+ 'PNPdPPrs': ereader_reader,
+ 'PNRdPPrs': ereader_reader,
+ 'zTXTGPlm': ztxt_reader,
+ 'TEXtREAd': palmdoc_reader,
+ '.pdfADBE': pdf_reader,
+ 'DataPlkr': plucker_reader,
+ }
-from calibre.ebooks.pdb.palmdoc.writer import Writer as palmdoc_writer
-from calibre.ebooks.pdb.ztxt.writer import Writer as ztxt_writer
-from calibre.ebooks.pdb.ereader.writer import Writer as ereader_writer
+ALL_FORMAT_WRITERS = {'doc', 'ztxt', 'ereader'}
+FORMAT_WRITERS = None
+def _import_writers():
+ global FORMAT_WRITERS
+ from calibre.ebooks.pdb.palmdoc.writer import Writer as palmdoc_writer
+ from calibre.ebooks.pdb.ztxt.writer import Writer as ztxt_writer
+ from calibre.ebooks.pdb.ereader.writer import Writer as ereader_writer
-FORMAT_WRITERS = {
- 'doc': palmdoc_writer,
- 'ztxt': ztxt_writer,
- 'ereader': ereader_writer,
-}
+ FORMAT_WRITERS = {
+ 'doc': palmdoc_writer,
+ 'ztxt': ztxt_writer,
+ 'ereader': ereader_writer,
+ }
IDENTITY_TO_NAME = {
'PNPdPPrs': 'eReader',
@@ -69,11 +76,17 @@ def get_reader(identity):
'''
Returns None if no reader is found for the identity.
'''
+ global FORMAT_READERS
+ if FORMAT_READERS is None:
+ _import_readers()
return FORMAT_READERS.get(identity, None)
def get_writer(extension):
'''
Returns None if no writer is found for extension.
'''
+ global FORMAT_WRITERS
+ if FORMAT_WRITERS is None:
+ _import_writers()
return FORMAT_WRITERS.get(extension, None)
diff --git a/src/calibre/ebooks/rb/reader.py b/src/calibre/ebooks/rb/reader.py
index f97c3d78c5..e68cef41d3 100644
--- a/src/calibre/ebooks/rb/reader.py
+++ b/src/calibre/ebooks/rb/reader.py
@@ -65,7 +65,7 @@ def get_toc(self):
name = urlunquote(self.stream.read(32).strip('\x00'))
size, offset, flags = self.read_i32(), self.read_i32(), self.read_i32()
toc.append(RBToc.Item(name=name, size=size, offset=offset, flags=flags))
-
+
return toc
def get_text(self, toc_item, output_dir):
@@ -89,7 +89,7 @@ def get_text(self, toc_item, output_dir):
output += self.stream.read(toc_item.size).decode('cp1252' if self.encoding is None else self.encoding, 'replace')
with open(os.path.join(output_dir, toc_item.name), 'wb') as html:
- html.write(output.encode('utf-8'))
+ html.write(output.replace('', '<TITLE> ').encode('utf-8'))
def get_image(self, toc_item, output_dir):
if toc_item.flags != 0:
@@ -105,7 +105,7 @@ def extract_content(self, output_dir):
self.log.debug('Extracting content from file...')
html = []
images = []
-
+
for item in self.toc:
if item.name.lower().endswith('html'):
self.log.debug('HTML item %s found...' % item.name)
diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py
index 5858824434..8e1a5ac775 100644
--- a/src/calibre/ebooks/rtf/input.py
+++ b/src/calibre/ebooks/rtf/input.py
@@ -2,42 +2,9 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import os, glob, re, textwrap
from lxml import etree
-from calibre.customize.conversion import InputFormatPlugin
-
-border_style_map = {
- 'single' : 'solid',
- 'double-thickness-border' : 'double',
- 'shadowed-border': 'outset',
- 'double-border': 'double',
- 'dotted-border': 'dotted',
- 'dashed': 'dashed',
- 'hairline': 'solid',
- 'inset': 'inset',
- 'dash-small': 'dashed',
- 'dot-dash': 'dotted',
- 'dot-dot-dash': 'dotted',
- 'outset': 'outset',
- 'tripple': 'double',
- 'triple': 'double',
- 'thick-thin-small': 'solid',
- 'thin-thick-small': 'solid',
- 'thin-thick-thin-small': 'solid',
- 'thick-thin-medium': 'solid',
- 'thin-thick-medium': 'solid',
- 'thin-thick-thin-medium': 'solid',
- 'thick-thin-large': 'solid',
- 'thin-thick-thin-large': 'solid',
- 'wavy': 'ridge',
- 'double-wavy': 'ridge',
- 'striped': 'ridge',
- 'emboss': 'inset',
- 'engrave': 'inset',
- 'frame': 'ridge',
-}
class InlineClass(etree.XSLTExtension):
@@ -71,261 +38,3 @@ def execute(self, context, self_node, input_node, output_parent):
output_parent.text = ' '.join(classes)
-class RTFInput(InputFormatPlugin):
-
- name = 'RTF Input'
- author = 'Kovid Goyal'
- description = 'Convert RTF files to HTML'
- file_types = set(['rtf'])
-
- def generate_xml(self, stream):
- from calibre.ebooks.rtf2xml.ParseRtf import ParseRtf
- ofile = 'dataxml.xml'
- run_lev, debug_dir, indent_out = 1, None, 0
- if getattr(self.opts, 'debug_pipeline', None) is not None:
- try:
- os.mkdir('rtfdebug')
- debug_dir = 'rtfdebug'
- run_lev = 4
- indent_out = 1
- self.log('Running RTFParser in debug mode')
- except:
- self.log.warn('Impossible to run RTFParser in debug mode')
- parser = ParseRtf(
- in_file = stream,
- out_file = ofile,
- # Convert symbol fonts to unicode equivalents. Default
- # is 1
- convert_symbol = 1,
-
- # Convert Zapf fonts to unicode equivalents. Default
- # is 1.
- convert_zapf = 1,
-
- # Convert Wingding fonts to unicode equivalents.
- # Default is 1.
- convert_wingdings = 1,
-
- # Convert RTF caps to real caps.
- # Default is 1.
- convert_caps = 1,
-
- # Indent resulting XML.
- # Default is 0 (no indent).
- indent = indent_out,
-
- # Form lists from RTF. Default is 1.
- form_lists = 1,
-
- # Convert headings to sections. Default is 0.
- headings_to_sections = 1,
-
- # Group paragraphs with the same style name. Default is 1.
- group_styles = 1,
-
- # Group borders. Default is 1.
- group_borders = 1,
-
- # Write or do not write paragraphs. Default is 0.
- empty_paragraphs = 1,
-
- #debug
- deb_dir = debug_dir,
- run_level = run_lev,
- )
- parser.parse_rtf()
- with open(ofile, 'rb') as f:
- return f.read()
-
- def extract_images(self, picts):
- import imghdr
- self.log('Extracting images...')
-
- with open(picts, 'rb') as f:
- raw = f.read()
- picts = filter(len, re.findall(r'\{\\pict([^}]+)\}', raw))
- hex = re.compile(r'[^a-fA-F0-9]')
- encs = [hex.sub('', pict) for pict in picts]
-
- count = 0
- imap = {}
- for enc in encs:
- if len(enc) % 2 == 1:
- enc = enc[:-1]
- data = enc.decode('hex')
- fmt = imghdr.what(None, data)
- if fmt is None:
- fmt = 'wmf'
- count += 1
- name = '%04d.%s' % (count, fmt)
- with open(name, 'wb') as f:
- f.write(data)
- imap[count] = name
- # with open(name+'.hex', 'wb') as f:
- # f.write(enc)
- return self.convert_images(imap)
-
- def convert_images(self, imap):
- self.default_img = None
- for count, val in imap.iteritems():
- try:
- imap[count] = self.convert_image(val)
- except:
- self.log.exception('Failed to convert', val)
- return imap
-
- def convert_image(self, name):
- if not name.endswith('.wmf'):
- return name
- try:
- return self.rasterize_wmf(name)
- except:
- self.log.exception('Failed to convert WMF image %r'%name)
- return self.replace_wmf(name)
-
- def replace_wmf(self, name):
- from calibre.ebooks import calibre_cover
- if self.default_img is None:
- self.default_img = calibre_cover('Conversion of WMF images is not supported',
- 'Use Microsoft Word or OpenOffice to save this RTF file'
- ' as HTML and convert that in calibre.', title_size=36,
- author_size=20)
- name = name.replace('.wmf', '.jpg')
- with open(name, 'wb') as f:
- f.write(self.default_img)
- return name
-
- def rasterize_wmf(self, name):
- from calibre.utils.wmf.parse import wmf_unwrap
- with open(name, 'rb') as f:
- data = f.read()
- data = wmf_unwrap(data)
- name = name.replace('.wmf', '.png')
- with open(name, 'wb') as f:
- f.write(data)
- return name
-
-
- def write_inline_css(self, ic, border_styles):
- font_size_classes = ['span.fs%d { font-size: %spt }'%(i, x) for i, x in
- enumerate(ic.font_sizes)]
- color_classes = ['span.col%d { color: %s }'%(i, x) for i, x in
- enumerate(ic.colors)]
- css = textwrap.dedent('''
- span.none {
- text-decoration: none; font-weight: normal;
- font-style: normal; font-variant: normal
- }
-
- span.italics { font-style: italic }
-
- span.bold { font-weight: bold }
-
- span.small-caps { font-variant: small-caps }
-
- span.underlined { text-decoration: underline }
-
- span.strike-through { text-decoration: line-through }
-
- ''')
- css += '\n'+'\n'.join(font_size_classes)
- css += '\n' +'\n'.join(color_classes)
-
- for cls, val in border_styles.iteritems():
- css += '\n\n.%s {\n%s\n}'%(cls, val)
-
- with open('styles.css', 'ab') as f:
- f.write(css)
-
- def convert_borders(self, doc):
- border_styles = []
- style_map = {}
- for elem in doc.xpath(r'//*[local-name()="cell"]'):
- style = ['border-style: hidden', 'border-width: 1px',
- 'border-color: black']
- for x in ('bottom', 'top', 'left', 'right'):
- bs = elem.get('border-cell-%s-style'%x, None)
- if bs:
- cbs = border_style_map.get(bs, 'solid')
- style.append('border-%s-style: %s'%(x, cbs))
- bw = elem.get('border-cell-%s-line-width'%x, None)
- if bw:
- style.append('border-%s-width: %spt'%(x, bw))
- bc = elem.get('border-cell-%s-color'%x, None)
- if bc:
- style.append('border-%s-color: %s'%(x, bc))
- style = ';\n'.join(style)
- if style not in border_styles:
- border_styles.append(style)
- idx = border_styles.index(style)
- cls = 'border_style%d'%idx
- style_map[cls] = style
- elem.set('class', cls)
- return style_map
-
- def convert(self, stream, options, file_ext, log,
- accelerators):
- from calibre.ebooks.metadata.meta import get_metadata
- from calibre.ebooks.metadata.opf2 import OPFCreator
- from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException
- self.opts = options
- self.log = log
- self.log('Converting RTF to XML...')
- try:
- xml = self.generate_xml(stream.name)
- except RtfInvalidCodeException as e:
- raise ValueError(_('This RTF file has a feature calibre does not '
- 'support. Convert it to HTML first and then try it.\n%s')%e)
-
- d = glob.glob(os.path.join('*_rtf_pict_dir', 'picts.rtf'))
- if d:
- imap = {}
- try:
- imap = self.extract_images(d[0])
- except:
- self.log.exception('Failed to extract images...')
-
- self.log('Parsing XML...')
- parser = etree.XMLParser(recover=True, no_network=True)
- doc = etree.fromstring(xml, parser=parser)
- border_styles = self.convert_borders(doc)
- for pict in doc.xpath('//rtf:pict[@num]',
- namespaces={'rtf':'http://rtf2xml.sourceforge.net/'}):
- num = int(pict.get('num'))
- name = imap.get(num, None)
- if name is not None:
- pict.set('num', name)
-
- self.log('Converting XML to HTML...')
- inline_class = InlineClass(self.log)
- styledoc = etree.fromstring(P('templates/rtf.xsl', data=True))
- extensions = { ('calibre', 'inline-class') : inline_class }
- transform = etree.XSLT(styledoc, extensions=extensions)
- result = transform(doc)
- html = 'index.xhtml'
- with open(html, 'wb') as f:
- res = transform.tostring(result)
- # res = res[:100].replace('xmlns:html', 'xmlns') + res[100:]
- #clean multiple \n
- res = re.sub('\n+', '\n', res)
- # Replace newlines inserted by the 'empty_paragraphs' option in rtf2xml with html blank lines
- # res = re.sub('\s*<body>', '<body>', res)
- # res = re.sub('(?<=\n)\n{2}',
- # u'<p>\u00a0</p>\n'.encode('utf-8'), res)
- f.write(res)
- self.write_inline_css(inline_class, border_styles)
- stream.seek(0)
- mi = get_metadata(stream, 'rtf')
- if not mi.title:
- mi.title = _('Unknown')
- if not mi.authors:
- mi.authors = [_('Unknown')]
- opf = OPFCreator(os.getcwd(), mi)
- opf.create_manifest([('index.xhtml', None)])
- opf.create_spine(['index.xhtml'])
- opf.render(open('metadata.opf', 'wb'))
- return os.path.abspath('metadata.opf')
-
-#ebook-convert "bad.rtf" test.epub -v -d "E:\Mes eBooks\Developpement\debug"
-# os.makedirs("E:\\Mes eBooks\\Developpement\\rtfdebug")
-# debug_dir = "E:\\Mes eBooks\\Developpement\\rtfdebug"
diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index 4cff648fa5..0880eca4ca 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -16,7 +16,7 @@
from calibre.ebooks.conversion.preprocess import DocAnalysis
from calibre.utils.cleantext import clean_ascii_chars
-HTML_TEMPLATE = u'<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>%s\n%s\n'
+HTML_TEMPLATE = u'%s \n%s\n'
def clean_txt(txt):
'''
@@ -28,7 +28,7 @@ def clean_txt(txt):
# Strip whitespace from the end of the line. Also replace
# all line breaks with \n.
txt = '\n'.join([line.rstrip() for line in txt.splitlines()])
-
+
# Replace whitespace at the beginning of the line with
txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt)
@@ -75,7 +75,7 @@ def convert_basic(txt, title='', epub_split_size_kb=0):
'''
Converts plain text to html by putting all paragraphs in
tags. It condense and retains blank lines when necessary.
-
+
Requires paragraphs to be in single line format.
'''
txt = clean_txt(txt)
@@ -215,7 +215,7 @@ def detect_paragraph_type(txt):
def detect_formatting_type(txt):
'''
Tries to determine the formatting of the document.
-
+
markdown: Markdown formatting is used.
textile: Textile formatting is used.
heuristic: When none of the above formatting types are
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 35cc249acb..b3e128af82 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -102,6 +102,7 @@
gprefs.defaults['worker_max_time'] = 0
gprefs.defaults['show_files_after_save'] = True
gprefs.defaults['auto_add_path'] = None
+gprefs.defaults['auto_add_check_for_duplicates'] = False
# }}}
NONE = QVariant() #: Null value to return from the data function of item models
diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py
index 7cdac3b845..972ea57cb9 100644
--- a/src/calibre/gui2/add.py
+++ b/src/calibre/gui2/add.py
@@ -382,7 +382,8 @@ def process_duplicates(self):
if not duplicates:
return self.duplicates_processed()
self.pd.hide()
- files = [x[0].title for x in duplicates]
+ files = [_('%s by %s')%(x[0].title, x[0].format_field('authors')[1])
+ for x in duplicates]
if question_dialog(self._parent, _('Duplicates found!'),
_('Books with the same title as the following already '
'exist in the database. Add them anyway?'),
diff --git a/src/calibre/gui2/auto_add.py b/src/calibre/gui2/auto_add.py
index 71d2b8ecd0..6860f386d6 100644
--- a/src/calibre/gui2/auto_add.py
+++ b/src/calibre/gui2/auto_add.py
@@ -7,7 +7,7 @@
__copyright__ = '2012, Kovid Goyal '
__docformat__ = 'restructuredtext en'
-import os, tempfile, shutil
+import os, tempfile, shutil, time
from threading import Thread, Event
from PyQt4.Qt import (QFileSystemWatcher, QObject, Qt, pyqtSignal, QTimer)
@@ -15,6 +15,7 @@
from calibre import prints
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.gui2 import question_dialog, gprefs
class Worker(Thread):
@@ -41,25 +42,58 @@ def run(self):
traceback.print_exc()
def auto_add(self):
- from calibre.utils.ipc.simple_worker import fork_job
+ from calibre.utils.ipc.simple_worker import fork_job, WorkerError
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.metadata.meta import metadata_from_filename
- files = [x for x in os.listdir(self.path) if x not in self.staging
- and os.path.isfile(os.path.join(self.path, x)) and
- os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) and
- os.path.splitext(x)[1][1:].lower() in self.be]
+ files = [x for x in os.listdir(self.path) if
+ # Must not be in the process of being added to the db
+ x not in self.staging
+ # Firefox creates 0 byte placeholder files when downloading
+ and os.stat(os.path.join(self.path, x)).st_size > 0
+ # Must be a file
+ and os.path.isfile(os.path.join(self.path, x))
+ # Must have read and write permissions
+ and os.access(os.path.join(self.path, x), os.R_OK|os.W_OK)
+ # Must be a known ebook file type
+ and os.path.splitext(x)[1][1:].lower() in self.be
+ ]
data = {}
+ # Give any in progress copies time to complete
+ time.sleep(2)
+
for fname in files:
f = os.path.join(self.path, fname)
+
+ # Try opening the file for reading, if the OS prevents us, then at
+ # least on windows, it means the file is open in another
+ # application for writing. We will get notified by
+ # QFileSystemWatcher when writing is completed, so ignore for now.
+ try:
+ open(f, 'rb').close()
+ except:
+ continue
tdir = tempfile.mkdtemp(dir=self.tdir)
try:
fork_job('calibre.ebooks.metadata.meta',
'forked_read_metadata', (f, tdir), no_output=True)
+ except WorkerError as e:
+ prints('Failed to read metadata from:', fname)
+ prints(e.orig_tb)
except:
import traceback
traceback.print_exc()
+ # Ensure that the pre-metadata file size is present. If it isn't,
+ # write 0 so that the file is rescanned
+ szpath = os.path.join(tdir, 'size.txt')
+ try:
+ with open(szpath, 'rb') as f:
+ int(f.read())
+ except:
+ with open(szpath, 'wb') as f:
+ f.write(b'0')
+
opfpath = os.path.join(tdir, 'metadata.opf')
try:
if os.stat(opfpath).st_size < 30:
@@ -125,25 +159,71 @@ def add_to_db(self, data):
m = gui.library_view.model()
count = 0
+ needs_rescan = False
+ duplicates = []
+
for fname, tdir in data.iteritems():
paths = [os.path.join(self.worker.path, fname)]
+ sz = os.path.join(tdir, 'size.txt')
+ try:
+ with open(sz, 'rb') as f:
+ sz = int(f.read())
+ if sz != os.stat(paths[0]).st_size:
+ raise Exception('Looks like the file was written to after'
+ ' we tried to read metadata')
+ except:
+ needs_rescan = True
+ try:
+ self.worker.staging.remove(fname)
+ except KeyError:
+ pass
+
+ continue
+
mi = os.path.join(tdir, 'metadata.opf')
if not os.access(mi, os.R_OK):
continue
mi = [OPF(open(mi, 'rb'), tdir,
populate_spine=False).to_book_metadata()]
- m.add_books(paths, [os.path.splitext(fname)[1][1:].upper()], mi,
- add_duplicates=True)
+ dups, num = m.add_books(paths,
+ [os.path.splitext(fname)[1][1:].upper()], mi,
+ add_duplicates=not gprefs['auto_add_check_for_duplicates'])
+ if dups:
+ path = dups[0][0]
+ with open(os.path.join(tdir, 'dup_cache.'+dups[1][0].lower()),
+ 'wb') as dest, open(path, 'rb') as src:
+ shutil.copyfileobj(src, dest)
+ dups[0][0] = dest.name
+ duplicates.append(dups)
+
+ try:
+ os.remove(paths[0])
+ self.worker.staging.remove(fname)
+ except:
+ pass
+ count += num
+
+ if duplicates:
+ paths, formats, metadata = [], [], []
+ for p, f, mis in duplicates:
+ paths.extend(p)
+ formats.extend(f)
+ metadata.extend(mis)
+ files = [_('%s by %s')%(mi.title, mi.format_field('authors')[1])
+ for mi in metadata]
+ if question_dialog(self.parent(), _('Duplicates found!'),
+ _('Books with the same title as the following already '
+ 'exist in the database. Add them anyway?'),
+ '\n'.join(files)):
+ dups, num = m.add_books(paths, formats, metadata,
+ add_duplicates=True)
+ count += num
+
+ for tdir in data.itervalues():
try:
- os.remove(os.path.join(self.worker.path, fname))
- try:
- self.worker.staging.remove(fname)
- except KeyError:
- pass
shutil.rmtree(tdir)
except:
pass
- count += 1
if count > 0:
m.books_added(count)
@@ -153,4 +233,7 @@ def add_to_db(self, data):
if hasattr(gui, 'db_images'):
gui.db_images.reset()
+ if needs_rescan:
+ QTimer.singleShot(2000, self.dir_changed)
+
diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py
index 6ad1aaf0c4..628f846aea 100644
--- a/src/calibre/gui2/book_details.py
+++ b/src/calibre/gui2/book_details.py
@@ -38,14 +38,24 @@ def color_to_string(col):
ans = unicode(col.name())
return ans
- f = QFontInfo(QApplication.font(widget)).pixelSize()
+ fi = QFontInfo(QApplication.font(widget))
+ f = fi.pixelSize()+1
+ fam = unicode(fi.family()).strip().replace('"', '')
+ if not fam:
+ fam = 'sans-serif'
+
c = color_to_string(QApplication.palette().color(QPalette.Normal,
QPalette.WindowText))
templ = u'''\
-