Backup_Repos/calibre
1
0
Fork 0
mirror of git://github.com/kovidgoyal/calibre.git synced 2025-12-23 23:15:20 +01:00
Code Issues Projects Releases Wiki Activity

Sync to trunk.

Browse source
This commit is contained in:
John Schember 2011-09-27 06:45:18 -04:00
commit 18d3044147
15 changed files with 141 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
recipes/den_of_geek.recipe Normal file
View file

@ -0,0 +1,21 @@
from calibre.web.feeds.news import BasicNewsRecipe
class AdvancedUserRecipe1316944753(BasicNewsRecipe):
title = u'Den of Geek'
__author__ = 'Jaded'
language = 'en'
description = 'From science fiction enthusiasts through to gaming fanatics, Den of Geek has become the one-stop UK website for people genuinely passionate about their entertainment media. Den of Geek covers popular culture but always with an edgy, UK centric slant that sets it apart from the crowd.'
category = 'Movies, TV, Games, Comics, Cult, News, Reviews'
language = 'en'
oldest_article = 7
max_articles_per_feed = 100
auto_cleanup = True
no_stylesheets = True
use_embedded_content = True
publication_type = 'newsportal'
masthead_url ='http://www.denofgeek.com/siteimage/scale/0/0/logo.gif'
cover_url ='http://a5.sphotos.ak.fbcdn.net/hphotos-ak-snc6/166479_180131695357862_139191826118516_354818_4993703_n.jpg'
feeds = [(u'Movies', u'http://www.denofgeek.com/movies/rss/'), (u'TV', u'http://www.denofgeek.com/television/rss/'), (u'Comics & Books', u'http://www.denofgeek.com/comics/rss/'), (u'Games', u'http://www.denofgeek.com/games/rss/'), (u'DVD/Blu-ray', u'http://www.denofgeek.com/Reviews/rss/')]

BIN
recipes/icons/den_of_geek.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1 KiB

1
src/calibre/db/__init__.py
View file

@ -7,6 +7,7 @@
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
SPOOL_SIZE = 30*1024*1024
'''
Rewrite of the calibre database backend.

9
src/calibre/db/backend.py
View file

@ -8,7 +8,7 @@
__docformat__ = 'restructuredtext en'
# Imports {{{
import os, shutil, uuid, json, glob, time, tempfile
import os, shutil, uuid, json, glob, time
from functools import partial
import apsw
@ -16,7 +16,8 @@
from calibre import isbytestring, force_unicode, prints
from calibre.constants import (iswindows, filesystem_encoding,
preferred_encoding)
from calibre.ptempfile import PersistentTemporaryFile
from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile
from calibre.db import SPOOL_SIZE
from calibre.db.schema_upgrades import SchemaUpgrade
from calibre.library.field_metadata import FieldMetadata
from calibre.ebooks.metadata import title_sort, author_to_author_sort
@ -38,7 +39,7 @@
'''
SPOOL_SIZE = 30*1024*1024
class DynamicFilter(object): # {{{
@ -805,7 +806,7 @@ def cover(self, path, as_file=False, as_image=False,
shutil.copyfileobj(f, pt)
return pt.name
if as_file:
ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
ret = SpooledTemporaryFile(SPOOL_SIZE)
shutil.copyfileobj(f, ret)
ret.seek(0)
else:

4
src/calibre/devices/kobo/driver.py
View file

@ -51,11 +51,11 @@ class KOBO(USBMS):
EXTRA_CUSTOMIZATION_MESSAGE = [
_('The Kobo supports several collections including ')+\
'Read, Closed, Im_Reading ' +\
'Read, Closed, Im_Reading. ' +\
_('Create tags for automatic management'),
]
EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['tags'])
EXTRA_CUSTOMIZATION_DEFAULT = [', '.join(['tags'])]
OPT_COLLECTIONS = 0

35
src/calibre/ebooks/metadata/sources/amazon.py
View file

@ -73,6 +73,20 @@ def __init__(self, url, result_queue, browser, log, relevance, domain, plugin, t
8: ['août'],
9: ['sept'],
12: ['déc'],
},
'es': {
1: ['enero'],
2: ['febrero'],
3: ['marzo'],
4: ['abril'],
5: ['mayo'],
6: ['junio'],
7: ['julio'],
8: ['agosto'],
9: ['septiembre', 'setiembre'],
10: ['octubre'],
11: ['noviembre'],
12: ['diciembre'],
},
'jp': {
1: [u'1月'],
@ -101,13 +115,16 @@ def __init__(self, url, result_queue, browser, log, relevance, domain, plugin, t
text()="Dettagli prodotto" or \
text()="Product details" or \
text()="Détails sur le produit" or \
text()="Detalles del producto" or \
text()="登録情報"]/../div[@class="content"]
'''
# Editor: is for Spanish
self.publisher_xpath = '''
descendant::*[starts-with(text(), "Publisher:") or \
starts-with(text(), "Verlag:") or \
starts-with(text(), "Editore:") or \
starts-with(text(), "Editeur") or \
starts-with(text(), "Editor:") or \
starts-with(text(), "出版社:")]
'''
self.language_xpath = '''
@ -116,12 +133,14 @@ def __init__(self, url, result_queue, browser, log, relevance, domain, plugin, t
or text() = "Language" \
or text() = "Sprache:" \
or text() = "Lingua:" \
or text() = "Idioma:" \
or starts-with(text(), "Langue") \
or starts-with(text(), "言語") \
]
'''
self.ratings_pat = re.compile(
r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち) ([\d\.]+)( (stars|Sternen|stelle)){0,1}')
r'([0-9.]+) ?(out of|von|su|étoiles sur|つ星のうち|de un máximo de) ([\d\.]+)( (stars|Sternen|stelle|estrellas)){0,1}')
lm = {
'eng': ('English', 'Englisch'),
@ -143,6 +162,7 @@ def delocalize_datestr(self, raw):
for i, vals in self.months.iteritems():
for x in vals:
ans = ans.replace(x, self.english_months[i])
ans = ans.replace(' de ', ' ')
return ans
def run(self):
@ -422,6 +442,7 @@ class Amazon(Source):
'uk' : _('UK'),
'it' : _('Italy'),
'jp' : _('Japan'),
'es' : _('Spain'),
}
options = (
@ -789,6 +810,16 @@ def download_cover(self, log, result_queue, abort, # {{{
),
] # }}}
es_tests = [ # {{{
(
{'identifiers':{'isbn': '8483460831'}},
[title_test('Tiempos Interesantes',
exact=True), authors_test(['Terry Pratchett'])
]
),
] # }}}
jp_tests = [ # {{{
( # isbn -> title, authors
{'identifiers':{'isbn': '9784101302720' }},
@ -804,6 +835,6 @@ def download_cover(self, log, result_queue, abort, # {{{
] # }}}
test_identify_plugin(Amazon.name, com_tests)
#test_identify_plugin(Amazon.name, jp_tests)
#test_identify_plugin(Amazon.name, es_tests)
# }}}

8
src/calibre/ebooks/metadata/worker.py
View file

@ -249,6 +249,7 @@ def _run(self, tdir):
recs[pref.name] = getattr(self.opts, pref.name)
plugboards = self.db.prefs.get('plugboards', {})
template_functions = self.db.prefs.get('user_template_functions', [])
for i, task in enumerate(tasks):
tids = [x[-1] for x in task]
@ -260,7 +261,7 @@ def _run(self, tdir):
job = ParallelJob('save_book',
'Save books (%d of %d)'%(i, len(tasks)),
lambda x,y:x,
args=[tids, dpath, plugboards, self.path, recs])
args=[tids, dpath, plugboards, template_functions, self.path, recs])
jobs.add(job)
server.add_job(job)
@ -312,9 +313,12 @@ def get_notifications(self, job, ids):
break
def save_book(ids, dpath, plugboards, path, recs, notification=lambda x,y:x):
def save_book(ids, dpath, plugboards, template_functions, path, recs,
notification=lambda x,y:x):
from calibre.library.save_to_disk import config, save_serialized_to_disk
from calibre.customize.ui import apply_null_metadata
from calibre.utils.formatter_functions import load_user_template_functions
load_user_template_functions(template_functions)
opts = config().parse()
for name in recs:
setattr(opts, name, recs[name])

2
src/calibre/ebooks/pdf/images.cpp
View file

@ -126,7 +126,7 @@ void XMLImages::add(GfxState *state, Object *ref, Stream *str,
if (img->type == jpeg) {
int c;
str = ((DCTStream *)str)->getRawStream();
str = str->getNextStream();
str->reset();
// copy the stream

5
src/calibre/ebooks/pdf/reflow.cpp
View file

@ -625,7 +625,12 @@ static string get_link_dest(LinkAction *link, PDFDoc *doc) {
return oss.str();
}
#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17)
void XMLOutputDev::process_link(Link* link){
#else
void XMLOutputDev::process_link(AnnotLink* link){
#endif
double _x1, _y1, _x2, _y2;
int x1, y1, x2, y2;

5
src/calibre/ebooks/pdf/reflow.h
View file

@ -244,6 +244,11 @@ class XMLOutputDev : public OutputDev {
XMLImages *images;
PDFDoc *doc;
#if (POPPLER_MAJOR_VERSION == 0) && (POPPLER_MINOR_VERSION < 17)
void process_link(Link* link);
#else
void process_link(AnnotLink* link);
#endif
};
}

15
src/calibre/gui2/metadata/basic_widgets.py
View file

@ -7,7 +7,7 @@
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import textwrap, re, os, errno
import textwrap, re, os, errno, shutil
from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal, QMessageBox,
QIcon, QToolButton, QWidget, QLabel, QGridLayout, QApplication,
@ -33,8 +33,9 @@
from calibre.library.comments import comments_to_html
from calibre.gui2.dialogs.tag_editor import TagEditor
from calibre.utils.icu import strcmp
from calibre.ptempfile import PersistentTemporaryFile
from calibre.ptempfile import PersistentTemporaryFile, SpooledTemporaryFile
from calibre.gui2.languages import LanguagesEdit as LE
from calibre.db import SPOOL_SIZE
def save_dialog(parent, title, msg, det_msg=''):
d = QMessageBox(parent)
@ -43,8 +44,6 @@ def save_dialog(parent, title, msg, det_msg=''):
d.setStandardButtons(QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel)
return d.exec_()
'''
The interface common to all widgets used to set basic metadata
class BasicMetadataWidget(object):
@ -731,8 +730,12 @@ def commit(self, db, id_):
else:
old_extensions.add(ext)
for ext in new_extensions:
db.add_format(id_, ext, open(paths[ext], 'rb'), notify=False,
index_is_id=True)
with SpooledTemporaryFile(SPOOL_SIZE) as spool:
with open(paths[ext], 'rb') as f:
shutil.copyfileobj(f, spool)
spool.seek(0)
db.add_format(id_, ext, spool, notify=False,
index_is_id=True)
dbfmts = db.formats(id_, index_is_id=True)
db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts
else [])])

11
src/calibre/library/database2.py
View file

@ -7,7 +7,7 @@
The database used to store ebook metadata
'''
import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
json, uuid, tempfile, hashlib, copy
json, uuid, hashlib, copy
from collections import defaultdict
import threading, random
from itertools import repeat
@ -26,7 +26,8 @@
from calibre.library.prefs import DBPrefs
from calibre.ebooks.metadata.book.base import Metadata
from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
from calibre.ptempfile import PersistentTemporaryFile, base_dir
from calibre.ptempfile import (PersistentTemporaryFile,
base_dir, SpooledTemporaryFile)
from calibre.customize.ui import run_plugins_on_import
from calibre import isbytestring
from calibre.utils.filenames import ascii_filename
@ -610,7 +611,7 @@ def set_path(self, index, index_is_id=False):
with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
f.write(cdata)
for format in formats:
with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream:
with SpooledTemporaryFile(SPOOL_SIZE) as stream:
try:
self.copy_format_to(id, format, stream, index_is_id=True)
stream.seek(0)
@ -694,7 +695,7 @@ def cover(self, index, index_is_id=False, as_file=False, as_image=False,
shutil.copyfileobj(f, pt)
return pt.name
if as_file:
ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
ret = SpooledTemporaryFile(SPOOL_SIZE)
shutil.copyfileobj(f, ret)
ret.seek(0)
else:
@ -1282,7 +1283,7 @@ def format(self, index, format, index_is_id=False, as_file=False,
shutil.copyfileobj(f, pt)
ret = pt.name
elif as_file:
ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE)
ret = SpooledTemporaryFile(SPOOL_SIZE)
shutil.copyfileobj(f, ret)
ret.seek(0)
# Various bits of code try to use the name as the default

13
src/calibre/ptempfile.py
View file

@ -181,4 +181,17 @@ def __exit__(self, *args):
class SpooledTemporaryFile(tempfile.SpooledTemporaryFile):
def __init__(self, max_size=0, suffix="", prefix="", dir=None, mode='w+b',
bufsize=-1):
if prefix == None:
prefix = ''
if suffix is None:
suffix = ''
if dir is None:
dir = base_dir()
tempfile.SpooledTemporaryFile.__init__(self, max_size=max_size, suffix=suffix,
prefix=prefix, dir=dir, mode=mode, bufsize=bufsize)

28
src/calibre/utils/date.py
View file

@ -170,32 +170,40 @@ def format_date(dt, format, assume_utc=False, as_utc=False):
if format == 'iso':
return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc)
if dt == UNDEFINED_DATE:
return ''
strf = partial(strftime, t=dt.timetuple())
def format_day(mo):
l = len(mo.group(0))
def format_day(dy):
l = len(dy)
if l == 1: return '%d'%dt.day
if l == 2: return '%02d'%dt.day
if l == 3: return strf('%a')
return strf('%A')
def format_month(mo):
l = len(mo.group(0))
l = len(mo)
if l == 1: return '%d'%dt.month
if l == 2: return '%02d'%dt.month
if l == 3: return strf('%b')
return strf('%B')
def format_year(mo):
if len(mo.group(0)) == 2: return '%02d'%(dt.year % 100)
def format_year(yr):
if len(yr) == 2: return '%02d'%(dt.year % 100)
return '%04d'%dt.year
if dt == UNDEFINED_DATE:
return ''
def repl_func(mo):
s = mo.group(0)
if s is None:
return ''
if s[0] == 'd':
return format_day(s)
if s[0] == 'M':
return format_month(s)
return format_year(s)
format = re.sub('d{1,4}', format_day, format)
format = re.sub('M{1,4}', format_month, format)
return re.sub('yyyy|yy', format_year, format)
return re.sub('(d{1,4}|M{1,4}|(?:yyyy|yy))', repl_func, format)
def replace_months(datestr, clang):
# Replace months by english equivalent for parse_date

21
src/calibre/web/feeds/news.py
View file

@ -13,7 +13,7 @@
from contextlib import nested, closing
from calibre import (browser, __appname__, iswindows,
from calibre import (browser, __appname__, iswindows, force_unicode,
strftime, preferred_encoding, as_unicode)
from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
from calibre.ebooks.metadata.opf2 import OPFCreator
@ -49,7 +49,7 @@ class BasicNewsRecipe(Recipe):
#: A couple of lines that describe the content this recipe downloads.
#: This will be used primarily in a GUI that presents a list of recipes.
description = ''
description = u''
#: The author of this recipe
__author__ = __appname__
@ -112,8 +112,6 @@ class BasicNewsRecipe(Recipe):
#: If set to "optional" the use of a username and password becomes optional
needs_subscription = False
#:
#: If True the navigation bar is center aligned, otherwise it is left aligned
center_navbar = True
@ -1205,12 +1203,22 @@ def create_opf(self, feeds, dir=None):
mi.author_sort = __appname__
mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
mi.timestamp = nowf()
article_titles, aseen = [], set()
for f in feeds:
for a in f:
if a.title and a.title not in aseen:
aseen.add(a.title)
article_titles.append(force_unicode(a.title, 'utf-8'))
mi.comments = self.description
if not isinstance(mi.comments, unicode):
mi.comments = mi.comments.decode('utf-8', 'replace')
mi.comments += ('\n\n' + _('Articles in this issue: ') + '\n' +
'\n\n'.join(article_titles))
language = canonicalize_lang(self.language)
if language is not None:
mi.language = language
if not isinstance(mi.comments, unicode):
mi.comments = mi.comments.decode('utf-8', 'replace')
mi.pubdate = nowf()
opf_path = os.path.join(dir, 'index.opf')
ncx_path = os.path.join(dir, 'index.ncx')
@ -1256,6 +1264,7 @@ def create_opf(self, feeds, dir=None):
self.play_order_counter = 0
self.play_order_map = {}
def feed_index(num, parent):
f = feeds[num]
for j, a in enumerate(f):