Backup_Repos/calibre
1
0
Fork 0
mirror of git://github.com/kovidgoyal/calibre.git synced 2026-05-08 17:34:04 +02:00
Code Issues Projects Releases Wiki Activity

Merge branch 'py3' of https://github.com/eli-schwartz/calibre

Browse source
This commit is contained in:
Kovid Goyal 2019-03-27 05:55:10 +05:30
commit 55b39e16bb
No known key found for this signature in database
GPG key ID: 06BC317B515ACE7C
63 changed files with 357 additions and 510 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
manual/custom.py
View file

@ -179,7 +179,7 @@ def generate_ebook_convert_help(preamble, app):
raw += '\n\n.. contents::\n :local:'
raw += '\n\n' + options
for pl in sorted(input_format_plugins(), key=lambda x:x.name):
for pl in sorted(input_format_plugins(), key=lambda x: x.name):
parser, plumber = create_option_parser(['ebook-convert',
'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log)
groups = [(pl.name+ ' Options', '', g.option_list) for g in
@ -279,7 +279,7 @@ def cli_docs(app):
info(bold('creating CLI documentation...'))
documented_cmds, undocumented_cmds = get_cli_docs()
documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0]))
documented_cmds.sort(key=lambda x: x[0])
undocumented_cmds.sort()
documented = [' '*4 + c[0] for c in documented_cmds]

4
src/calibre/__init__.py
View file

@ -240,7 +240,7 @@ def prints(*args, **kwargs):
file.write(arg)
count += len(arg)
except:
import repr as reprlib
from polyglot import reprlib
arg = reprlib.repr(arg)
file.write(arg)
count += len(arg)
@ -614,7 +614,7 @@ def check(ch):
return check(html5_entities[ent])
except KeyError:
pass
from htmlentitydefs import name2codepoint
from polyglot.html_entities import name2codepoint
try:
return check(my_unichr(name2codepoint[ent]))
except KeyError:

2
src/calibre/constants.py
View file

@ -168,7 +168,6 @@ def __init__(self):
'icu',
'speedup',
'unicode_names',
'zlib2',
'html',
'freetype',
'imageops',
@ -184,6 +183,7 @@ def __init__(self):
if not ispy3:
plugins.extend([
'monotonic',
'zlib2',
])
if iswindows:
plugins.extend(['winutil', 'wpd', 'winfonts'])

4
src/calibre/customize/profiles.py
View file

@ -233,7 +233,7 @@ class NookInput(InputProfile):
HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput,
IRexDR1000Input, IRexDR800Input, NookInput]
input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
input_profiles.sort(key=lambda x: x.name.lower())
# }}}
@ -870,4 +870,4 @@ class PocketBookPro912Output(OutputProfile):
KindlePaperWhite3Output, KindleOasisOutput
]
output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
output_profiles.sort(key=lambda x: x.name.lower())

4
src/calibre/customize/ui.py
View file

@ -727,9 +727,9 @@ def initialize_plugins(perf=False):
# ipython
sys.stdout, sys.stderr = ostdout, ostderr
if perf:
for x in sorted(times, key=lambda x:times[x]):
for x in sorted(times, key=lambda x: times[x]):
print('%50s: %.3f'%(x, times[x]))
_initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True)
_initialized_plugins.sort(key=lambda x: x.priority, reverse=True)
reread_filetype_plugins()
reread_metadata_plugins()

4
src/calibre/db/cli/cmd_list_categories.py
View file

@ -149,9 +149,7 @@ def category_metadata(k):
(not report_on or k in report_on)
]
categories.sort(
cmp=lambda x, y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:])
)
categories.sort(key=lambda x: x if x[0] != '#' else x[1:])
def fmtr(v):
v = v or 0

8
src/calibre/db/cli/main.py
View file

@ -4,7 +4,6 @@
from __future__ import absolute_import, division, print_function, unicode_literals
import httplib
import json
import os
import sys
@ -17,6 +16,7 @@
from calibre.utils.localization import localize_user_manual_link
from calibre.utils.lock import singleinstance
from calibre.utils.serialize import MSGPACK_MIME
from polyglot import http_client
from polyglot.urllib import urlencode, urlparse, urlunparse
COMMANDS = (
@ -191,13 +191,13 @@ def run(self, name, *args):
return m.implementation(self.db.new_api, None, *args)
def interpret_http_error(self, err):
if err.code == httplib.UNAUTHORIZED:
if err.code == http_client.UNAUTHORIZED:
if self.has_credentials:
raise SystemExit('The username/password combination is incorrect')
raise SystemExit('A username and password is required to access this server')
if err.code == httplib.FORBIDDEN:
if err.code == http_client.FORBIDDEN:
raise SystemExit(err.reason)
if err.code == httplib.NOT_FOUND:
if err.code == http_client.NOT_FOUND:
raise SystemExit(err.reason)
def remote_run(self, name, m, *args):

4
src/calibre/db/tests/legacy.py
View file

@ -8,13 +8,13 @@
import inspect, time, numbers
from io import BytesIO
from repr import repr
from functools import partial
from operator import itemgetter
from calibre.library.field_metadata import fm_as_dict
from calibre.db.tests.base import BaseTest
from polyglot.builtins import iteritems, range
from polyglot import reprlib
# Utils {{{
@ -32,7 +32,7 @@ def __call__(self, test):
oldres = getattr(old, self.func_name)(*self.args, **self.kwargs)
newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs)
test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % (
self.func_name, repr(self.args), repr(self.kwargs)))
self.func_name, reprlib.repr(self.args), reprlib.repr(self.kwargs)))
self.retval = newres
return newres

7
src/calibre/devices/__init__.py
View file

@ -8,7 +8,7 @@
import sys, time, pprint
from functools import partial
from StringIO import StringIO
from io import BytesIO
DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6)
MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12)
@ -77,13 +77,12 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None,
oldo, olde = sys.stdout, sys.stderr
if buf is None:
buf = StringIO()
buf = BytesIO()
sys.stdout = sys.stderr = buf
out = partial(prints, file=buf)
devplugins = device_plugins() if plugins is None else plugins
devplugins = list(sorted(devplugins, cmp=lambda
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
devplugins = list(sorted(devplugins, key=lambda x: x.__class__.__name__))
if plugins is None:
for d in devplugins:
try:

2
src/calibre/devices/prs505/sony_cache.py
View file

@ -321,7 +321,7 @@ def rebase_ids(root, base, sourceid, pl_sourceid):
# Only rebase ids of nodes that are immediate children of the
# record root (that way playlist/itemnodes are unaffected
items = root.xpath('child::*[@id]')
items.sort(cmp=lambda x,y:cmp(int(x.get('id')), int(y.get('id'))))
items.sort(key=lambda x: int(x.get('id')))
idmap = {}
for i, item in enumerate(items):
old = int(item.get('id'))

2
src/calibre/devices/usbms/device.py
View file

@ -537,7 +537,7 @@ def find_largest_partition(self, path):
if sz > 0:
nodes.append((x.split('/')[-1], sz))
nodes.sort(cmp=lambda x, y: cmp(x[1], y[1]))
nodes.sort(key=lambda x: x[1])
if not nodes:
return node
return nodes[-1][0]

3
src/calibre/devices/winusb.py
View file

@ -995,8 +995,7 @@ def develop(): # {{{
drive_letters = set()
pprint(usb_devices)
print()
devplugins = list(sorted(device_plugins(), cmp=lambda
x,y:cmp(x.__class__.__name__, y.__class__.__name__)))
devplugins = list(sorted(device_plugins(), key=lambda x: x.__class__.__name__))
for dev in devplugins:
dev.startup()
for dev in devplugins:

4
src/calibre/ebooks/conversion/plugins/tcr_input.py
View file

@ -4,7 +4,7 @@
__copyright__ = '2009, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
from cStringIO import StringIO
from io import BytesIO
from calibre.customize.conversion import InputFormatPlugin
@ -24,7 +24,7 @@ def convert(self, stream, options, file_ext, log, accelerators):
raw_txt = decompress(stream)
log.info('Converting text to OEB...')
stream = StringIO(raw_txt)
stream = BytesIO(raw_txt)
from calibre.customize.ui import plugin_for_input_format

2
src/calibre/ebooks/conversion/plumber.py
View file

@ -821,7 +821,7 @@ def find_html_index(self, files):
if not html_files:
raise ValueError(_('Could not find an e-book inside the archive'))
html_files = [(f, os.stat(f).st_size) for f in html_files]
html_files.sort(cmp=lambda x, y: cmp(x[1], y[1]))
html_files.sort(key=lambda x: x[1])
html_files = [f[0] for f in html_files]
for q in ('toc', 'index'):
for f in html_files:

67
src/calibre/ebooks/lit/from_any.py
View file

@ -1,67 +0,0 @@
from __future__ import with_statement
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
Convert any ebook format to LIT.
'''
import sys, os, glob, logging
from calibre.ebooks.epub.from_any import any2epub, formats, USAGE
from calibre.ebooks.epub import config as common_config
from calibre.ptempfile import TemporaryDirectory
from calibre.ebooks.lit.writer import oeb2lit
def config(defaults=None):
c = common_config(defaults=defaults, name='lit')
return c
def option_parser(usage=USAGE):
return config().option_parser(usage=usage%('LIT', formats()))
def any2lit(opts, path):
ext = os.path.splitext(path)[1]
if not ext:
raise ValueError('Unknown file type: '+path)
ext = ext.lower()[1:]
if opts.output is None:
opts.output = os.path.splitext(os.path.basename(path))[0]+'.lit'
opts.output = os.path.abspath(opts.output)
orig_output = opts.output
with TemporaryDirectory('_any2lit') as tdir:
oebdir = os.path.join(tdir, 'oeb')
os.mkdir(oebdir)
opts.output = os.path.join(tdir, 'dummy.epub')
opts.profile = 'None'
orig_bfs = opts.base_font_size2
opts.base_font_size2 = 0
any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir)
opts.base_font_size2 = orig_bfs
opf = glob.glob(os.path.join(oebdir, '*.opf'))[0]
opts.output = orig_output
logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...'))
oeb2lit(opts, opf)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) < 2:
parser.print_help()
print('No input file specified.')
return 1
any2lit(opts, args[1])
return 0
if __name__ == '__main__':
sys.exit(main())

3
src/calibre/ebooks/lit/writer.py
View file

@ -670,8 +670,7 @@ def _calculate_deskey(self, hashdata):
def _build_dchunks(self):
ddata = []
directory = list(self._directory)
directory.sort(cmp=lambda x, y:
cmp(x.name.lower(), y.name.lower()))
directory.sort(key=lambda x: x.name.lower())
qrn = 1 + (1 << 2)
dchunk = io.BytesIO()
dcount = 0

128
src/calibre/ebooks/lrf/html/convert_to.py
View file

@ -1,128 +0,0 @@
from __future__ import print_function
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
import sys, logging, os
from calibre import setup_cli_handlers
from calibre.utils.config import OptionParser
from calibre.ebooks import ConversionError
from calibre.ebooks.lrf.meta import get_metadata
from calibre.ebooks.lrf.lrfparser import LRFDocument
from calibre.ebooks.metadata.opf import OPFCreator
from calibre.ebooks.lrf.objects import PageAttr, BlockAttr, TextAttr
from calibre.ebooks.lrf.pylrs.pylrs import TextStyle
class BlockStyle(object):
def __init__(self, ba):
self.ba = ba
def __str__(self):
ans = '.'+str(self.ba.id)+' {\n'
if hasattr(self.ba, 'sidemargin'):
margin = str(self.ba.sidemargin) + 'px'
ans += '\tmargin-left: %(m)s; margin-right: %(m)s;\n'%dict(m=margin)
if hasattr(self.ba, 'topskip'):
ans += '\tmargin-top: %dpx;\n'%(self.ba.topskip,)
if hasattr(self.ba, 'footskip'):
ans += '\tmargin-bottom: %dpx;\n'%(self.ba.footskip,)
if hasattr(self.ba, 'framewidth'):
ans += '\tborder-width: %dpx;\n'%(self.ba.framewidth,)
ans += '\tborder-style: solid;\n'
if hasattr(self.ba, 'framecolor'):
if self.ba.framecolor.a < 255:
ans += '\tborder-color: %s;\n'%(self.ba.framecolor.to_html())
if hasattr(self.ba, 'bgcolor'):
if self.ba.bgcolor.a < 255:
ans += '\tbackground-color: %s;\n'%(self.ba.bgcolor.to_html())
# TODO: Fixed size blocks
return ans + '}\n'
class LRFConverter(object):
def __init__(self, document, opts, logger):
self.lrf = document
self.opts = opts
self.output_dir = opts.out
self.logger = logger
logger.info('Parsing LRF...')
self.lrf.parse()
self.create_metadata()
self.create_styles()
def create_metadata(self):
self.logger.info('Reading metadata...')
mi = get_metadata(self.lrf)
self.opf = OPFCreator(self.output_dir, mi)
def create_page_styles(self):
self.page_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, PageAttr):
selector = 'body.'+str(obj.id)
self.page_css = selector + ' {\n'
# TODO: Headers and footers
self.page_css += '}\n'
def create_block_styles(self):
self.block_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, BlockAttr):
self.block_css += str(BlockStyle(obj))
def create_text_styles(self):
self.text_css = ''
for obj in self.lrf.objects.values():
if isinstance(obj, TextAttr):
self.text_css += str(TextStyle(obj))
print(self.text_css)
def create_styles(self):
self.logger.info('Creating CSS stylesheet...')
self.create_page_styles()
self.create_block_styles()
def option_parser():
parser = OptionParser(usage='%prog book.lrf')
parser.add_option('--output-dir', '-o', default=None, help=(
'Output directory in which to store created HTML files. If it does not exist, it is created. By default the current directory is used.'), dest='out')
parser.add_option('--verbose', default=False, action='store_true', dest='verbose')
return parser
def process_file(lrfpath, opts, logger=None):
if logger is None:
level = logging.DEBUG if opts.verbose else logging.INFO
logger = logging.getLogger('lrf2html')
setup_cli_handlers(logger, level)
if opts.out is None:
opts.out = os.getcwdu()
else:
opts.out = os.path.abspath(opts.out)
if not os.path.isdir(opts.out):
raise ConversionError(opts.out + ' is not a directory')
if not os.path.exists(opts.out):
os.makedirs(opts.out)
document = LRFDocument(open(lrfpath, 'rb'))
LRFConverter(document, opts, logger)
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
return 1
process_file(args[1], opts)
return 0
if __name__ == '__main__':
sys.exit(main())

2
src/calibre/ebooks/metadata/lit.py
View file

@ -32,7 +32,7 @@ def get_metadata(stream):
except:
pass
break
covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True)
covers.sort(key=lambda x: len(x[0]), reverse=True)
idx = 0
if len(covers) > 1:
if covers[1][1] == covers[0][1]+'-standard':

4
src/calibre/ebooks/metadata/meta.py
View file

@ -41,8 +41,7 @@ def metadata_from_formats(formats, force_read_metadata=False, pattern=None):
def _metadata_from_formats(formats, force_read_metadata=False, pattern=None):
mi = MetaInformation(None, None)
formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],
METADATA_PRIORITIES[path_to_ext(y)]))
formats.sort(key=lambda x: METADATA_PRIORITIES[path_to_ext(x)])
extensions = list(map(path_to_ext, formats))
if 'opf' in extensions:
opf = formats[extensions.index('opf')]
@ -241,4 +240,3 @@ def forked_read_metadata(path, tdir):
opf = metadata_to_opf(mi, default_lang='und')
with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f:
f.write(opf)

3
src/calibre/ebooks/oeb/transforms/guide.py
View file

@ -28,7 +28,7 @@ def __call__(self, oeb, opts):
else:
covers.append([self.oeb.guide[x], len(item.data)])
covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True)
covers.sort(key=lambda x: x[1], reverse=True)
if covers:
ref = covers[0][0]
if len(covers) > 1:
@ -53,4 +53,3 @@ def __call__(self, oeb, opts):
if item.title and item.title.lower() == 'start':
continue
self.oeb.guide.remove(x)

21
src/calibre/ebooks/pdf/from_comic.py
View file

@ -1,21 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'Convert a comic in CBR/CBZ format to pdf'
import sys
from functools import partial
from calibre.ebooks.lrf.comic.convert_from import do_convert, option_parser, config, main as _main
convert = partial(do_convert, output_format='pdf')
main = partial(_main, output_format='pdf')
if __name__ == '__main__':
sys.exit(main())
if False:
option_parser
config

10
src/calibre/ebooks/pdf/reflow.py
View file

@ -169,7 +169,7 @@ def prepend(self, elem):
self._post_add()
def _post_add(self):
self.elements.sort(cmp=lambda x,y:cmp(x.bottom,y.bottom))
self.elements.sort(key=lambda x: x.bottom)
self.top = self.elements[0].top
self.bottom = self.elements[-1].bottom
self.left, self.right = sys.maxint, 0
@ -260,7 +260,7 @@ def __init__(self, opts, log):
def add(self, columns):
if not self.columns:
for x in sorted(columns, cmp=lambda x,y: cmp(x.left, y.left)):
for x in sorted(columns, key=lambda x: x.left):
self.columns.append(x)
else:
for i in range(len(columns)):
@ -458,7 +458,7 @@ def __init__(self, page, font_map, opts, log, idc):
self.elements = list(self.texts)
for img in page.xpath('descendant::img'):
self.elements.append(Image(img, self.opts, self.log, idc))
self.elements.sort(cmp=lambda x,y:cmp(x.top, y.top))
self.elements.sort(key=lambda x: x.top)
def coalesce_fragments(self):
@ -580,7 +580,7 @@ def coalesce_regions(self):
def sort_into_columns(self, elem, neighbors):
neighbors.add(elem)
neighbors = sorted(neighbors, cmp=lambda x,y:cmp(x.left, y.left))
neighbors = sorted(neighbors, key=lambda x: x.left)
if self.opts.verbose > 3:
self.log.debug('Neighbors:', [x.to_html() for x in neighbors])
columns = [Column()]
@ -595,7 +595,7 @@ def sort_into_columns(self, elem, neighbors):
if not added:
columns.append(Column())
columns[-1].add(x)
columns.sort(cmp=lambda x,y:cmp(x.left, y.left))
columns.sort(key=lambda x: x.left)
return columns
def find_elements_in_row_of(self, x):

3
src/calibre/gui2/actions/choose_library.py
View file

@ -51,8 +51,7 @@ def read_stats(self):
def write_stats(self):
locs = list(self.stats.keys())
locs.sort(cmp=lambda x, y: cmp(self.stats[x], self.stats[y]),
reverse=True)
locs.sort(key=lambda x: self.stats[x], reverse=True)
for key in locs[500:]:
self.stats.pop(key)
gprefs.set('library_usage_stats', self.stats)

2
src/calibre/gui2/dialogs/catalog.py
View file

@ -93,7 +93,7 @@ def __init__(self, parent, dbspec, ids, db):
else:
info("No dynamic tab resources found for %s" % name)
self.widgets = sorted(self.widgets, cmp=lambda x,y:cmp(x.TITLE, y.TITLE))
self.widgets = sorted(self.widgets, key=lambda x: x.TITLE)
# Generate a sorted list of installed catalog formats/sync_enabled pairs
fmts = sorted([x[0] for x in self.fmts])

7
src/calibre/gui2/icon_theme.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, errno, json, importlib, math, httplib, bz2, shutil, sys
import os, errno, json, importlib, math, bz2, shutil, sys
from itertools import count
from io import BytesIO
from threading import Thread, Event
@ -35,8 +35,9 @@
from calibre.utils.zipfile import ZipFile, ZIP_STORED
from calibre.utils.filenames import atomic_rename
from lzma.xz import compress, decompress
from polyglot.queue import Queue, Empty
from polyglot.builtins import iteritems, map, range, reraise
from polyglot import http_client
from polyglot.queue import Queue, Empty
IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'}
THEME_COVER = 'icon-theme-cover.jpg'
@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''):
etag = response.getheader('ETag', None) or None
return cached, etag
except HTTPError as e:
if etag and e.code == httplib.NOT_MODIFIED:
if etag and e.code == http_client.NOT_MODIFIED:
return cached, etag
raise

2
src/calibre/gui2/library/models.py
View file

@ -323,7 +323,7 @@ def col_idx(name):
return 100000
return self.db.field_metadata[name]['rec_index']
self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y)))
self.column_map.sort(key=lambda x: col_idx(x))
for col in self.column_map:
if col in self.orig_headers:
self.headers[col] = self.orig_headers[col]

2
src/calibre/gui2/library/views.py
View file

@ -1031,7 +1031,7 @@ def set_current_row(self, row=0, select=True, for_sync=False):
h.visualIndex(x) > -1]
if not pairs:
pairs = [(0, 0)]
pairs.sort(cmp=lambda x,y:cmp(x[1], y[1]))
pairs.sort(key=lambda x: x[1])
i = pairs[0][0]
index = self.model().index(row, i)
if for_sync:

10
src/calibre/gui2/preferences/columns.py
View file

@ -69,7 +69,7 @@ def init_columns(self, defaults=False):
state = self.columns_state(defaults)
self.hidden_cols = state['hidden_columns']
positions = state['column_positions']
colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y]))
colmap.sort(key=lambda x: positions[x])
self.opt_columns.clear()
db = model.db
@ -248,12 +248,10 @@ def apply_custom_column_changes(self):
if 'ondevice' in hidden_cols:
hidden_cols.remove('ondevice')
def col_pos(x, y):
xidx = config_cols.index(x) if x in config_cols else sys.maxint
yidx = config_cols.index(y) if y in config_cols else sys.maxint
return cmp(xidx, yidx)
def col_pos(x):
return config_cols.index(x) if x in config_cols else sys.maxint
positions = {}
for i, col in enumerate((sorted(model.column_map, cmp=col_pos))):
for i, col in enumerate((sorted(model.column_map, key=col_pos))):
positions[col] = i
state = {'hidden_columns': hidden_cols, 'column_positions':positions}
self.gui.library_view.apply_state(state)

2
src/calibre/gui2/preferences/look_feel.py
View file

@ -456,7 +456,7 @@ def get_esc_lang(l):
if l != lang]
if lang != 'en':
items.append(('en', get_esc_lang('en')))
items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower()))
items.sort(key=lambda x: x[1].lower())
choices = [(y, x) for x, y in items]
# Default language is the autodetected one
choices = [(get_language(lang), lang)] + choices

4
src/calibre/gui2/preferences/main.py
View file

@ -171,7 +171,7 @@ def __init__(self, parent=None):
self.category_names = category_names
categories = list(category_map.keys())
categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y]))
categories.sort(key=lambda x: category_map[x])
self.category_map = OrderedDict()
for c in categories:
@ -181,7 +181,7 @@ def __init__(self, parent=None):
self.category_map[plugin.category].append(plugin)
for plugins in self.category_map.values():
plugins.sort(cmp=lambda x, y: cmp(x.name_order, y.name_order))
plugins.sort(key=lambda x: x.name_order)
self.widgets = []
self._layout = QVBoxLayout()

13
src/calibre/gui2/preferences/plugboard.py
View file

@ -34,17 +34,6 @@ def genesis(self, gui):
self.db = gui.library_view.model().db
def initialize(self):
def field_cmp(x, y):
if x.startswith('#'):
if y.startswith('#'):
return cmp(x.lower(), y.lower())
else:
return 1
elif y.startswith('#'):
return -1
else:
return cmp(x.lower(), y.lower())
ConfigWidgetBase.initialize(self)
self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{}))
@ -73,7 +62,7 @@ def field_cmp(x, y):
if n not in self.disabled_devices:
self.disabled_devices.append(n)
self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower()))
self.devices.sort(key=lambda x: x.lower())
self.devices.insert(1, plugboard_save_to_disk_value)
self.devices.insert(1, plugboard_content_server_value)
self.device_to_formats_map[plugboard_content_server_value] = \

2
src/calibre/gui2/preferences/plugins.py
View file

@ -61,7 +61,7 @@ def populate(self):
self.categories = sorted(self._data.keys())
for plugins in self._data.values():
plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower()))
plugins.sort(key=lambda x: x.name.lower())
def universal_set(self):
ans = set([])

4
src/calibre/gui2/wizard/__init__.py
View file

@ -427,7 +427,7 @@ def get_manufacturers():
def get_devices_of(manufacturer):
ans = [d for d in get_devices() if d.manufacturer == manufacturer]
return sorted(ans, cmp=lambda x,y:cmp(x.name, y.name))
return sorted(ans, key=lambda x: x.name)
class ManufacturerModel(QAbstractListModel):
@ -682,7 +682,7 @@ def get_esc_lang(l):
if l != lang]
if lang != 'en':
items.append(('en', get_esc_lang('en')))
items.sort(cmp=lambda x, y: cmp(x[1], y[1]))
items.sort(key=lambda x: x[1])
for item in items:
self.language.addItem(item[1], (item[0]))
self.language.blockSignals(False)

5
src/calibre/library/catalogs/bibtex.py
View file

@ -7,7 +7,6 @@
import re, codecs, os, numbers
from collections import namedtuple
from types import StringType, UnicodeType
from calibre import (strftime)
from calibre.customize import CatalogPlugin
@ -15,7 +14,7 @@
from calibre.customize.conversion import DummyReporter
from calibre.constants import preferred_encoding
from calibre.ebooks.metadata import format_isbn
from polyglot.builtins import string_or_bytes
from polyglot.builtins import string_or_bytes, unicode_type
class BIBTEX(CatalogPlugin):
@ -351,7 +350,7 @@ def tpl_replace(objtplname) :
bibtexc.ascii_bibtex = True
# Check citation choice and go to default in case of bad CLI
if isinstance(opts.impcit, (StringType, UnicodeType)) :
if isinstance(opts.impcit, (str, unicode_type)) :
if opts.impcit == 'False' :
citation_bibtex= False
elif opts.impcit == 'True' :

4
src/calibre/library/custom_columns.py
View file

@ -216,7 +216,7 @@ def get_custom(self, idx, label=None, num=None, index_is_id=False):
if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
if data['display'].get('sort_alpha', False):
ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
ans.sort(key=lambda x:x.lower())
return ans
def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@ -243,7 +243,7 @@ def get_custom_and_extra(self, idx, label=None, num=None, index_is_id=False):
if data['is_multiple'] and data['datatype'] == 'text':
ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else []
if data['display'].get('sort_alpha', False):
ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
ans.sort(key=lambda x: x.lower())
if data['datatype'] != 'series':
return (ans, None)
ign,lt = self.custom_table_names(data['num'])

2
src/calibre/library/database.py
View file

@ -1018,7 +1018,7 @@ def books_in_series(self, series_id):
if not ans:
return []
ans = [id[0] for id in ans]
ans.sort(cmp=lambda x, y: cmp(self.series_index(x, True), self.series_index(y, True)))
ans.sort(key=lambda x: self.series_index(x, True))
return ans
def books_in_series_of(self, index, index_is_id=False):

2
src/calibre/library/sqlite.py
View file

@ -8,7 +8,6 @@
all calls.
'''
import sqlite3 as sqlite, traceback, time, uuid, sys, os
import repr as reprlib
from sqlite3 import IntegrityError, OperationalError
from threading import Thread
from threading import RLock
@ -22,6 +21,7 @@
from calibre.utils.icu import sort_key
from calibre import prints
from polyglot.builtins import unicode_type
from polyglot import reprlib
from polyglot.queue import Queue
from dateutil.tz import tzoffset

4
src/calibre/spell/dictionary.py
View file

@ -54,7 +54,7 @@ def builtin_dictionaries():
if _builtins is None:
dics = []
for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')):
locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())
locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()))
locale = locales[0]
base = os.path.dirname(lc)
dics.append(Dictionary(
@ -69,7 +69,7 @@ def custom_dictionaries(reread=False):
if _custom is None or reread:
dics = []
for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')):
locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())
locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()))
try:
name, locale, locales = locales[0], locales[1], locales[1:]
except IndexError:

21
src/calibre/srv/auth.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import binascii, os, random, struct, base64, httplib
import binascii, os, random, struct, base64
from collections import OrderedDict
from hashlib import md5, sha256
from itertools import permutations
@ -16,6 +16,7 @@
from calibre.srv.http_request import parse_uri
from calibre.srv.utils import parse_http_dict, encode_path
from calibre.utils.monotonic import monotonic
from polyglot import http_client
MAX_AGE_SECONDS = 3600
nonce_counter, nonce_counter_lock = 0, Lock()
@ -133,19 +134,19 @@ def __init__(self, header_val):
self.nonce_count = data.get('nc')
if self.algorithm not in self.valid_algorithms:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm')
if not (self.username and self.realm and self.nonce and self.uri and self.response):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing')
if self.qop:
if self.qop not in self.valid_qops:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop')
if not (self.cnonce and self.nonce_count):
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent')
else:
if self.cnonce or self.nonce_count:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing')
def H(self, val):
return md5_hex(val)
@ -201,7 +202,7 @@ def validate_request(self, pw, data, log=None):
if log is not None:
log.warn('Authorization URI mismatch: %s != %s from client: %s' % (
data.path, path, data.remote_addr))
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match')
return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response
# }}}
@ -290,16 +291,16 @@ def do_http_auth(self, data, endpoint):
try:
un, pw = base64_decode(rest.strip()).partition(':')[::2]
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters')
if not un or not pw:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty')
if self.check(un, pw):
data.username = un
return
log_msg = 'Failed login attempt from: %s' % data.remote_addr
self.ban_list.failed(ban_key)
else:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method')
if self.prefer_basic_auth:
raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg)

14
src/calibre/srv/errors.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib
from polyglot import http_client
class JobQueueFull(Exception):
@ -30,38 +30,38 @@ def __init__(self, http_code, http_message='', close_connection=False, location=
class HTTPRedirect(HTTPSimpleResponse):
def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False):
def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location)
class HTTPNotFound(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection)
HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection)
class HTTPAuthRequired(HTTPSimpleResponse):
def __init__(self, payload, log=None):
HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log)
HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log)
class HTTPBadRequest(HTTPSimpleResponse):
def __init__(self, message, close_connection=False):
HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection)
HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection)
class HTTPForbidden(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log)
HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log)
class HTTPInternalServerError(HTTPSimpleResponse):
def __init__(self, http_message='', close_connection=True, log=None):
HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log)
class BookNotFound(HTTPNotFound):

43
src/calibre/srv/http_request.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import re, httplib, repr as reprlib
import re
from io import BytesIO, DEFAULT_BUFFER_SIZE
from calibre import as_unicode, force_unicode
@ -14,6 +14,7 @@
from calibre.srv.errors import HTTPSimpleResponse
from calibre.srv.loop import Connection, READ, WRITE
from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator
from polyglot import http_client, reprlib
from polyglot.urllib import unquote
protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11}
@ -68,29 +69,29 @@ def parse_request_uri(uri):
def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri)
if path is None:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component")
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component")
if b'#' in path:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme:
try:
scheme = scheme.decode('ascii')
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2]
if parse_query:
try:
query = MultiDict.create_from_query_string(qs)
except Exception:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string')
raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string')
else:
query = None
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e))
raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query
@ -233,7 +234,7 @@ def readline(self, buf):
if line.endswith(b'\n'):
line = buf.getvalue()
if not line.endswith(b'\r\n'):
self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators')
self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators')
return
return line
if not line:
@ -247,7 +248,7 @@ def connection_ready(self):
self.forwarded_for = None
self.path = self.query = None
self.close_after_response = False
self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG
self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG
self.response_started = False
self.set_state(READ, self.parse_request_line, Accumulator(), first=True)
@ -260,28 +261,28 @@ def parse_request_line(self, buf, event, first=False): # {{{
# Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first:
return self.set_state(READ, self.parse_request_line, Accumulator())
return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed')
return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed')
try:
method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper()
except Exception:
return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line")
return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line")
if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method")
return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method")
try:
self.request_protocol = protocol_map[rp]
except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)]
try:
self.scheme, self.path, self.query = parse_uri(uri)
except HTTPSimpleResponse as e:
return self.simple_response(e.http_code, e.message, close_after_response=False)
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE
self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
# }}}
@ -299,7 +300,7 @@ def parse_header_line(self, parser, buf, event):
try:
parser(line)
except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line')
self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line')
return
if parser.finished:
self.finalize_headers(parser.hdict)
@ -307,7 +308,7 @@ def parse_header_line(self, parser, buf, event):
def finalize_headers(self, inheaders):
request_content_length = int(inheaders.get('Content-Length', 0))
if request_content_length > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
"The entity sent with the request exceeds the maximum "
"allowed bytes (%d)." % self.max_request_body_size)
# Persistent connection support
@ -334,7 +335,7 @@ def finalize_headers(self, inheaders):
else:
# Note that, even if we see "chunked", we must reject
# if there is an extension we don't recognize.
return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
if inheaders.get("Expect", '').lower() == "100-continue":
buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii'))
@ -369,9 +370,9 @@ def read_chunk_length(self, inheaders, line_buf, buf, bytes_read, event):
try:
chunk_size = int(line.strip(), 16)
except Exception:
return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip()))
if bytes_read[0] + chunk_size + 2 > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size)
if chunk_size == 0:
self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True)
@ -389,10 +390,10 @@ def read_chunk_separator(self, inheaders, line_buf, buf, bytes_read, event, last
if line is None:
return
if line != b'\r\n':
return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF')
return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF')
bytes_read[0] += len(line)
if bytes_read[0] > self.max_request_body_size:
return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE,
return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE,
'Chunked request is larger than %d bytes' % self.max_request_body_size)
if last:
self.prepare_response(inheaders, buf)
@ -402,7 +403,7 @@ def read_chunk_separator(self, inheaders, line_buf, buf, bytes_read, event, last
def handle_timeout(self):
if self.response_started:
return False
self.simple_response(httplib.REQUEST_TIMEOUT)
self.simple_response(http_client.REQUEST_TIMEOUT)
return True
def write(self, buf, end=None):

43
src/calibre/srv/http_response.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import os, httplib, hashlib, uuid, struct, repr as reprlib
import os, hashlib, uuid, struct
from collections import namedtuple
from io import BytesIO, DEFAULT_BUFFER_SIZE
from itertools import chain, repeat
@ -26,9 +26,10 @@
sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime)
from calibre.utils.speedups import ReadOnlyFileBuffer
from calibre.utils.monotonic import monotonic
from polyglot import http_client, reprlib
Range = namedtuple('Range', 'start stop size')
MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii')
MULTIPART_SEPARATOR = uuid.uuid4().hex
COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'}
if is_py3:
import zlib
@ -226,7 +227,7 @@ def __init__(self, method, path, query, inheaders, request_body_file, outheaders
self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection
self.forwarded_for = forwarded_for
self.opts = opts
self.status_code = httplib.OK
self.status_code = http_client.OK
self.outcookie = Cookie()
self.lang_code = self.gettext_func = self.ngettext_func = None
self.set_translator(self.get_preferred_language())
@ -402,16 +403,16 @@ def simple_response(self, status_code, msg='', close_after_response=True, extra_
if self.response_protocol is HTTP1:
# HTTP/1.0 has no 413/414/303 codes
status_code = {
httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST,
httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST,
httplib.SEE_OTHER:httplib.FOUND
http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST,
http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST,
http_client.SEE_OTHER:http_client.FOUND
}.get(status_code, status_code)
self.close_after_response = close_after_response
msg = msg.encode('utf-8')
ct = 'http' if self.method == 'TRACE' else 'plain'
buf = [
'%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]),
'%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]),
"Content-Length: %s" % len(msg),
"Content-Type: text/%s; charset=UTF-8" % ct,
"Date: " + http_date(),
@ -432,7 +433,7 @@ def simple_response(self, status_code, msg='', close_after_response=True, extra_
def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False)
return self.simple_response(http_client.OK, msg, close_after_response=False)
request_body_file.seek(0)
outheaders = MultiDict()
data = RequestData(
@ -449,28 +450,28 @@ def run_request_handler(self, data):
def send_range_not_satisfiable(self, content_length):
buf = [
'%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]),
'%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]),
"Date: " + http_date(),
"Content-Range: bytes */%d" % content_length,
]
response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz)
self.response_ready(response_data)
def send_not_modified(self, etag=None):
buf = [
'%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]),
'%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]),
"Content-Length: 0",
"Date: " + http_date(),
]
if etag is not None:
buf.append('ETag: ' + etag)
response_data = header_list_to_file(buf)
self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz)
self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz)
self.response_ready(response_data)
def report_busy(self):
self.simple_response(httplib.SERVICE_UNAVAILABLE)
self.simple_response(http_client.SERVICE_UNAVAILABLE)
def job_done(self, ok, result):
if not ok:
@ -509,7 +510,7 @@ def job_done(self, ok, result):
if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True)
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]]
buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]]
for header, value in sorted(iteritems(outheaders), key=itemgetter(0)):
buf.append('%s: %s' % (header, value))
for morsel in itervalues(data.outcookie):
@ -530,7 +531,7 @@ def job_done(self, ok, result):
def log_access(self, status_code, response_size=None, username=None):
if self.access_log is None:
return
if not self.opts.log_not_found and status_code == httplib.NOT_FOUND:
if not self.opts.log_not_found and status_code == http_client.NOT_FOUND:
return
ff = self.forwarded_for
if ff:
@ -623,7 +624,7 @@ def reset_state(self):
self.ready = ready
def report_unhandled_exception(self, e, formatted_traceback):
self.simple_response(httplib.INTERNAL_SERVER_ERROR)
self.simple_response(http_client.INTERNAL_SERVER_ERROR)
def finalize_output(self, output, request, is_http1):
none_match = parse_if_none_match(request.inheaders.get('If-None-Match', ''))
@ -633,7 +634,7 @@ def finalize_output(self, output, request, is_http1):
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
self.simple_response(http_client.PRECONDITION_FAILED)
return
opts = self.opts
@ -660,10 +661,10 @@ def finalize_output(self, output, request, is_http1):
ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or
ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and
compressible = (compressible and request.status_code == http_client.OK and
(opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and
acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1)
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and
accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and
not is_http1)
ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None
if_range = (request.inheaders.get('If-Range') or '').strip()
@ -680,7 +681,7 @@ def finalize_output(self, output, request, is_http1):
if self.method in ('GET', 'HEAD'):
self.send_not_modified(output.etag)
else:
self.simple_response(httplib.PRECONDITION_FAILED)
self.simple_response(http_client.PRECONDITION_FAILED)
return
output.ranges = None
@ -712,7 +713,7 @@ def finalize_output(self, output, request, is_http1):
outheaders.set('Content-Length', '%d' % size, replace_all=True)
outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True)
output.ranges = zip_longest(ranges, range_parts)
request.status_code = httplib.PARTIAL_CONTENT
request.status_code = http_client.PARTIAL_CONTENT
return output

5
src/calibre/srv/routes.py
View file

@ -6,13 +6,14 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap
import sys, inspect, re, time, numbers, json as jsonlib, textwrap
from operator import attrgetter
from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
from calibre.srv.utils import http_date
from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip
from polyglot import http_client
from polyglot.urllib import quote as urlquote
default_methods = frozenset(('HEAD', 'GET'))
@ -297,7 +298,7 @@ def read_cookies(self, data):
def dispatch(self, data):
endpoint_, args = self.find_route(data.path)
if data.method not in endpoint_.methods:
raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED)
raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED)
self.read_cookies(data)

20
src/calibre/srv/tests/ajax.py
View file

@ -6,13 +6,13 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, base64, os
import zlib, json, base64, os
from io import BytesIO
from functools import partial
from httplib import OK, NOT_FOUND, FORBIDDEN
from calibre.ebooks.metadata.meta import get_metadata
from calibre.srv.tests.base import LibraryBaseTest
from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN
from polyglot.urllib import urlencode, quote
@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password=
conn.request(method, prefix + url, headers=headers, body=data)
r = conn.getresponse()
data = r.read()
if r.status == httplib.OK and data and data[0] in b'{[':
if r.status == OK and data and data[0] in b'{[':
data = json.loads(data)
return r, data
@ -37,10 +37,10 @@ def test_ajax_book(self): # {{{
request = partial(make_request, conn, prefix='/ajax/book')
r, data = request('/x')
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, NOT_FOUND)
r, onedata = request('/1')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(request('/1/' + db.server_library_id)[1], onedata)
self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata)
@ -63,22 +63,22 @@ def test_ajax_categories(self): # {{{
request = partial(make_request, conn)
r, data = request('/categories')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
r, xdata = request('/categories/' + db.server_library_id)
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(data, xdata)
names = {x['name']:x['url'] for x in data}
for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'):
self.assertIn(q, names)
r, data = request(names['Tags'], prefix='')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
names = {x['name']:x['url'] for x in data['items']}
self.ae(set(names), set('Tag One,Tag Two,News'.split(',')))
r, data = request(names['Tag One'], prefix='')
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
self.ae(r.status, httplib.OK)
self.ae(r.status, OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
self.ae(set(data['book_ids']), {2})

56
src/calibre/srv/tests/auth.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, base64, subprocess, os, cookielib, time
import base64, subprocess, os, time
from collections import namedtuple
try:
from distutils.spawn import find_executable
@ -18,6 +18,8 @@
from calibre.srv.tests.base import BaseTest, TestServer
from calibre.srv.routes import endpoint, Router
from polyglot.builtins import iteritems, itervalues
from polyglot import http_client
from polyglot.http_cookie import CookieJar
from polyglot.urllib import (build_opener, HTTPBasicAuthHandler,
HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError)
@ -91,18 +93,18 @@ def test_basic_auth(self): # {{{
conn = server.connect()
conn.request('GET', '/open')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'open')
conn.request('GET', '/closed')
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
self.ae(r.status, http_client.UNAUTHORIZED)
self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM))
self.assertFalse(r.read())
conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')})
r = conn.getresponse()
self.ae(r.read(), b'closed')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(b'closed', urlopen(server, method='basic').read())
self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read())
@ -113,14 +115,14 @@ def request(un='testuser', pw='testpw'):
warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae((http_client.OK, b'closed'), request())
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', ''))
self.ae(1, len(warnings))
self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw'))
self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y'))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', ''))
self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request(''))
self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw'))
# }}}
def test_library_restrictions(self): # {{{
@ -169,7 +171,7 @@ def test_digest_auth(self): # {{{
with TestServer(r.dispatch) as server:
r.auth_controller.log = server.log
def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''):
def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''):
conn.request('GET', path, request_body, headers)
r = conn.getresponse()
self.ae(r.status, status)
@ -177,9 +179,9 @@ def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''):
return {normalize_header_name(k):v for k, v in r.getheaders()}
conn = server.connect()
test(conn, '/open', body=b'open')
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
nonce = auth['nonce']
auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used')
self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth')
self.assertNotIn('stale', auth)
@ -199,14 +201,14 @@ def ok_test(conn, dh, **args):
# Check stale nonces
orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1
auth = parse_http_dict(test(conn, '/closed', headers={
'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2])
self.assertIn('stale', auth)
r.auth_controller.max_age_seconds = orig
ok_test(conn, digest(**args))
def fail_test(conn, modify, **kw):
kw['body'] = kw.get('body', b'')
kw['status'] = kw.get('status', httplib.UNAUTHORIZED)
kw['status'] = kw.get('status', http_client.UNAUTHORIZED)
args['modify'] = modify
return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw)
@ -258,13 +260,13 @@ def request(un='testuser', pw='testpw'):
warnings = []
server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args))
self.ae((httplib.OK, b'closed'), request())
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(httplib.FORBIDDEN, request('x', 'y')[0])
self.ae(httplib.FORBIDDEN, request()[0])
self.ae((http_client.OK, b'closed'), request())
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y'))
self.ae(http_client.FORBIDDEN, request('x', 'y')[0])
self.ae(http_client.FORBIDDEN, request()[0])
time.sleep(ban_for * 60 + 0.01)
self.ae((httplib.OK, b'closed'), request())
self.ae((http_client.OK, b'closed'), request())
# }}}
def test_android_auth_workaround(self): # {{{
@ -277,28 +279,28 @@ def test_android_auth_workaround(self): # {{{
# First check that unauth access fails
conn.request('GET', '/android')
r = conn.getresponse()
self.ae(r.status, httplib.UNAUTHORIZED)
self.ae(r.status, http_client.UNAUTHORIZED)
auth_handler = HTTPDigestAuthHandler()
url = 'http://localhost:%d%s' % (server.address[1], '/android')
auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw')
cj = cookielib.CookieJar()
cj = CookieJar()
cookie_handler = HTTPCookieProcessor(cj)
r = build_opener(auth_handler, cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.getcode(), http_client.OK)
cookies = tuple(cj)
self.ae(len(cookies), 1)
cookie = cookies[0]
self.assertIn(b':', cookie.value)
self.ae(cookie.path, b'/android')
r = build_opener(cookie_handler).open(url)
self.ae(r.getcode(), httplib.OK)
self.ae(r.getcode(), http_client.OK)
self.ae(r.read(), b'android')
# Test that a replay attack against a different URL does not work
try:
build_opener(cookie_handler).open(url+'2')
assert ('Replay attack succeeded')
except HTTPError as e:
self.ae(e.code, httplib.UNAUTHORIZED)
self.ae(e.code, http_client.UNAUTHORIZED)
# }}}

5
src/calibre/srv/tests/base.py
View file

@ -7,12 +7,13 @@
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import unittest, time, httplib, shutil, gc, tempfile, atexit, os
import unittest, time, shutil, gc, tempfile, atexit, os
from io import BytesIO
from functools import partial
from threading import Thread
from calibre.srv.utils import ServerLog
from polyglot import http_client
rmtree = partial(shutil.rmtree, ignore_errors=True)
@ -120,7 +121,7 @@ def connect(self, timeout=None, interface=None):
timeout = self.loop.opts.timeout
if interface is None:
interface = self.address[0]
return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout)
def change_handler(self, handler):
from calibre.srv.http_response import create_http_handler

43
src/calibre/srv/tests/content.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, zlib, json, binascii, time, os
import zlib, json, binascii, time, os
from io import BytesIO
from calibre.ebooks.metadata.epub import get_metadata
@ -14,6 +14,7 @@
from calibre.srv.tests.base import LibraryBaseTest
from calibre.utils.imghdr import identify
from calibre.utils.shared_file import share_open
from polyglot import http_client
def setUpModule():
@ -32,7 +33,7 @@ def test_static(self): # {{{
def missing(url, body=b''):
conn.request('GET', url)
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), body)
for prefix in ('static', 'icon'):
@ -51,7 +52,7 @@ def test(src, url, sz=None):
raw = P(src, data=True)
conn.request('GET', url)
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
data = r.read()
if sz is None:
self.ae(data, raw)
@ -60,7 +61,7 @@ def test(src, url, sz=None):
test_response(r)
conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(b'', r.read())
test('content-server/empty.html', '/static/empty.html')
@ -85,7 +86,7 @@ def get(what, book_id, library_id=None, q=''):
# Test various invalid parameters
def bad(*args):
r, data = get(*args)
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
bad('xxx', 1)
bad('fmt1', 10)
bad('fmt1', 1, 'zzzz')
@ -103,7 +104,7 @@ def bad(*args):
# Test fetching of format with metadata update
raw = P('quick_start/eng.epub', data=True)
r, data = get('epub', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
etag = r.getheader('ETag')
self.assertIsNotNone(etag)
self.ae(r.getheader('Used-Cache'), 'no')
@ -145,39 +146,39 @@ def change_cover(count, book_id=2):
os.utime(cpath, (t, t))
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(r.getheader('Content-Type'), 'image/jpeg')
r, data = get('cover', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(1))
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('cover', 3)
self.ae(r.status, httplib.OK) # Auto generated cover
self.ae(r.status, http_client.OK) # Auto generated cover
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 60, 60))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
r, data = get('thumb', 1, q='sz=100x100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Used-Cache'), 'yes')
change_cover(1, 1)
r, data = get('thumb', 1, q='sz=100')
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(identify(data), ('jpeg', 100, 100))
self.ae(r.getheader('Used-Cache'), 'no')
# Test file sharing in cache
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -185,7 +186,7 @@ def change_cover(count, book_id=2):
# Now force an update
change_cover(1)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8')
@ -193,7 +194,7 @@ def change_cover(count, book_id=2):
# Do it again
change_cover(2)
r, data = get('cover', 2)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(data, db.cover(2))
self.ae(r.getheader('Used-Cache'), 'no')
self.ae(f.read(), fdata)
@ -201,7 +202,7 @@ def change_cover(count, book_id=2):
# Test serving of metadata as opf
r, data = get('opf', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8')
self.assertIsNotNone(r.getheader('Last-Modified'))
opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False)
@ -209,17 +210,17 @@ def change_cover(count, book_id=2):
self.ae(db.field_for('authors', 1), tuple(opf.authors))
conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)
# Test serving metadata as json
r, data = get('json', 1)
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(db.field_for('title', 1), json.loads(data)['title'])
conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip')
raw = r.read()
self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data)

79
src/calibre/srv/tests/http.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, hashlib, zlib, string, time, os
import hashlib, zlib, string, time, os
from io import BytesIO
from tempfile import NamedTemporaryFile
@ -15,6 +15,7 @@
from calibre.srv.utils import eintr_retry_call
from calibre.utils.monotonic import monotonic
from polyglot.builtins import iteritems, range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true'
@ -94,7 +95,7 @@ def handler(data):
def test(al, q):
conn.request('GET', '/', headers={'Accept-Language': al})
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
q += get_translator(q)[-1].ugettext('Unknown')
self.ae(r.read(), q)
@ -136,7 +137,7 @@ def handler(data):
def raw_send(conn, raw):
conn.send(raw)
conn._HTTPConnection__state = httplib._CS_REQ_SENT
conn._HTTPConnection__state = http_client._CS_REQ_SENT
return conn.getresponse()
base_timeout = 0.5 if is_ci else 0.1
@ -144,31 +145,31 @@ def raw_send(conn, raw):
with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server:
conn = server.connect()
r = raw_send(conn, b'hello\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'HTTP requires CRLF line terminators')
r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found')
r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Multiple leading empty lines not allowed')
r = raw_send(conn, b'hello world\r\n')
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'Malformed Request-Line')
r = raw_send(conn, b'x' * 200)
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.ae(r.read(), b'')
r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n')
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method')
# Test 404
conn.request('HEAD', '/moose')
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.assertIsNotNone(r.getheader('Date', None))
self.ae(r.getheader('Content-Length'), str(len(body)))
self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8')
@ -176,7 +177,7 @@ def raw_send(conn, raw):
self.ae(r.read(), '')
conn.request('GET', '/choose')
r = conn.getresponse()
self.ae(r.status, httplib.NOT_FOUND)
self.ae(r.status, http_client.NOT_FOUND)
self.ae(r.read(), b'Requested resource not found')
# Test 500
@ -186,7 +187,7 @@ def raw_send(conn, raw):
conn = server.connect()
conn.request('GET', '/test/')
r = conn.getresponse()
self.ae(r.status, httplib.INTERNAL_SERVER_ERROR)
self.ae(r.status, http_client.INTERNAL_SERVER_ERROR)
server.loop.log.filter_level = orig
# Test 301
@ -196,7 +197,7 @@ def handler(data):
conn = server.connect()
conn.request('GET', '/')
r = conn.getresponse()
self.ae(r.status, httplib.MOVED_PERMANENTLY)
self.ae(r.status, http_client.MOVED_PERMANENTLY)
self.ae(r.getheader('Location'), '/somewhere-else')
self.ae('', r.read())
@ -206,26 +207,26 @@ def handler(data):
# Test simple GET
conn.request('GET', '/test/')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'test')
# Test TRACE
lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', '']
r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii'))
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2]))
# Test POST with simple body
conn.request('POST', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
# Test POST with chunked transfer encoding
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody1234567890')
# Test various incorrect input
@ -233,39 +234,39 @@ def handler(data):
conn.request('GET', '/test' + ('a' * 200))
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
conn = server.connect()
conn.request('GET', '/test', ('a' * 200))
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'x\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST)
self.ae(r.status, http_client.BAD_REQUEST)
self.assertIn(b'not a valid chunk size', r.read())
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'4\r\nbody\r\n200\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn.request('POST', '/test', body='a'*200)
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE)
self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE)
conn = server.connect()
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'3\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF')
conn = server.connect(timeout=base_timeout * 5)
conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'})
conn.send(b'30\r\nbody\r\n0\r\n\r\n')
r = conn.getresponse()
self.ae(r.status, httplib.REQUEST_TIMEOUT)
self.ae(r.status, http_client.REQUEST_TIMEOUT)
self.assertIn(b'', r.read())
server.log.filter_level = orig_level
@ -273,14 +274,14 @@ def handler(data):
# Test pipelining
responses = []
for i in range(10):
conn._HTTPConnection__state = httplib._CS_IDLE
conn._HTTPConnection__state = http_client._CS_IDLE
conn.request('GET', '/%d'%i)
responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method))
for i in range(10):
r = responses[i]
r.begin()
self.ae(r.read(), ('%d' % i).encode('ascii'))
conn._HTTPConnection__state = httplib._CS_IDLE
conn._HTTPConnection__state = http_client._CS_IDLE
# Test closing
server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout
@ -319,12 +320,12 @@ def handler(conn):
conn = server.connect()
conn.request('GET', '/an_etagged_path')
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path')
self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path')
etag = r.getheader('ETag')
self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest())
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'')
# Test gzip
@ -334,7 +335,7 @@ def handler(conn):
conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse()
self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length'))
self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
# Test dynamic etagged content
num_calls = [0]
@ -346,13 +347,13 @@ def edfunc():
conn = server.connect()
conn.request('GET', '/an_etagged_path')
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), b'data')
self.ae(r.status, http_client.OK), self.ae(r.read(), b'data')
etag = r.getheader('ETag')
self.ae(etag, b'"xxx"')
self.ae(r.getheader('Content-Length'), '4')
conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag})
r = conn.getresponse()
self.ae(r.status, httplib.NOT_MODIFIED)
self.ae(r.status, http_client.NOT_MODIFIED)
self.ae(r.read(), b'')
self.ae(num_calls[0], 1)
@ -368,11 +369,11 @@ def edfunc():
self.ae(r.getheader('Content-Type'), guess_type(f.name)[0])
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(int(r.getheader('Content-Length')), len(fdata))
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata)
self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=2-25'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT)
self.ae(r.status, http_client.PARTIAL_CONTENT)
self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes')
self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata))
self.ae(int(r.getheader('Content-Length')), 24)
@ -380,27 +381,27 @@ def edfunc():
conn.request('GET', '/test', headers={'Range':'bytes=100000-'})
r = conn.getresponse()
self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE)
self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51])
self.ae(int(r.getheader('Content-Length')), 26)
conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata)
self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata)
conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'})
r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(r.read(), fdata)
self.ae(r.status, http_client.OK), self.ae(r.read(), fdata)
self.assertFalse(r.getheader('Content-Range'))
self.ae(int(r.getheader('Content-Length')), len(fdata))
conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'})
r = conn.getresponse()
self.ae(r.status, httplib.PARTIAL_CONTENT)
self.ae(r.status, http_client.PARTIAL_CONTENT)
clen = int(r.getheader('Content-Length'))
data = r.read()
self.ae(clen, len(data))
@ -415,7 +416,7 @@ def edfunc():
conn = server.connect(timeout=1)
conn.request('GET', '/test')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
rdata = r.read()
self.ae(len(data), len(rdata))
self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest())

13
src/calibre/srv/tests/loop.py
View file

@ -6,7 +6,7 @@
__license__ = 'GPL v3'
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import httplib, ssl, os, socket, time
import ssl, os, socket, time
from collections import namedtuple
from unittest import skipIf
from glob import glob
@ -18,6 +18,7 @@
from calibre.utils.certgen import create_server_cert
from calibre.utils.monotonic import monotonic
from polyglot.builtins import range
from polyglot import http_client
is_ci = os.environ.get('CI', '').lower() == 'true'
@ -92,7 +93,7 @@ def test_workers(self):
conn.request('GET', '/')
with self.assertRaises(socket.timeout):
res = conn.getresponse()
if str(res.status) == str(httplib.REQUEST_TIMEOUT):
if str(res.status) == str(http_client.REQUEST_TIMEOUT):
raise socket.timeout('Timeout')
raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % (
res.status, res.reason, res.getheaders(), res.read()))
@ -135,7 +136,7 @@ def test_dual_stack(self):
conn = server.connect(interface='127.0.0.1')
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
def test_ring_buffer(self):
@ -203,10 +204,10 @@ def test_ssl(self):
create_server_cert(address, ca_file, cert_file, key_file, key_size=1024)
ctx = ssl.create_default_context(cafile=ca_file)
with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server:
conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx)
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
cert = conn.sock.getpeercert()
subject = dict(x[0] for x in cert['subject'])
@ -226,7 +227,7 @@ def test_socket_activation(self):
conn = server.connect()
conn.request('GET', '/test', 'body')
r = conn.getresponse()
self.ae(r.status, httplib.OK)
self.ae(r.status, http_client.OK)
self.ae(r.read(), b'testbody')
self.ae(server.loop.bound_address[1], port)

4
src/calibre/srv/utils.py
View file

@ -7,9 +7,7 @@
__copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
import errno, socket, select, os, time
from Cookie import SimpleCookie
from contextlib import closing
import repr as reprlib
from email.utils import formatdate
from operator import itemgetter
from binascii import hexlify, unhexlify
@ -23,6 +21,8 @@
from calibre.utils.logging import ThreadSafeLog
from calibre.utils.shared_file import share_open, raise_winerror
from polyglot.builtins import iteritems, map, unicode_type, range
from polyglot import reprlib
from polyglot.http_cookie import SimpleCookie
from polyglot.urllib import parse_qs, quote as urlquote
HTTP1 = 'HTTP/1.0'

7
src/calibre/srv/web_socket.py
View file

@ -5,7 +5,7 @@
from __future__ import (unicode_literals, division, absolute_import,
print_function)
import httplib, os, weakref, socket
import os, weakref, socket
from base64 import standard_b64encode
from collections import deque
from hashlib import sha1
@ -19,6 +19,7 @@
from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE
from calibre.utils.speedups import ReadOnlyFileBuffer
from polyglot.queue import Queue, Empty
from polyglot import http_client
speedup, err = plugins['speedup']
if not speedup:
raise RuntimeError('Failed to load speedup module with error: ' + err)
@ -286,9 +287,9 @@ def finalize_headers(self, inheaders):
except Exception:
ver_ok = False
if not ver_ok:
return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
if self.method != 'GET':
return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)
response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest())
self.optimize_for_sending_packet()

29
src/calibre/test_build.py
View file

@ -73,6 +73,29 @@ def test_html5_parser(self):
from html5_parser import parse
parse('<p>xxx')
def test_imports(self):
import importlib
exclude = ['dbus_export.demo', 'dbus_export.gtk', 'upstream']
if not iswindows:
exclude.extend(['iphlpapi', 'windows', 'winreg', 'winusb'])
if not isosx:
exclude.append('osx')
if not islinux:
exclude.extend(['dbus', 'linux'])
base = os.path.dirname(__file__)
trimpath = len(os.path.dirname(base)) + 1
for root, dirs, files in os.walk(base):
for dir in dirs:
if not os.path.isfile(os.path.join(root, dir, '__init__.py')):
dirs.remove(dir)
for file in files:
file, ext = os.path.splitext(file)
if ext != '.py':
continue
name = '.'.join(root[trimpath:].split(os.path.sep) + [file])
if not any(x for x in exclude if x in name):
importlib.import_module(name)
def test_plugins(self):
exclusions = set()
if is_ci:
@ -99,7 +122,7 @@ def test_lxml(self):
from calibre.utils.cleantext import test_clean_xml_chars
test_clean_xml_chars()
from lxml import etree
raw = '<a/>'
raw = b'<a/>'
root = etree.fromstring(raw)
self.assertEqual(etree.tostring(root), raw)
@ -175,7 +198,7 @@ def test_qt(self):
# it should just work because the hard-coded paths of the Qt
# installation should work. If they do not, then it is a distro
# problem.
fmts = set(map(unicode_type, QImageReader.supportedImageFormats()))
fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats()))
testf = {'jpg', 'png', 'svg', 'ico', 'gif'}
self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts)
data = P('images/blank.png', allow_user_override=False, data=True)
@ -254,7 +277,7 @@ def test_executables(self):
def test_netifaces(self):
import netifaces
self.assertGreaterEqual(netifaces.interfaces(), 1, 'netifaces could find no network interfaces')
self.assertGreaterEqual(len(netifaces.interfaces()), 1, 'netifaces could find no network interfaces')
def test_psutil(self):
import psutil

8
src/calibre/utils/browser.py
View file

@ -5,11 +5,13 @@
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import copy, httplib, ssl
from cookielib import CookieJar, Cookie
import copy, ssl
from mechanize import Browser as B, HTTPSHandler
from polyglot import http_client
from polyglot.http_cookie import CookieJar, Cookie
class ModernHTTPSHandler(HTTPSHandler):
@ -24,7 +26,7 @@ def https_open(self, req):
def conn_factory(hostport, **kw):
kw['context'] = self.ssl_context
return httplib.HTTPSConnection(hostport, **kw)
return http_client.HTTPSConnection(hostport, **kw)
return self.do_open(conn_factory, req)

5
src/calibre/utils/cleantext.py
View file

@ -2,8 +2,9 @@
__copyright__ = '2010, sengian <sengian1@gmail.com>'
__docformat__ = 'restructuredtext en'
import re, htmlentitydefs
import re
from polyglot.builtins import codepoint_to_chr, map, range
from polyglot.html_entities import name2codepoint
from calibre.constants import plugins, preferred_encoding
try:
@ -80,7 +81,7 @@ def fixup(m, rm=rm, rchar=rchar):
else:
# named entity
try:
text = codepoint_to_chr(htmlentitydefs.name2codepoint[text[1:-1]])
text = codepoint_to_chr(name2codepoint[text[1:-1]])
except KeyError:
pass
if rm:

21
src/calibre/utils/https.py
View file

@ -10,7 +10,7 @@
from contextlib import closing
from calibre import get_proxies
from calibre.constants import ispy3
from polyglot import http_client
from polyglot.urllib import urlsplit
has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context')
@ -19,19 +19,14 @@ class HTTPError(ValueError):
def __init__(self, url, code):
msg = '%s returned an unsupported http response code: %d (%s)' % (
url, code, httplib.responses.get(code, None))
url, code, http_client.responses.get(code, None))
ValueError.__init__(self, msg)
self.code = code
self.url = url
if ispy3:
import http.client as httplib
else:
import httplib
if has_ssl_verify:
class HTTPSConnection(httplib.HTTPSConnection):
class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs):
cafile = kwargs.pop('cert_file', None)
@ -39,7 +34,7 @@ def __init__(self, ssl_version, *args, **kwargs):
kwargs['context'] = ssl._create_unverified_context()
else:
kwargs['context'] = ssl.create_default_context(cafile=cafile)
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
http_client.HTTPSConnection.__init__(self, *args, **kwargs)
else:
# Check certificate hostname {{{
# Implementation taken from python 3
@ -136,10 +131,10 @@ def match_hostname(cert, hostname):
"subjectAltName fields were found")
# }}}
class HTTPSConnection(httplib.HTTPSConnection):
class HTTPSConnection(http_client.HTTPSConnection):
def __init__(self, ssl_version, *args, **kwargs):
httplib.HTTPSConnection.__init__(self, *args, **kwargs)
http_client.HTTPSConnection.__init__(self, *args, **kwargs)
self.calibre_ssl_version = ssl_version
def connect(self):
@ -204,7 +199,7 @@ def get_https_resource_securely(
path += '?' + p.query
c.request('GET', path, headers=headers or {})
response = c.getresponse()
if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER):
if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER):
if max_redirects <= 0:
raise ValueError('Too many redirects, giving up')
newurl = response.getheader('Location', None)
@ -212,7 +207,7 @@ def get_https_resource_securely(
raise ValueError('%s returned a redirect response with no Location header' % url)
return get_https_resource_securely(
newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response)
if response.status != httplib.OK:
if response.status != http_client.OK:
raise HTTPError(url, response.status)
if get_response:
return response

3
src/calibre/utils/smtp.py
View file

@ -98,8 +98,7 @@ def get_mx(host, verbose=0):
if verbose:
print('Find mail exchanger for', host)
answers = list(dns.resolver.query(host, 'MX'))
answers.sort(cmp=lambda x, y: cmp(int(getattr(x, 'preference', sys.maxint)),
int(getattr(y, 'preference', sys.maxint))))
answers.sort(key=lambda x: int(getattr(x, 'preference', sys.maxint)))
return [str(x.exchange) for x in answers if hasattr(x, 'exchange')]

22
src/calibre/utils/unrar.py
View file

@ -114,20 +114,20 @@ def test_basic():
b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0<D\xfe\xc8\xef\xbc\xd1\x04I?\xfd\xff\xdbF)]\xe8\xb9\xe1t \x90/\x00\x13\x00\x00\x00\x08\x00\x00\x00\x03\x1a$\x932\xc2]yA\x1d3\r\x00\xa4\x81\x00\x002\\sub-two.txt\x00\xc0\x10\x00S\xec\xcb\x7f\x8b\xa5(\x0b\x01\xcb\xef\xdf\xf6t\x89\x97z\x0eft \x90)\x00\r\x00\x00\x00\r\x00\x00\x00\x03c\x89K\xd3\xc8fyA\x140\x07\x00\xff\xa1\x00\x00symlink\x00\xc02/sub-two.txt\xeb\x86t\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xb9]yA\x140\x01\x00\xedA\x00\x001\x00\xc0\xe0Dt\xe0\x90#\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\xc2]yA\x140\x01\x00\xedA\x00\x002\x00\xc0u\xa1t \x80,\x00\r\x00\x00\x00\r\x00\x00\x00\x03T\xea\x04\xca\xe6\x84yA\x140\x0c\x00\xa4\x81\x00\x00uncompresseduncompressed\n\xda\x10t \x900\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x035K.\xa6\x18\x85yA\x1d5\x0e\x00\xa4\x81\x00\x00max-compressed\x00\xc0\x00\x08\xbf\x08\xae\xf2\xcc\x01s\xf8\xff\xec\x96\xe8\xc4={\x00@\x07\x00") # noqa }}}
tdata = {
u'1': b'',
u'1/sub-one': b'sub-one\n',
u'2': b'',
u'2/sub-two.txt': b'sub-two\n',
u'F\xfc\xdfe.txt': b'unicode\n',
u'max-compressed': b'max\n',
u'one.txt': b'one\n',
u'symlink': b'2/sub-two.txt',
u'uncompressed': b'uncompressed\n',
u'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'}
'1': b'',
'1/sub-one': b'sub-one\n',
'2': b'',
'2/sub-two.txt': b'sub-two\n',
'F\xfc\xdfe.txt': b'unicode\n',
'max-compressed': b'max\n',
'one.txt': b'one\n',
'symlink': b'2/sub-two.txt',
'uncompressed': b'uncompressed\n',
'\u8bf6\u6bd4\u5c41.txt': b'chinese unicode\n'}
def do_test(stream):
c = comment(stream)
if c != b'some comment\n':
if c != 'some comment\n':
raise ValueError('Comment not read: %r != %r' % (c, b'some comment\n'))
if set(names(stream)) != {
'1/sub-one', 'one.txt', '2/sub-two.txt', '诶比屁.txt', 'Füße.txt',

10
src/calibre/web/feeds/news.py
View file

@ -9,7 +9,7 @@
import os, time, traceback, re, sys, io
from collections import defaultdict
from contextlib import nested, closing
from contextlib import closing
from calibre import (browser, __appname__, iswindows, force_unicode,
@ -760,7 +760,7 @@ def sort_index_by(self, index, weights):
in index are not in weights, they are assumed to have a weight of 0.
'''
weights = defaultdict(lambda: 0, weights)
index.sort(cmp=lambda x, y: cmp(weights[x], weights[y]))
index.sort(key=lambda x: weights[x])
return index
def parse_index(self):
@ -1097,7 +1097,7 @@ def feed2index(self, f, feeds):
if bn:
img = os.path.join(imgdir, 'feed_image_%d%s'%(self.image_counter, os.path.splitext(bn)))
try:
with nested(open(img, 'wb'), closing(self.browser.open(feed.image_url))) as (fi, r):
with open(img, 'wb') as fi, closing(self.browser.open(feed.image_url)) as r:
fi.write(r.read())
self.image_counter += 1
feed.image_url = img
@ -1346,7 +1346,7 @@ def _download_masthead(self, mu):
with open(mpath, 'wb') as mfile:
mfile.write(open(mu, 'rb').read())
else:
with nested(open(mpath, 'wb'), closing(self.browser.open(mu))) as (mfile, r):
with open(mpath, 'wb') as mfile, closing(self.browser.open(mu)) as r:
mfile.write(r.read())
self.report_progress(1, _('Masthead image downloaded'))
self.prepare_masthead_image(mpath, outfile)
@ -1564,7 +1564,7 @@ def feed_index(num, parent):
opf.create_spine(entries)
opf.set_toc(toc)
with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
with open(opf_path, 'wb') as opf_file, open(ncx_path, 'wb') as ncx_file:
opf.render(opf_file, ncx_file)
def article_downloaded(self, request, result):

2
src/calibre/web/fetch/simple.py
View file

@ -18,7 +18,6 @@
import time
import traceback
from base64 import b64decode
from httplib import responses
from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows
@ -31,6 +30,7 @@
from calibre.utils.logging import Log
from calibre.web.fetch.utils import rescale_image
from polyglot.builtins import unicode_type
from polyglot.http_client import responses
from polyglot.urllib import (
URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse,
urlunsplit

10
src/polyglot/html_entities.py Normal file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from html.entities import name2codepoint
else:
from htmlentitydefs import name2codepoint

22
src/polyglot/http_client.py Normal file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from http.client import (responses, HTTPConnection, HTTPSConnection,
BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED,
INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY,
NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT,
PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG,
REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER,
SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT)
else:
from httplib import (responses, HTTPConnection, HTTPSConnection,
BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED,
INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY,
NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT,
PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG,
REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER,
SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT)

12
src/polyglot/http_cookie.py Normal file
View file

@ -0,0 +1,12 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from http.cookies import SimpleCookie # noqa
from http.cookiejar import CookieJar, Cookie # noqa
else:
from Cookie import SimpleCookie # noqa
from cookielib import CookieJar, Cookie # noqa

10
src/polyglot/reprlib.py Executable file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPL v3 Copyright: 2019, Eli Schwartz <eschwartz@archlinux.org>
from polyglot.builtins import is_py3
if is_py3:
from reprlib import repr
else:
from repr import repr