From d7410fe7b33c3ad2fdaf761703ede0e1257dbe76 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 10:27:57 -0400 Subject: [PATCH 01/19] python3: add polyglot wrapper for html.entities/htmlentitydefs --- src/calibre/__init__.py | 2 +- src/calibre/utils/cleantext.py | 5 +++-- src/polyglot/html_entities.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 src/polyglot/html_entities.py diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 867163fc2b..c71af8e0f4 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -614,7 +614,7 @@ def check(ch): return check(html5_entities[ent]) except KeyError: pass - from htmlentitydefs import name2codepoint + from polyglot.html_entities import name2codepoint try: return check(my_unichr(name2codepoint[ent])) except KeyError: diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py index 9e04d5d4d6..f8ba151de3 100644 --- a/src/calibre/utils/cleantext.py +++ b/src/calibre/utils/cleantext.py @@ -2,8 +2,9 @@ __copyright__ = '2010, sengian ' __docformat__ = 'restructuredtext en' -import re, htmlentitydefs +import re from polyglot.builtins import codepoint_to_chr, map, range +from polyglot.html_entities import name2codepoint from calibre.constants import plugins, preferred_encoding try: @@ -80,7 +81,7 @@ def fixup(m, rm=rm, rchar=rchar): else: # named entity try: - text = codepoint_to_chr(htmlentitydefs.name2codepoint[text[1:-1]]) + text = codepoint_to_chr(name2codepoint[text[1:-1]]) except KeyError: pass if rm: diff --git a/src/polyglot/html_entities.py b/src/polyglot/html_entities.py new file mode 100644 index 0000000000..2e20eee7f4 --- /dev/null +++ b/src/polyglot/html_entities.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from html.entities import name2codepoint +else: + from htmlentitydefs import name2codepoint From 92c621c718c2073f124c5a816516ce4254ed6faf Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 10:36:48 -0400 Subject: [PATCH 02/19] python3: remove deprecated use of contextlib.nested Using `with Foo() as a, Bar() as b:` is introduced in python 2.7 and deprecates the use of nested(), which is removed entirely in python3 --- src/calibre/web/feeds/news.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index fb8e75e235..f51f7e5a21 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -9,7 +9,7 @@ import os, time, traceback, re, sys, io from collections import defaultdict -from contextlib import nested, closing +from contextlib import closing from calibre import (browser, __appname__, iswindows, force_unicode, @@ -1097,7 +1097,7 @@ def feed2index(self, f, feeds): if bn: img = os.path.join(imgdir, 'feed_image_%d%s'%(self.image_counter, os.path.splitext(bn))) try: - with nested(open(img, 'wb'), closing(self.browser.open(feed.image_url))) as (fi, r): + with open(img, 'wb') as fi, closing(self.browser.open(feed.image_url)) as r: fi.write(r.read()) self.image_counter += 1 feed.image_url = img @@ -1346,7 +1346,7 @@ def _download_masthead(self, mu): with open(mpath, 'wb') as mfile: mfile.write(open(mu, 'rb').read()) else: - with nested(open(mpath, 'wb'), closing(self.browser.open(mu))) as (mfile, r): + with open(mpath, 'wb') as mfile, closing(self.browser.open(mu)) as r: mfile.write(r.read()) self.report_progress(1, _('Masthead image downloaded')) self.prepare_masthead_image(mpath, outfile) @@ -1564,7 +1564,7 @@ def feed_index(num, parent): opf.create_spine(entries) opf.set_toc(toc) - with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file): + with open(opf_path, 'wb') as opf_file, open(ncx_path, 'wb') as ncx_file: opf.render(opf_file, ncx_file) def article_downloaded(self, request, result): From bfcb301fe3da0cef93f61d3785567e27cc0597ad Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 12:07:48 -0400 Subject: [PATCH 03/19] python3: add httplib/http.client polyglot wrapper --- src/calibre/db/cli/main.py | 8 ++-- src/calibre/gui2/icon_theme.py | 7 +-- src/calibre/srv/auth.py | 21 +++++---- src/calibre/srv/errors.py | 14 +++--- src/calibre/srv/http_request.py | 43 ++++++++--------- src/calibre/srv/http_response.py | 41 +++++++++-------- src/calibre/srv/routes.py | 5 +- src/calibre/srv/tests/ajax.py | 20 ++++---- src/calibre/srv/tests/auth.py | 53 ++++++++++----------- src/calibre/srv/tests/base.py | 5 +- src/calibre/srv/tests/content.py | 43 ++++++++--------- src/calibre/srv/tests/http.py | 79 ++++++++++++++++---------------- src/calibre/srv/tests/loop.py | 13 +++--- src/calibre/srv/web_socket.py | 7 +-- src/calibre/utils/browser.py | 6 ++- src/calibre/utils/https.py | 21 ++++----- src/calibre/web/fetch/simple.py | 2 +- 17 files changed, 198 insertions(+), 190 deletions(-) diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 9833d919d4..a80da22180 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import httplib import json import os import sys @@ -17,6 +16,7 @@ from calibre.utils.localization import localize_user_manual_link from calibre.utils.lock import singleinstance from calibre.utils.serialize import MSGPACK_MIME +from polyglot import http_client from polyglot.urllib import urlencode, urlparse, urlunparse COMMANDS = ( @@ -191,13 +191,13 @@ def run(self, name, *args): return m.implementation(self.db.new_api, None, *args) def interpret_http_error(self, err): - if err.code == httplib.UNAUTHORIZED: + if err.code == http_client.UNAUTHORIZED: if self.has_credentials: raise SystemExit('The username/password combination is incorrect') raise SystemExit('A username and password is required to access this server') - if err.code == httplib.FORBIDDEN: + if err.code == http_client.FORBIDDEN: raise SystemExit(err.reason) - if err.code == httplib.NOT_FOUND: + if err.code == http_client.NOT_FOUND: raise SystemExit(err.reason) def remote_run(self, name, m, *args): diff --git a/src/calibre/gui2/icon_theme.py b/src/calibre/gui2/icon_theme.py index e4c48f9562..e5d1c3a0a0 100644 --- a/src/calibre/gui2/icon_theme.py +++ b/src/calibre/gui2/icon_theme.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, errno, json, importlib, math, httplib, bz2, shutil, sys +import os, errno, json, importlib, math, bz2, shutil, sys from itertools import count from io import BytesIO from threading import Thread, Event @@ -35,8 +35,9 @@ from calibre.utils.zipfile import ZipFile, ZIP_STORED from calibre.utils.filenames import atomic_rename from lzma.xz import compress, decompress -from polyglot.queue import Queue, Empty from polyglot.builtins import iteritems, map, range, reraise +from polyglot import http_client +from polyglot.queue import Queue, Empty IMAGE_EXTENSIONS = {'png', 'jpg', 'jpeg'} THEME_COVER = 'icon-theme-cover.jpg' @@ -439,7 +440,7 @@ def download_cover(cover_url, etag=None, cached=b''): etag = response.getheader('ETag', None) or None return cached, etag except HTTPError as e: - if etag and e.code == httplib.NOT_MODIFIED: + if etag and e.code == http_client.NOT_MODIFIED: return cached, etag raise diff --git a/src/calibre/srv/auth.py b/src/calibre/srv/auth.py index 51f884e28f..84fd260219 100644 --- a/src/calibre/srv/auth.py +++ b/src/calibre/srv/auth.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import binascii, os, random, struct, base64, httplib +import binascii, os, random, struct, base64 from collections import OrderedDict from hashlib import md5, sha256 from itertools import permutations @@ -16,6 +16,7 @@ from calibre.srv.http_request import parse_uri from calibre.srv.utils import parse_http_dict, encode_path from calibre.utils.monotonic import monotonic +from polyglot import http_client MAX_AGE_SECONDS = 3600 nonce_counter, nonce_counter_lock = 0, Lock() @@ -133,19 +134,19 @@ def __init__(self, header_val): self.nonce_count = data.get('nc') if self.algorithm not in self.valid_algorithms: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest algorithm') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest algorithm') if not (self.username and self.realm and self.nonce and self.uri and self.response): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Digest algorithm required fields missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Digest algorithm required fields missing') if self.qop: if self.qop not in self.valid_qops: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported digest qop') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported digest qop') if not (self.cnonce and self.nonce_count): - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop present, but cnonce and nonce_count absent') else: if self.cnonce or self.nonce_count: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'qop missing') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'qop missing') def H(self, val): return md5_hex(val) @@ -201,7 +202,7 @@ def validate_request(self, pw, data, log=None): if log is not None: log.warn('Authorization URI mismatch: %s != %s from client: %s' % ( data.path, path, data.remote_addr)) - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The uri in the Request Line and the Authorization header do not match') return self.response is not None and data.path == path and self.request_digest(pw, data) == self.response # }}} @@ -290,16 +291,16 @@ def do_http_auth(self, data, endpoint): try: un, pw = base64_decode(rest.strip()).partition(':')[::2] except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password contained non-UTF8 encoded characters') if not un or not pw: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'The username or password was empty') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'The username or password was empty') if self.check(un, pw): data.username = un return log_msg = 'Failed login attempt from: %s' % data.remote_addr self.ban_list.failed(ban_key) else: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unsupported authentication method') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unsupported authentication method') if self.prefer_basic_auth: raise HTTPAuthRequired('Basic realm="%s"' % self.realm, log=log_msg) diff --git a/src/calibre/srv/errors.py b/src/calibre/srv/errors.py index db251b34cf..02c7c4f65e 100644 --- a/src/calibre/srv/errors.py +++ b/src/calibre/srv/errors.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib +from polyglot import http_client class JobQueueFull(Exception): @@ -30,38 +30,38 @@ def __init__(self, http_code, http_message='', close_connection=False, location= class HTTPRedirect(HTTPSimpleResponse): - def __init__(self, location, http_code=httplib.MOVED_PERMANENTLY, http_message='', close_connection=False): + def __init__(self, location, http_code=http_client.MOVED_PERMANENTLY, http_message='', close_connection=False): HTTPSimpleResponse.__init__(self, http_code, http_message, close_connection, location) class HTTPNotFound(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.NOT_FOUND, http_message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.NOT_FOUND, http_message, close_connection) class HTTPAuthRequired(HTTPSimpleResponse): def __init__(self, payload, log=None): - HTTPSimpleResponse.__init__(self, httplib.UNAUTHORIZED, authenticate=payload, log=log) + HTTPSimpleResponse.__init__(self, http_client.UNAUTHORIZED, authenticate=payload, log=log) class HTTPBadRequest(HTTPSimpleResponse): def __init__(self, message, close_connection=False): - HTTPSimpleResponse.__init__(self, httplib.BAD_REQUEST, message, close_connection) + HTTPSimpleResponse.__init__(self, http_client.BAD_REQUEST, message, close_connection) class HTTPForbidden(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.FORBIDDEN, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.FORBIDDEN, http_message, close_connection, log=log) class HTTPInternalServerError(HTTPSimpleResponse): def __init__(self, http_message='', close_connection=True, log=None): - HTTPSimpleResponse.__init__(self, httplib.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) + HTTPSimpleResponse.__init__(self, http_client.INTERNAL_SERVER_ERROR, http_message, close_connection, log=log) class BookNotFound(HTTPNotFound): diff --git a/src/calibre/srv/http_request.py b/src/calibre/srv/http_request.py index 384fe5a04f..74ac506790 100644 --- a/src/calibre/srv/http_request.py +++ b/src/calibre/srv/http_request.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import re, httplib, repr as reprlib +import re, repr as reprlib from io import BytesIO, DEFAULT_BUFFER_SIZE from calibre import as_unicode, force_unicode @@ -14,6 +14,7 @@ from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator +from polyglot import http_client from polyglot.urllib import unquote protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} @@ -68,29 +69,29 @@ def parse_request_uri(uri): def parse_uri(uri, parse_query=True): scheme, authority, path = parse_request_uri(uri) if path is None: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "No path component") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component") if b'#' in path: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.") + raise HTTPSimpleResponse(http_client.BAD_REQUEST, "Illegal #fragment in Request-URI.") if scheme: try: scheme = scheme.decode('ascii') except ValueError: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Un-decodeable scheme') path, qs = path.partition(b'?')[::2] if parse_query: try: query = MultiDict.create_from_query_string(qs) except Exception: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string') + raise HTTPSimpleResponse(http_client.BAD_REQUEST, 'Unparseable query string') else: query = None try: path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path)) except ValueError as e: - raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e)) + raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e)) path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/')))) return scheme, path, query @@ -233,7 +234,7 @@ def readline(self, buf): if line.endswith(b'\n'): line = buf.getvalue() if not line.endswith(b'\r\n'): - self.simple_response(httplib.BAD_REQUEST, 'HTTP requires CRLF line terminators') + self.simple_response(http_client.BAD_REQUEST, 'HTTP requires CRLF line terminators') return return line if not line: @@ -247,7 +248,7 @@ def connection_ready(self): self.forwarded_for = None self.path = self.query = None self.close_after_response = False - self.header_line_too_long_error_code = httplib.REQUEST_URI_TOO_LONG + self.header_line_too_long_error_code = http_client.REQUEST_URI_TOO_LONG self.response_started = False self.set_state(READ, self.parse_request_line, Accumulator(), first=True) @@ -260,28 +261,28 @@ def parse_request_line(self, buf, event, first=False): # {{{ # Ignore a single leading empty line, as per RFC 2616 sec 4.1 if first: return self.set_state(READ, self.parse_request_line, Accumulator()) - return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed') + return self.simple_response(http_client.BAD_REQUEST, 'Multiple leading empty lines not allowed') try: method, uri, req_protocol = line.strip().split(b' ', 2) rp = int(req_protocol[5]), int(req_protocol[7]) self.method = method.decode('ascii').upper() except Exception: - return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line") + return self.simple_response(http_client.BAD_REQUEST, "Malformed Request-Line") if self.method not in HTTP_METHODS: - return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method") + return self.simple_response(http_client.BAD_REQUEST, "Unknown HTTP method") try: self.request_protocol = protocol_map[rp] except KeyError: - return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED) + return self.simple_response(http_client.HTTP_VERSION_NOT_SUPPORTED) self.response_protocol = protocol_map[min((1, 1), rp)] try: self.scheme, self.path, self.query = parse_uri(uri) except HTTPSimpleResponse as e: return self.simple_response(e.http_code, e.message, close_after_response=False) - self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE + self.header_line_too_long_error_code = http_client.REQUEST_ENTITY_TOO_LARGE self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator()) # }}} @@ -299,7 +300,7 @@ def parse_header_line(self, parser, buf, event): try: parser(line) except ValueError: - self.simple_response(httplib.BAD_REQUEST, 'Failed to parse header line') + self.simple_response(http_client.BAD_REQUEST, 'Failed to parse header line') return if parser.finished: self.finalize_headers(parser.hdict) @@ -307,7 +308,7 @@ def parse_header_line(self, parser, buf, event): def finalize_headers(self, inheaders): request_content_length = int(inheaders.get('Content-Length', 0)) if request_content_length > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, "The entity sent with the request exceeds the maximum " "allowed bytes (%d)." % self.max_request_body_size) # Persistent connection support @@ -334,7 +335,7 @@ def finalize_headers(self, inheaders): else: # Note that, even if we see "chunked", we must reject # if there is an extension we don't recognize. - return self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) + return self.simple_response(http_client.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc) if inheaders.get("Expect", '').lower() == "100-continue": buf = BytesIO((HTTP11 + " 100 Continue\r\n\r\n").encode('ascii')) @@ -369,9 +370,9 @@ def read_chunk_length(self, inheaders, line_buf, buf, bytes_read, event): try: chunk_size = int(line.strip(), 16) except Exception: - return self.simple_response(httplib.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) + return self.simple_response(http_client.BAD_REQUEST, '%s is not a valid chunk size' % reprlib.repr(line.strip())) if bytes_read[0] + chunk_size + 2 > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if chunk_size == 0: self.set_state(READ, self.read_chunk_separator, inheaders, Accumulator(), buf, bytes_read, last=True) @@ -389,10 +390,10 @@ def read_chunk_separator(self, inheaders, line_buf, buf, bytes_read, event, last if line is None: return if line != b'\r\n': - return self.simple_response(httplib.BAD_REQUEST, 'Chunk does not have trailing CRLF') + return self.simple_response(http_client.BAD_REQUEST, 'Chunk does not have trailing CRLF') bytes_read[0] += len(line) if bytes_read[0] > self.max_request_body_size: - return self.simple_response(httplib.REQUEST_ENTITY_TOO_LARGE, + return self.simple_response(http_client.REQUEST_ENTITY_TOO_LARGE, 'Chunked request is larger than %d bytes' % self.max_request_body_size) if last: self.prepare_response(inheaders, buf) @@ -402,7 +403,7 @@ def read_chunk_separator(self, inheaders, line_buf, buf, bytes_read, event, last def handle_timeout(self): if self.response_started: return False - self.simple_response(httplib.REQUEST_TIMEOUT) + self.simple_response(http_client.REQUEST_TIMEOUT) return True def write(self, buf, end=None): diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index b64db6dab6..fc11fd0782 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, httplib, hashlib, uuid, struct, repr as reprlib +import os, hashlib, uuid, struct, repr as reprlib from collections import namedtuple from io import BytesIO, DEFAULT_BUFFER_SIZE from itertools import chain, repeat @@ -26,6 +26,7 @@ sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.monotonic import monotonic +from polyglot import http_client Range = namedtuple('Range', 'start stop size') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') @@ -226,7 +227,7 @@ def __init__(self, method, path, query, inheaders, request_body_file, outheaders self.remote_addr, self.remote_port, self.is_local_connection = remote_addr, remote_port, is_local_connection self.forwarded_for = forwarded_for self.opts = opts - self.status_code = httplib.OK + self.status_code = http_client.OK self.outcookie = Cookie() self.lang_code = self.gettext_func = self.ngettext_func = None self.set_translator(self.get_preferred_language()) @@ -402,16 +403,16 @@ def simple_response(self, status_code, msg='', close_after_response=True, extra_ if self.response_protocol is HTTP1: # HTTP/1.0 has no 413/414/303 codes status_code = { - httplib.REQUEST_ENTITY_TOO_LARGE:httplib.BAD_REQUEST, - httplib.REQUEST_URI_TOO_LONG:httplib.BAD_REQUEST, - httplib.SEE_OTHER:httplib.FOUND + http_client.REQUEST_ENTITY_TOO_LARGE:http_client.BAD_REQUEST, + http_client.REQUEST_URI_TOO_LONG:http_client.BAD_REQUEST, + http_client.SEE_OTHER:http_client.FOUND }.get(status_code, status_code) self.close_after_response = close_after_response msg = msg.encode('utf-8') ct = 'http' if self.method == 'TRACE' else 'plain' buf = [ - '%s %d %s' % (self.response_protocol, status_code, httplib.responses[status_code]), + '%s %d %s' % (self.response_protocol, status_code, http_client.responses[status_code]), "Content-Length: %s" % len(msg), "Content-Type: text/%s; charset=UTF-8" % ct, "Date: " + http_date(), @@ -432,7 +433,7 @@ def simple_response(self, status_code, msg='', close_after_response=True, extra_ def prepare_response(self, inheaders, request_body_file): if self.method == 'TRACE': msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty() - return self.simple_response(httplib.OK, msg, close_after_response=False) + return self.simple_response(http_client.OK, msg, close_after_response=False) request_body_file.seek(0) outheaders = MultiDict() data = RequestData( @@ -449,28 +450,28 @@ def run_request_handler(self, data): def send_range_not_satisfiable(self, content_length): buf = [ - '%s %d %s' % (self.response_protocol, httplib.REQUESTED_RANGE_NOT_SATISFIABLE, httplib.responses[httplib.REQUESTED_RANGE_NOT_SATISFIABLE]), + '%s %d %s' % (self.response_protocol, http_client.REQUESTED_RANGE_NOT_SATISFIABLE, http_client.responses[http_client.REQUESTED_RANGE_NOT_SATISFIABLE]), "Date: " + http_date(), "Content-Range: bytes */%d" % content_length, ] response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) + self.log_access(status_code=http_client.REQUESTED_RANGE_NOT_SATISFIABLE, response_size=response_data.sz) self.response_ready(response_data) def send_not_modified(self, etag=None): buf = [ - '%s %d %s' % (self.response_protocol, httplib.NOT_MODIFIED, httplib.responses[httplib.NOT_MODIFIED]), + '%s %d %s' % (self.response_protocol, http_client.NOT_MODIFIED, http_client.responses[http_client.NOT_MODIFIED]), "Content-Length: 0", "Date: " + http_date(), ] if etag is not None: buf.append('ETag: ' + etag) response_data = header_list_to_file(buf) - self.log_access(status_code=httplib.NOT_MODIFIED, response_size=response_data.sz) + self.log_access(status_code=http_client.NOT_MODIFIED, response_size=response_data.sz) self.response_ready(response_data) def report_busy(self): - self.simple_response(httplib.SERVICE_UNAVAILABLE) + self.simple_response(http_client.SERVICE_UNAVAILABLE) def job_done(self, ok, result): if not ok: @@ -509,7 +510,7 @@ def job_done(self, ok, result): if ct.startswith('text/') and 'charset=' not in ct: outheaders.set('Content-Type', ct + '; charset=UTF-8', replace_all=True) - buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]] + buf = [HTTP11 + (' %d ' % data.status_code) + http_client.responses[data.status_code]] for header, value in sorted(iteritems(outheaders), key=itemgetter(0)): buf.append('%s: %s' % (header, value)) for morsel in itervalues(data.outcookie): @@ -530,7 +531,7 @@ def job_done(self, ok, result): def log_access(self, status_code, response_size=None, username=None): if self.access_log is None: return - if not self.opts.log_not_found and status_code == httplib.NOT_FOUND: + if not self.opts.log_not_found and status_code == http_client.NOT_FOUND: return ff = self.forwarded_for if ff: @@ -623,7 +624,7 @@ def reset_state(self): self.ready = ready def report_unhandled_exception(self, e, formatted_traceback): - self.simple_response(httplib.INTERNAL_SERVER_ERROR) + self.simple_response(http_client.INTERNAL_SERVER_ERROR) def finalize_output(self, output, request, is_http1): none_match = parse_if_none_match(request.inheaders.get('If-None-Match', '')) @@ -633,7 +634,7 @@ def finalize_output(self, output, request, is_http1): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return opts = self.opts @@ -660,10 +661,10 @@ def finalize_output(self, output, request, is_http1): ct = outheaders.get('Content-Type', '').partition(';')[0] compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or ct.partition(';')[0] in COMPRESSIBLE_TYPES) - compressible = (compressible and request.status_code == httplib.OK and + compressible = (compressible and request.status_code == http_client.OK and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) - accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == httplib.OK and + accept_ranges = (not compressible and output.accept_ranges is not None and request.status_code == http_client.OK and not is_http1) ranges = get_ranges(request.inheaders.get('Range'), output.content_length) if output.accept_ranges and self.method in ('GET', 'HEAD') else None if_range = (request.inheaders.get('If-Range') or '').strip() @@ -680,7 +681,7 @@ def finalize_output(self, output, request, is_http1): if self.method in ('GET', 'HEAD'): self.send_not_modified(output.etag) else: - self.simple_response(httplib.PRECONDITION_FAILED) + self.simple_response(http_client.PRECONDITION_FAILED) return output.ranges = None @@ -712,7 +713,7 @@ def finalize_output(self, output, request, is_http1): outheaders.set('Content-Length', '%d' % size, replace_all=True) outheaders.set('Content-Type', 'multipart/byteranges; boundary=' + MULTIPART_SEPARATOR, replace_all=True) output.ranges = zip_longest(ranges, range_parts) - request.status_code = httplib.PARTIAL_CONTENT + request.status_code = http_client.PARTIAL_CONTENT return output diff --git a/src/calibre/srv/routes.py b/src/calibre/srv/routes.py index 9be1cde382..621b412c6c 100644 --- a/src/calibre/srv/routes.py +++ b/src/calibre/srv/routes.py @@ -6,13 +6,14 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, sys, inspect, re, time, numbers, json as jsonlib, textwrap +import sys, inspect, re, time, numbers, json as jsonlib, textwrap from operator import attrgetter from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.utils import http_date from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME from polyglot.builtins import iteritems, itervalues, unicode_type, range, zip +from polyglot import http_client from polyglot.urllib import quote as urlquote default_methods = frozenset(('HEAD', 'GET')) @@ -297,7 +298,7 @@ def read_cookies(self, data): def dispatch(self, data): endpoint_, args = self.find_route(data.path) if data.method not in endpoint_.methods: - raise HTTPSimpleResponse(httplib.METHOD_NOT_ALLOWED) + raise HTTPSimpleResponse(http_client.METHOD_NOT_ALLOWED) self.read_cookies(data) diff --git a/src/calibre/srv/tests/ajax.py b/src/calibre/srv/tests/ajax.py index 7707bd6d2b..0fade7e9d0 100644 --- a/src/calibre/srv/tests/ajax.py +++ b/src/calibre/srv/tests/ajax.py @@ -6,13 +6,13 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, base64, os +import zlib, json, base64, os from io import BytesIO from functools import partial -from httplib import OK, NOT_FOUND, FORBIDDEN from calibre.ebooks.metadata.meta import get_metadata from calibre.srv.tests.base import LibraryBaseTest +from polyglot.http_client import OK, NOT_FOUND, FORBIDDEN from polyglot.urllib import urlencode, quote @@ -22,7 +22,7 @@ def make_request(conn, url, headers={}, prefix='/ajax', username=None, password= conn.request(method, prefix + url, headers=headers, body=data) r = conn.getresponse() data = r.read() - if r.status == httplib.OK and data and data[0] in b'{[': + if r.status == OK and data and data[0] in b'{[': data = json.loads(data) return r, data @@ -37,10 +37,10 @@ def test_ajax_book(self): # {{{ request = partial(make_request, conn, prefix='/ajax/book') r, data = request('/x') - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, NOT_FOUND) r, onedata = request('/1') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(request('/1/' + db.server_library_id)[1], onedata) self.ae(request('/%s?id_is_uuid=true' % db.field_for('uuid', 1))[1], onedata) @@ -63,22 +63,22 @@ def test_ajax_categories(self): # {{{ request = partial(make_request, conn) r, data = request('/categories') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) r, xdata = request('/categories/' + db.server_library_id) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(data, xdata) names = {x['name']:x['url'] for x in data} for q in ('Newest', 'All books', 'Tags', 'Series', 'Authors', 'Enum', 'Composite Tags'): self.assertIn(q, names) r, data = request(names['Tags'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) names = {x['name']:x['url'] for x in data['items']} self.ae(set(names), set('Tag One,Tag Two,News'.split(','))) r, data = request(names['Tag One'], prefix='') - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'})) - self.ae(r.status, httplib.OK) + self.ae(r.status, OK) self.ae(set(data['book_ids']), {1, 2}) r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'})) self.ae(set(data['book_ids']), {2}) diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index ea08759d04..9407516cf0 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, base64, subprocess, os, cookielib, time +import base64, subprocess, os, cookielib, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -18,6 +18,7 @@ from calibre.srv.tests.base import BaseTest, TestServer from calibre.srv.routes import endpoint, Router from polyglot.builtins import iteritems, itervalues +from polyglot import http_client from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) @@ -91,18 +92,18 @@ def test_basic_auth(self): # {{{ conn = server.connect() conn.request('GET', '/open') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'open') conn.request('GET', '/closed') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) self.ae(r.getheader('WWW-Authenticate'), b'Basic realm="%s"' % bytes(REALM)) self.assertFalse(r.read()) conn.request('GET', '/closed', headers={'Authorization': b'Basic ' + base64.standard_b64encode(b'testuser:testpw')}) r = conn.getresponse() self.ae(r.read(), b'closed') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(b'closed', urlopen(server, method='basic').read()) self.ae(b'closed', urlopen(server, un='!@#$%^&*()-=_+', pw='!@#$%^&*()-=_+', method='basic').read()) @@ -113,14 +114,14 @@ def request(un='testuser', pw='testpw'): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('', '')) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('', '')) self.ae(1, len(warnings)) - self.ae((httplib.UNAUTHORIZED, b''), request('testuser', 'y')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) - self.ae((httplib.BAD_REQUEST, b'The username or password was empty'), request('')) - self.ae((httplib.UNAUTHORIZED, b''), request('asf', 'testpw')) + self.ae((http_client.UNAUTHORIZED, b''), request('testuser', 'y')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('testuser', '')) + self.ae((http_client.BAD_REQUEST, b'The username or password was empty'), request('')) + self.ae((http_client.UNAUTHORIZED, b''), request('asf', 'testpw')) # }}} def test_library_restrictions(self): # {{{ @@ -169,7 +170,7 @@ def test_digest_auth(self): # {{{ with TestServer(r.dispatch) as server: r.auth_controller.log = server.log - def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): + def test(conn, path, headers={}, status=http_client.OK, body=b'', request_body=b''): conn.request('GET', path, request_body, headers) r = conn.getresponse() self.ae(r.status, status) @@ -177,9 +178,9 @@ def test(conn, path, headers={}, status=httplib.OK, body=b'', request_body=b''): return {normalize_header_name(k):v for k, v in r.getheaders()} conn = server.connect() test(conn, '/open', body=b'open') - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) nonce = auth['nonce'] - auth = parse_http_dict(test(conn, '/closed', status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + auth = parse_http_dict(test(conn, '/closed', status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertNotEqual(nonce, auth['nonce'], 'nonce was re-used') self.ae(auth[b'realm'], bytes(REALM)), self.ae(auth[b'algorithm'], b'MD5'), self.ae(auth[b'qop'], b'auth') self.assertNotIn('stale', auth) @@ -199,14 +200,14 @@ def ok_test(conn, dh, **args): # Check stale nonces orig, r.auth_controller.max_age_seconds = r.auth_controller.max_age_seconds, -1 auth = parse_http_dict(test(conn, '/closed', headers={ - 'Authorization':digest(**args)},status=httplib.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) + 'Authorization':digest(**args)},status=http_client.UNAUTHORIZED)['WWW-Authenticate'].partition(b' ')[2]) self.assertIn('stale', auth) r.auth_controller.max_age_seconds = orig ok_test(conn, digest(**args)) def fail_test(conn, modify, **kw): kw['body'] = kw.get('body', b'') - kw['status'] = kw.get('status', httplib.UNAUTHORIZED) + kw['status'] = kw.get('status', http_client.UNAUTHORIZED) args['modify'] = modify return test(conn, '/closed', headers={'Authorization':digest(**args)}, **kw) @@ -258,13 +259,13 @@ def request(un='testuser', pw='testpw'): warnings = [] server.loop.log.warn = lambda *args, **kwargs: warnings.append(' '.join(args)) - self.ae((httplib.OK, b'closed'), request()) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae((httplib.UNAUTHORIZED, b''), request('x', 'y')) - self.ae(httplib.FORBIDDEN, request('x', 'y')[0]) - self.ae(httplib.FORBIDDEN, request()[0]) + self.ae((http_client.OK, b'closed'), request()) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae((http_client.UNAUTHORIZED, b''), request('x', 'y')) + self.ae(http_client.FORBIDDEN, request('x', 'y')[0]) + self.ae(http_client.FORBIDDEN, request()[0]) time.sleep(ban_for * 60 + 0.01) - self.ae((httplib.OK, b'closed'), request()) + self.ae((http_client.OK, b'closed'), request()) # }}} def test_android_auth_workaround(self): # {{{ @@ -277,7 +278,7 @@ def test_android_auth_workaround(self): # {{{ # First check that unauth access fails conn.request('GET', '/android') r = conn.getresponse() - self.ae(r.status, httplib.UNAUTHORIZED) + self.ae(r.status, http_client.UNAUTHORIZED) auth_handler = HTTPDigestAuthHandler() url = 'http://localhost:%d%s' % (server.address[1], '/android') @@ -285,20 +286,20 @@ def test_android_auth_workaround(self): # {{{ cj = cookielib.CookieJar() cookie_handler = HTTPCookieProcessor(cj) r = build_opener(auth_handler, cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) cookies = tuple(cj) self.ae(len(cookies), 1) cookie = cookies[0] self.assertIn(b':', cookie.value) self.ae(cookie.path, b'/android') r = build_opener(cookie_handler).open(url) - self.ae(r.getcode(), httplib.OK) + self.ae(r.getcode(), http_client.OK) self.ae(r.read(), b'android') # Test that a replay attack against a different URL does not work try: build_opener(cookie_handler).open(url+'2') assert ('Replay attack succeeded') except HTTPError as e: - self.ae(e.code, httplib.UNAUTHORIZED) + self.ae(e.code, http_client.UNAUTHORIZED) # }}} diff --git a/src/calibre/srv/tests/base.py b/src/calibre/srv/tests/base.py index e62dcc605d..165aa2e0c5 100644 --- a/src/calibre/srv/tests/base.py +++ b/src/calibre/srv/tests/base.py @@ -7,12 +7,13 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import unittest, time, httplib, shutil, gc, tempfile, atexit, os +import unittest, time, shutil, gc, tempfile, atexit, os from io import BytesIO from functools import partial from threading import Thread from calibre.srv.utils import ServerLog +from polyglot import http_client rmtree = partial(shutil.rmtree, ignore_errors=True) @@ -120,7 +121,7 @@ def connect(self, timeout=None, interface=None): timeout = self.loop.opts.timeout if interface is None: interface = self.address[0] - return httplib.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) + return http_client.HTTPConnection(interface, self.address[1], strict=True, timeout=timeout) def change_handler(self, handler): from calibre.srv.http_response import create_http_handler diff --git a/src/calibre/srv/tests/content.py b/src/calibre/srv/tests/content.py index e7b1950eec..fa9a12ccc7 100644 --- a/src/calibre/srv/tests/content.py +++ b/src/calibre/srv/tests/content.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, zlib, json, binascii, time, os +import zlib, json, binascii, time, os from io import BytesIO from calibre.ebooks.metadata.epub import get_metadata @@ -14,6 +14,7 @@ from calibre.srv.tests.base import LibraryBaseTest from calibre.utils.imghdr import identify from calibre.utils.shared_file import share_open +from polyglot import http_client def setUpModule(): @@ -32,7 +33,7 @@ def test_static(self): # {{{ def missing(url, body=b''): conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), body) for prefix in ('static', 'icon'): @@ -51,7 +52,7 @@ def test(src, url, sz=None): raw = P(src, data=True) conn.request('GET', url) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) data = r.read() if sz is None: self.ae(data, raw) @@ -60,7 +61,7 @@ def test(src, url, sz=None): test_response(r) conn.request('GET', url, headers={'If-None-Match':r.getheader('ETag')}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(b'', r.read()) test('content-server/empty.html', '/static/empty.html') @@ -85,7 +86,7 @@ def get(what, book_id, library_id=None, q=''): # Test various invalid parameters def bad(*args): r, data = get(*args) - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) bad('xxx', 1) bad('fmt1', 10) bad('fmt1', 1, 'zzzz') @@ -103,7 +104,7 @@ def bad(*args): # Test fetching of format with metadata update raw = P('quick_start/eng.epub', data=True) r, data = get('epub', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) etag = r.getheader('ETag') self.assertIsNotNone(etag) self.ae(r.getheader('Used-Cache'), 'no') @@ -145,39 +146,39 @@ def change_cover(count, book_id=2): os.utime(cpath, (t, t)) r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(r.getheader('Content-Type'), 'image/jpeg') r, data = get('cover', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(1)) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('cover', 3) - self.ae(r.status, httplib.OK) # Auto generated cover + self.ae(r.status, http_client.OK) # Auto generated cover r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 60, 60)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') r, data = get('thumb', 1, q='sz=100x100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Used-Cache'), 'yes') change_cover(1, 1) r, data = get('thumb', 1, q='sz=100') - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(identify(data), ('jpeg', 100, 100)) self.ae(r.getheader('Used-Cache'), 'no') # Test file sharing in cache r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -185,7 +186,7 @@ def change_cover(count, book_id=2): # Now force an update change_cover(1) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') path = binascii.unhexlify(r.getheader('Tempfile')).decode('utf-8') @@ -193,7 +194,7 @@ def change_cover(count, book_id=2): # Do it again change_cover(2) r, data = get('cover', 2) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(data, db.cover(2)) self.ae(r.getheader('Used-Cache'), 'no') self.ae(f.read(), fdata) @@ -201,7 +202,7 @@ def change_cover(count, book_id=2): # Test serving of metadata as opf r, data = get('opf', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.getheader('Content-Type'), 'application/oebps-package+xml; charset=UTF-8') self.assertIsNotNone(r.getheader('Last-Modified')) opf = OPF(BytesIO(data), populate_spine=False, try_to_guess_cover=False) @@ -209,17 +210,17 @@ def change_cover(count, book_id=2): self.ae(db.field_for('authors', 1), tuple(opf.authors)) conn.request('GET', '/get/opf/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) # Test serving metadata as json r, data = get('json', 1) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(db.field_for('title', 1), json.loads(data)['title']) conn.request('GET', '/get/json/1', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') + self.ae(r.status, http_client.OK), self.ae(r.getheader('Content-Encoding'), 'gzip') raw = r.read() self.ae(zlib.decompress(raw, 16+zlib.MAX_WBITS), data) diff --git a/src/calibre/srv/tests/http.py b/src/calibre/srv/tests/http.py index f42279795e..05b487b2a3 100644 --- a/src/calibre/srv/tests/http.py +++ b/src/calibre/srv/tests/http.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, hashlib, zlib, string, time, os +import hashlib, zlib, string, time, os from io import BytesIO from tempfile import NamedTemporaryFile @@ -15,6 +15,7 @@ from calibre.srv.utils import eintr_retry_call from calibre.utils.monotonic import monotonic from polyglot.builtins import iteritems, range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -94,7 +95,7 @@ def handler(data): def test(al, q): conn.request('GET', '/', headers={'Accept-Language': al}) r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) q += get_translator(q)[-1].ugettext('Unknown') self.ae(r.read(), q) @@ -136,7 +137,7 @@ def handler(data): def raw_send(conn, raw): conn.send(raw) - conn._HTTPConnection__state = httplib._CS_REQ_SENT + conn._HTTPConnection__state = http_client._CS_REQ_SENT return conn.getresponse() base_timeout = 0.5 if is_ci else 0.1 @@ -144,31 +145,31 @@ def raw_send(conn, raw): with TestServer(handler, timeout=base_timeout, max_header_line_size=100./1024, max_request_body_size=100./(1024*1024)) as server: conn = server.connect() r = raw_send(conn, b'hello\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'HTTP requires CRLF line terminators') r = raw_send(conn, b'\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') + self.ae(r.status, http_client.NOT_FOUND), self.ae(r.read(), b'Requested resource not found') r = raw_send(conn, b'\r\n\r\nGET /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Multiple leading empty lines not allowed') r = raw_send(conn, b'hello world\r\n') - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'Malformed Request-Line') r = raw_send(conn, b'x' * 200) - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.ae(r.read(), b'') r = raw_send(conn, b'XXX /index.html HTTP/1.1\r\n\r\n') - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Unknown HTTP method') # Test 404 conn.request('HEAD', '/moose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.assertIsNotNone(r.getheader('Date', None)) self.ae(r.getheader('Content-Length'), str(len(body))) self.ae(r.getheader('Content-Type'), 'text/plain; charset=UTF-8') @@ -176,7 +177,7 @@ def raw_send(conn, raw): self.ae(r.read(), '') conn.request('GET', '/choose') r = conn.getresponse() - self.ae(r.status, httplib.NOT_FOUND) + self.ae(r.status, http_client.NOT_FOUND) self.ae(r.read(), b'Requested resource not found') # Test 500 @@ -186,7 +187,7 @@ def raw_send(conn, raw): conn = server.connect() conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.INTERNAL_SERVER_ERROR) + self.ae(r.status, http_client.INTERNAL_SERVER_ERROR) server.loop.log.filter_level = orig # Test 301 @@ -196,7 +197,7 @@ def handler(data): conn = server.connect() conn.request('GET', '/') r = conn.getresponse() - self.ae(r.status, httplib.MOVED_PERMANENTLY) + self.ae(r.status, http_client.MOVED_PERMANENTLY) self.ae(r.getheader('Location'), '/somewhere-else') self.ae('', r.read()) @@ -206,26 +207,26 @@ def handler(data): # Test simple GET conn.request('GET', '/test/') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'test') # Test TRACE lines = ['TRACE /xxx HTTP/1.1', 'Test: value', 'Xyz: abc, def', '', ''] r = raw_send(conn, ('\r\n'.join(lines)).encode('ascii')) - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read().decode('utf-8'), '\n'.join(lines[:-2])) # Test POST with simple body conn.request('POST', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') # Test POST with chunked transfer encoding conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\na\r\n1234567890\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody1234567890') # Test various incorrect input @@ -233,39 +234,39 @@ def handler(data): conn.request('GET', '/test' + ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) conn = server.connect() conn.request('GET', '/test', ('a' * 200)) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'x\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST) + self.ae(r.status, http_client.BAD_REQUEST) self.assertIn(b'not a valid chunk size', r.read()) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'4\r\nbody\r\n200\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn.request('POST', '/test', body='a'*200) r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_ENTITY_TOO_LARGE) + self.ae(r.status, http_client.REQUEST_ENTITY_TOO_LARGE) conn = server.connect() conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'3\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') + self.ae(r.status, http_client.BAD_REQUEST), self.ae(r.read(), b'Chunk does not have trailing CRLF') conn = server.connect(timeout=base_timeout * 5) conn.request('POST', '/test', headers={'Transfer-Encoding': 'chunked'}) conn.send(b'30\r\nbody\r\n0\r\n\r\n') r = conn.getresponse() - self.ae(r.status, httplib.REQUEST_TIMEOUT) + self.ae(r.status, http_client.REQUEST_TIMEOUT) self.assertIn(b'', r.read()) server.log.filter_level = orig_level @@ -273,14 +274,14 @@ def handler(data): # Test pipelining responses = [] for i in range(10): - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE conn.request('GET', '/%d'%i) responses.append(conn.response_class(conn.sock, strict=conn.strict, method=conn._method)) for i in range(10): r = responses[i] r.begin() self.ae(r.read(), ('%d' % i).encode('ascii')) - conn._HTTPConnection__state = httplib._CS_IDLE + conn._HTTPConnection__state = http_client._CS_IDLE # Test closing server.loop.opts.timeout = 10 # ensure socket is not closed because of timeout @@ -319,12 +320,12 @@ def handler(conn): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'an_etagged_path') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'an_etagged_path') etag = r.getheader('ETag') self.ae(etag, '"%s"' % hashlib.sha1('an_etagged_path').hexdigest()) conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') # Test gzip @@ -334,7 +335,7 @@ def handler(conn): conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) r = conn.getresponse() self.ae(str(len(raw)), r.getheader('Calibre-Uncompressed-Length')) - self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) + self.ae(r.status, http_client.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw) # Test dynamic etagged content num_calls = [0] @@ -346,13 +347,13 @@ def edfunc(): conn = server.connect() conn.request('GET', '/an_etagged_path') r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), b'data') + self.ae(r.status, http_client.OK), self.ae(r.read(), b'data') etag = r.getheader('ETag') self.ae(etag, b'"xxx"') self.ae(r.getheader('Content-Length'), '4') conn.request('GET', '/an_etagged_path', headers={'If-None-Match':etag}) r = conn.getresponse() - self.ae(r.status, httplib.NOT_MODIFIED) + self.ae(r.status, http_client.NOT_MODIFIED) self.ae(r.read(), b'') self.ae(num_calls[0], 1) @@ -368,11 +369,11 @@ def edfunc(): self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(int(r.getheader('Content-Length')), len(fdata)) - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=2-25'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') self.ae(type('')(r.getheader('Content-Range')), 'bytes 2-25/%d' % len(fdata)) self.ae(int(r.getheader('Content-Length')), 24) @@ -380,27 +381,27 @@ def edfunc(): conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) r = conn.getresponse() - self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) + self.ae(r.status, http_client.REQUESTED_RANGE_NOT_SATISFIABLE) self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) self.ae(int(r.getheader('Content-Length')), 26) conn.request('GET', '/test', headers={'Range':'bytes=0-1000000'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata) + self.ae(r.status, http_client.PARTIAL_CONTENT), self.ae(r.read(), fdata) conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) r = conn.getresponse() - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + self.ae(r.status, http_client.OK), self.ae(r.read(), fdata) self.assertFalse(r.getheader('Content-Range')) self.ae(int(r.getheader('Content-Length')), len(fdata)) conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) r = conn.getresponse() - self.ae(r.status, httplib.PARTIAL_CONTENT) + self.ae(r.status, http_client.PARTIAL_CONTENT) clen = int(r.getheader('Content-Length')) data = r.read() self.ae(clen, len(data)) @@ -415,7 +416,7 @@ def edfunc(): conn = server.connect(timeout=1) conn.request('GET', '/test') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) rdata = r.read() self.ae(len(data), len(rdata)) self.ae(hashlib.sha1(data).hexdigest(), hashlib.sha1(rdata).hexdigest()) diff --git a/src/calibre/srv/tests/loop.py b/src/calibre/srv/tests/loop.py index af628456c7..897721cc40 100644 --- a/src/calibre/srv/tests/loop.py +++ b/src/calibre/srv/tests/loop.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import httplib, ssl, os, socket, time +import ssl, os, socket, time from collections import namedtuple from unittest import skipIf from glob import glob @@ -18,6 +18,7 @@ from calibre.utils.certgen import create_server_cert from calibre.utils.monotonic import monotonic from polyglot.builtins import range +from polyglot import http_client is_ci = os.environ.get('CI', '').lower() == 'true' @@ -92,7 +93,7 @@ def test_workers(self): conn.request('GET', '/') with self.assertRaises(socket.timeout): res = conn.getresponse() - if str(res.status) == str(httplib.REQUEST_TIMEOUT): + if str(res.status) == str(http_client.REQUEST_TIMEOUT): raise socket.timeout('Timeout') raise Exception('Got unexpected response: code: %s %s headers: %r data: %r' % ( res.status, res.reason, res.getheaders(), res.read())) @@ -135,7 +136,7 @@ def test_dual_stack(self): conn = server.connect(interface='127.0.0.1') conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') def test_ring_buffer(self): @@ -203,10 +204,10 @@ def test_ssl(self): create_server_cert(address, ca_file, cert_file, key_file, key_size=1024) ctx = ssl.create_default_context(cafile=ca_file) with TestServer(lambda data:(data.path[0] + data.read()), ssl_certfile=cert_file, ssl_keyfile=key_file, listen_on=address, port=0) as server: - conn = httplib.HTTPSConnection(address, server.address[1], strict=True, context=ctx) + conn = http_client.HTTPSConnection(address, server.address[1], strict=True, context=ctx) conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') cert = conn.sock.getpeercert() subject = dict(x[0] for x in cert['subject']) @@ -226,7 +227,7 @@ def test_socket_activation(self): conn = server.connect() conn.request('GET', '/test', 'body') r = conn.getresponse() - self.ae(r.status, httplib.OK) + self.ae(r.status, http_client.OK) self.ae(r.read(), b'testbody') self.ae(server.loop.bound_address[1], port) diff --git a/src/calibre/srv/web_socket.py b/src/calibre/srv/web_socket.py index 5fdf7ab309..59a5cfc9f8 100644 --- a/src/calibre/srv/web_socket.py +++ b/src/calibre/srv/web_socket.py @@ -5,7 +5,7 @@ from __future__ import (unicode_literals, division, absolute_import, print_function) -import httplib, os, weakref, socket +import os, weakref, socket from base64 import standard_b64encode from collections import deque from hashlib import sha1 @@ -19,6 +19,7 @@ from calibre.srv.utils import DESIRED_SEND_BUFFER_SIZE from calibre.utils.speedups import ReadOnlyFileBuffer from polyglot.queue import Queue, Empty +from polyglot import http_client speedup, err = plugins['speedup'] if not speedup: raise RuntimeError('Failed to load speedup module with error: ' + err) @@ -286,9 +287,9 @@ def finalize_headers(self, inheaders): except Exception: ver_ok = False if not ver_ok: - return self.simple_response(httplib.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) + return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver) if self.method != 'GET': - return self.simple_response(httplib.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) + return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method) response = HANDSHAKE_STR % standard_b64encode(sha1(key + GUID_STR).digest()) self.optimize_for_sending_packet() diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index def783eaee..47835dec1e 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -5,11 +5,13 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, httplib, ssl +import copy, ssl from cookielib import CookieJar, Cookie from mechanize import Browser as B, HTTPSHandler +from polyglot import http_client + class ModernHTTPSHandler(HTTPSHandler): @@ -24,7 +26,7 @@ def https_open(self, req): def conn_factory(hostport, **kw): kw['context'] = self.ssl_context - return httplib.HTTPSConnection(hostport, **kw) + return http_client.HTTPSConnection(hostport, **kw) return self.do_open(conn_factory, req) diff --git a/src/calibre/utils/https.py b/src/calibre/utils/https.py index 690c1f454b..9c26970d23 100644 --- a/src/calibre/utils/https.py +++ b/src/calibre/utils/https.py @@ -10,7 +10,7 @@ from contextlib import closing from calibre import get_proxies -from calibre.constants import ispy3 +from polyglot import http_client from polyglot.urllib import urlsplit has_ssl_verify = hasattr(ssl, 'create_default_context') and hasattr(ssl, '_create_unverified_context') @@ -19,19 +19,14 @@ class HTTPError(ValueError): def __init__(self, url, code): msg = '%s returned an unsupported http response code: %d (%s)' % ( - url, code, httplib.responses.get(code, None)) + url, code, http_client.responses.get(code, None)) ValueError.__init__(self, msg) self.code = code self.url = url -if ispy3: - import http.client as httplib -else: - import httplib - if has_ssl_verify: - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): cafile = kwargs.pop('cert_file', None) @@ -39,7 +34,7 @@ def __init__(self, ssl_version, *args, **kwargs): kwargs['context'] = ssl._create_unverified_context() else: kwargs['context'] = ssl.create_default_context(cafile=cafile) - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) else: # Check certificate hostname {{{ # Implementation taken from python 3 @@ -136,10 +131,10 @@ def match_hostname(cert, hostname): "subjectAltName fields were found") # }}} - class HTTPSConnection(httplib.HTTPSConnection): + class HTTPSConnection(http_client.HTTPSConnection): def __init__(self, ssl_version, *args, **kwargs): - httplib.HTTPSConnection.__init__(self, *args, **kwargs) + http_client.HTTPSConnection.__init__(self, *args, **kwargs) self.calibre_ssl_version = ssl_version def connect(self): @@ -204,7 +199,7 @@ def get_https_resource_securely( path += '?' + p.query c.request('GET', path, headers=headers or {}) response = c.getresponse() - if response.status in (httplib.MOVED_PERMANENTLY, httplib.FOUND, httplib.SEE_OTHER): + if response.status in (http_client.MOVED_PERMANENTLY, http_client.FOUND, http_client.SEE_OTHER): if max_redirects <= 0: raise ValueError('Too many redirects, giving up') newurl = response.getheader('Location', None) @@ -212,7 +207,7 @@ def get_https_resource_securely( raise ValueError('%s returned a redirect response with no Location header' % url) return get_https_resource_securely( newurl, cacerts=cacerts, timeout=timeout, max_redirects=max_redirects-1, ssl_version=ssl_version, get_response=get_response) - if response.status != httplib.OK: + if response.status != http_client.OK: raise HTTPError(url, response.status) if get_response: return response diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index e9a5c1457f..852b00b6fa 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -18,7 +18,6 @@ import time import traceback from base64 import b64decode -from httplib import responses from calibre import browser, relpath, unicode_path from calibre.constants import filesystem_encoding, iswindows @@ -31,6 +30,7 @@ from calibre.utils.logging import Log from calibre.web.fetch.utils import rescale_image from polyglot.builtins import unicode_type +from polyglot.http_client import responses from polyglot.urllib import ( URLError, quote, url2pathname, urljoin, urlparse, urlsplit, urlunparse, urlunsplit From 727e65a203c5f4ea1aff5a64accc65e8672207cd Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 12:07:48 -0400 Subject: [PATCH 04/19] python3: add httplib/http.client polyglot wrapper --- src/polyglot/http_client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/polyglot/http_client.py diff --git a/src/polyglot/http_client.py b/src/polyglot/http_client.py new file mode 100644 index 0000000000..740d9799ff --- /dev/null +++ b/src/polyglot/http_client.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from http.client import (responses, HTTPConnection, HTTPSConnection, + BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED, + INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY, + NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT, + PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, + REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER, + SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT) +else: + from httplib import (responses, HTTPConnection, HTTPSConnection, + BAD_REQUEST, FOUND, FORBIDDEN, HTTP_VERSION_NOT_SUPPORTED, + INTERNAL_SERVER_ERROR, METHOD_NOT_ALLOWED, MOVED_PERMANENTLY, + NOT_FOUND, NOT_IMPLEMENTED, NOT_MODIFIED, OK, PARTIAL_CONTENT, + PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, + REQUESTED_RANGE_NOT_SATISFIABLE, REQUEST_TIMEOUT, SEE_OTHER, + SERVICE_UNAVAILABLE, UNAUTHORIZED, _CS_IDLE, _CS_REQ_SENT) From 6d6509df577bdd9d0aaee7a276d100baa76099a1 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 25 Mar 2019 23:57:10 -0400 Subject: [PATCH 05/19] python3: make qt image test work on python3 QByteArrays are natively bytes and ended up looking like this: u"b'xpm'" Instead use the builtin method to get the data back, and decode it to unicode. --- src/calibre/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 7ce4075714..17259fecd0 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -175,7 +175,7 @@ def test_qt(self): # it should just work because the hard-coded paths of the Qt # installation should work. If they do not, then it is a distro # problem. - fmts = set(map(unicode_type, QImageReader.supportedImageFormats())) + fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats())) testf = {'jpg', 'png', 'svg', 'ico', 'gif'} self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts) data = P('images/blank.png', allow_user_override=False, data=True) From 7196fa0773946a4eac62d695bfa525f050231ce8 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:02:45 -0400 Subject: [PATCH 06/19] python3: mark zlib plugin as only existing on python2 Fixes test failure due to checking whether all plugins work, and considering this a plugin even though it is not. --- src/calibre/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index e5781fe5b3..05ac5bbb3e 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -168,7 +168,6 @@ def __init__(self): 'icu', 'speedup', 'unicode_names', - 'zlib2', 'html', 'freetype', 'imageops', @@ -184,6 +183,7 @@ def __init__(self): if not ispy3: plugins.extend([ 'monotonic', + 'zlib2', ]) if iswindows: plugins.extend(['winutil', 'wpd', 'winfonts']) From b6d4442b23b75834d802823206db68db525022a2 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:04:19 -0400 Subject: [PATCH 07/19] python3: mark bytestring as needed --- src/calibre/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 17259fecd0..45c85d1bf8 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -99,7 +99,7 @@ def test_lxml(self): from calibre.utils.cleantext import test_clean_xml_chars test_clean_xml_chars() from lxml import etree - raw = '' + raw = b'' root = etree.fromstring(raw) self.assertEqual(etree.tostring(root), raw) From 2271463d2746d7cc2bea411db46bc60ebcacaadc Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:10:02 -0400 Subject: [PATCH 08/19] python3: don't require bytestring for unrardll comments unrardll explictly decodes to unicode Note: unrar.py already uses unicode_literals --- src/calibre/utils/unrar.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/utils/unrar.py b/src/calibre/utils/unrar.py index 882a636ba6..fc6a767843 100644 --- a/src/calibre/utils/unrar.py +++ b/src/calibre/utils/unrar.py @@ -114,20 +114,20 @@ def test_basic(): b"Rar!\x1a\x07\x00\xcf\x90s\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x14\xe7z\x00\x80#\x00\x17\x00\x00\x00\r\x00\x00\x00\x03\xc2\xb3\x96o\x00\x00\x00\x00\x1d3\x03\x00\x00\x00\x00\x00CMT\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe6h\x04\x17\xff\xcd\x0f\xffk9b\x11]^\x80\xd3dt \x90+\x00\x14\x00\x00\x00\x08\x00\x00\x00\x03\xf1\x84\x93\\\xb9]yA\x1d3\t\x00\xa4\x81\x00\x001\\sub-one\x00\xc0\x0c\x00\x8f\xec\x89\xfe.JM\x86\x82\x0c_\xfd\xfd\xd7\x11\x1a\xef@\x9eHt \x80'\x00\x0e\x00\x00\x00\x04\x00\x00\x00\x03\x9f\xa8\x17\xf8\xaf]yA\x1d3\x07\x00\xa4\x81\x00\x00one.txt\x00\x08\xbf\x08\xae\xf3\xca\x87\xfeo\xfe\xd2n\x80-Ht \x82:\x00\x18\x00\x00\x00\x10\x00\x00\x00\x03\xa86\x81\xdf\xf9fyA\x1d3\x1a\x00\xa4\x81\x00\x00\xe8\xaf\xb6\xe6\xaf\x94\xe5\xb1\x81.txt\x00\x8bh\xf6\xd4kA\\.\x00txt\x0c\x00\x8b\xec\x8e\xef\x14\xf6\xe2l\x91\x189\xff\xdf\xfe\xc2\xd3:g\x9a\x19F=cYt \x928\x00\x11\x00\x00\x00\x08\x00\x00\x00\x03\x7f\xd6\xb6\x7f\xeafyA\x1d3\x16\x00\xa4\x81\x00\x00F\xc3\xbc\xc3\x9fe.txt\x00\x01\x00F\xfc\xdfe\x00.txt\x00\xc0 Date: Tue, 26 Mar 2019 01:20:18 -0400 Subject: [PATCH 09/19] test: fix broken netifaces check This will always return True on python2 [] > 1 On python3 it is instead an error, because you cannot compare a list to an int -- instead, we want to compare the length of the returned list, to see how many interfaces are available. --- src/calibre/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 45c85d1bf8..2ba6492114 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -254,7 +254,7 @@ def test_executables(self): def test_netifaces(self): import netifaces - self.assertGreaterEqual(netifaces.interfaces(), 1, 'netifaces could find no network interfaces') + self.assertGreaterEqual(len(netifaces.interfaces()), 1, 'netifaces could find no network interfaces') def test_psutil(self): import psutil From 696afe85bdbfe8e231d7484b6b8504f7acf39ea9 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 01:31:34 -0400 Subject: [PATCH 10/19] python3: convert filter iterables to list in order to access as a list --- src/calibre/spell/dictionary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/spell/dictionary.py b/src/calibre/spell/dictionary.py index 598c6e302d..5086dc08b4 100644 --- a/src/calibre/spell/dictionary.py +++ b/src/calibre/spell/dictionary.py @@ -54,7 +54,7 @@ def builtin_dictionaries(): if _builtins is None: dics = [] for lc in glob.glob(os.path.join(P('dictionaries', allow_user_override=False), '*/locales')): - locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) + locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())) locale = locales[0] base = os.path.dirname(lc) dics.append(Dictionary( @@ -69,7 +69,7 @@ def custom_dictionaries(reread=False): if _custom is None or reread: dics = [] for lc in glob.glob(os.path.join(config_dir, 'dictionaries', '*/locales')): - locales = filter(None, open(lc, 'rb').read().decode('utf-8').splitlines()) + locales = list(filter(None, open(lc, 'rb').read().decode('utf-8').splitlines())) try: name, locale, locales = locales[0], locales[1], locales[1:] except IndexError: From 564f053c026fe3946d5c9c3e915ab4cbcd2694d2 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 03:15:21 -0400 Subject: [PATCH 11/19] python3: add reprlib wrapper to polyglot --- src/calibre/__init__.py | 2 +- src/calibre/db/tests/legacy.py | 4 ++-- src/calibre/library/sqlite.py | 2 +- src/calibre/srv/http_request.py | 4 ++-- src/calibre/srv/http_response.py | 4 ++-- src/calibre/srv/utils.py | 2 +- src/polyglot/reprlib.py | 10 ++++++++++ 7 files changed, 19 insertions(+), 9 deletions(-) create mode 100755 src/polyglot/reprlib.py diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index c71af8e0f4..11a2317d69 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -240,7 +240,7 @@ def prints(*args, **kwargs): file.write(arg) count += len(arg) except: - import repr as reprlib + from polyglot import reprlib arg = reprlib.repr(arg) file.write(arg) count += len(arg) diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index b0f88feb30..b7406bbca8 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -8,13 +8,13 @@ import inspect, time, numbers from io import BytesIO -from repr import repr from functools import partial from operator import itemgetter from calibre.library.field_metadata import fm_as_dict from calibre.db.tests.base import BaseTest from polyglot.builtins import iteritems, range +from polyglot import reprlib # Utils {{{ @@ -32,7 +32,7 @@ def __call__(self, test): oldres = getattr(old, self.func_name)(*self.args, **self.kwargs) newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs) test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % ( - self.func_name, repr(self.args), repr(self.kwargs))) + self.func_name, reprlib.repr(self.args), reprlib.repr(self.kwargs))) self.retval = newres return newres diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index d876e96ccf..488925fe76 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -8,7 +8,6 @@ all calls. ''' import sqlite3 as sqlite, traceback, time, uuid, sys, os -import repr as reprlib from sqlite3 import IntegrityError, OperationalError from threading import Thread from threading import RLock @@ -22,6 +21,7 @@ from calibre.utils.icu import sort_key from calibre import prints from polyglot.builtins import unicode_type +from polyglot import reprlib from polyglot.queue import Queue from dateutil.tz import tzoffset diff --git a/src/calibre/srv/http_request.py b/src/calibre/srv/http_request.py index 74ac506790..02afb5252f 100644 --- a/src/calibre/srv/http_request.py +++ b/src/calibre/srv/http_request.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import re, repr as reprlib +import re from io import BytesIO, DEFAULT_BUFFER_SIZE from calibre import as_unicode, force_unicode @@ -14,7 +14,7 @@ from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.loop import Connection, READ, WRITE from calibre.srv.utils import MultiDict, HTTP1, HTTP11, Accumulator -from polyglot import http_client +from polyglot import http_client, reprlib from polyglot.urllib import unquote protocol_map = {(1, 0):HTTP1, (1, 1):HTTP11} diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index fc11fd0782..b5b16fbe8f 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, hashlib, uuid, struct, repr as reprlib +import os, hashlib, uuid, struct from collections import namedtuple from io import BytesIO, DEFAULT_BUFFER_SIZE from itertools import chain, repeat @@ -26,7 +26,7 @@ sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) from calibre.utils.speedups import ReadOnlyFileBuffer from calibre.utils.monotonic import monotonic -from polyglot import http_client +from polyglot import http_client, reprlib Range = namedtuple('Range', 'start stop size') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index 7e74d2b057..0037e4e8a7 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -9,7 +9,6 @@ import errno, socket, select, os, time from Cookie import SimpleCookie from contextlib import closing -import repr as reprlib from email.utils import formatdate from operator import itemgetter from binascii import hexlify, unhexlify @@ -23,6 +22,7 @@ from calibre.utils.logging import ThreadSafeLog from calibre.utils.shared_file import share_open, raise_winerror from polyglot.builtins import iteritems, map, unicode_type, range +from polyglot import reprlib from polyglot.urllib import parse_qs, quote as urlquote HTTP1 = 'HTTP/1.0' diff --git a/src/polyglot/reprlib.py b/src/polyglot/reprlib.py new file mode 100755 index 0000000000..2444ad1788 --- /dev/null +++ b/src/polyglot/reprlib.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from reprlib import repr +else: + from repr import repr From 67f3ca17ddc878515daf99f7949bde77eb8f30f0 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 02:17:33 -0400 Subject: [PATCH 12/19] test: add test to try importing every file The testsuite does not currently exercise every file, but we can at least try importing it to make sure it isn't obviously broken. This additionally helps to iterate through python3 syntax-level incompatibilities. --- src/calibre/test_build.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 2ba6492114..397299264b 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -73,6 +73,27 @@ def test_html5_parser(self): from html5_parser import parse parse('

xxx') + def test_imports(self): + import importlib + exclude = ['dbus_export.demo', 'dbus_export.gtk', 'upstream'] + if not iswindows: + exclude.extend(['iphlpapi', 'windows', 'winreg', 'winusb']) + if not isosx: + exclude.append('osx') + if not islinux: + exclude.extend(['dbus', 'linux']) + for root, dirs, files in os.walk('src/calibre'): + for dir in dirs: + if not os.path.isfile(os.path.join(root, dir, '__init__.py')): + dirs.remove(dir) + for file in files: + file, ext = os.path.splitext(file) + if ext != '.py': + continue + name = '.'.join(root.split(os.path.sep)[1:] + [file]) + if not any(x for x in exclude if x in name): + importlib.import_module(name) + def test_plugins(self): exclusions = set() if is_ci: From 06d8560ab2685f22168b502236bb64d96ed52c36 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 02:24:48 -0400 Subject: [PATCH 13/19] remove unused files which import non-existing code --- src/calibre/ebooks/lit/from_any.py | 67 ----------- src/calibre/ebooks/lrf/html/convert_to.py | 128 ---------------------- src/calibre/ebooks/pdf/from_comic.py | 21 ---- 3 files changed, 216 deletions(-) delete mode 100644 src/calibre/ebooks/lit/from_any.py delete mode 100644 src/calibre/ebooks/lrf/html/convert_to.py delete mode 100644 src/calibre/ebooks/pdf/from_comic.py diff --git a/src/calibre/ebooks/lit/from_any.py b/src/calibre/ebooks/lit/from_any.py deleted file mode 100644 index 05345b6749..0000000000 --- a/src/calibre/ebooks/lit/from_any.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import with_statement -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Convert any ebook format to LIT. -''' - -import sys, os, glob, logging - -from calibre.ebooks.epub.from_any import any2epub, formats, USAGE -from calibre.ebooks.epub import config as common_config -from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks.lit.writer import oeb2lit - - -def config(defaults=None): - c = common_config(defaults=defaults, name='lit') - return c - - -def option_parser(usage=USAGE): - return config().option_parser(usage=usage%('LIT', formats())) - - -def any2lit(opts, path): - ext = os.path.splitext(path)[1] - if not ext: - raise ValueError('Unknown file type: '+path) - ext = ext.lower()[1:] - - if opts.output is None: - opts.output = os.path.splitext(os.path.basename(path))[0]+'.lit' - - opts.output = os.path.abspath(opts.output) - orig_output = opts.output - - with TemporaryDirectory('_any2lit') as tdir: - oebdir = os.path.join(tdir, 'oeb') - os.mkdir(oebdir) - opts.output = os.path.join(tdir, 'dummy.epub') - opts.profile = 'None' - orig_bfs = opts.base_font_size2 - opts.base_font_size2 = 0 - any2epub(opts, path, create_epub=False, oeb_cover=True, extract_to=oebdir) - opts.base_font_size2 = orig_bfs - opf = glob.glob(os.path.join(oebdir, '*.opf'))[0] - opts.output = orig_output - logging.getLogger('html2epub').info(_('Creating LIT file from EPUB...')) - oeb2lit(opts, opf) - - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) < 2: - parser.print_help() - print('No input file specified.') - return 1 - any2lit(opts, args[1]) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/lrf/html/convert_to.py b/src/calibre/ebooks/lrf/html/convert_to.py deleted file mode 100644 index 8ae9224d2c..0000000000 --- a/src/calibre/ebooks/lrf/html/convert_to.py +++ /dev/null @@ -1,128 +0,0 @@ -from __future__ import print_function -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' -import sys, logging, os - -from calibre import setup_cli_handlers -from calibre.utils.config import OptionParser -from calibre.ebooks import ConversionError -from calibre.ebooks.lrf.meta import get_metadata -from calibre.ebooks.lrf.lrfparser import LRFDocument -from calibre.ebooks.metadata.opf import OPFCreator - -from calibre.ebooks.lrf.objects import PageAttr, BlockAttr, TextAttr -from calibre.ebooks.lrf.pylrs.pylrs import TextStyle - - -class BlockStyle(object): - - def __init__(self, ba): - self.ba = ba - - def __str__(self): - ans = '.'+str(self.ba.id)+' {\n' - if hasattr(self.ba, 'sidemargin'): - margin = str(self.ba.sidemargin) + 'px' - ans += '\tmargin-left: %(m)s; margin-right: %(m)s;\n'%dict(m=margin) - if hasattr(self.ba, 'topskip'): - ans += '\tmargin-top: %dpx;\n'%(self.ba.topskip,) - if hasattr(self.ba, 'footskip'): - ans += '\tmargin-bottom: %dpx;\n'%(self.ba.footskip,) - if hasattr(self.ba, 'framewidth'): - ans += '\tborder-width: %dpx;\n'%(self.ba.framewidth,) - ans += '\tborder-style: solid;\n' - if hasattr(self.ba, 'framecolor'): - if self.ba.framecolor.a < 255: - ans += '\tborder-color: %s;\n'%(self.ba.framecolor.to_html()) - if hasattr(self.ba, 'bgcolor'): - if self.ba.bgcolor.a < 255: - ans += '\tbackground-color: %s;\n'%(self.ba.bgcolor.to_html()) - # TODO: Fixed size blocks - return ans + '}\n' - - -class LRFConverter(object): - - def __init__(self, document, opts, logger): - self.lrf = document - self.opts = opts - self.output_dir = opts.out - self.logger = logger - logger.info('Parsing LRF...') - self.lrf.parse() - - self.create_metadata() - self.create_styles() - - def create_metadata(self): - self.logger.info('Reading metadata...') - mi = get_metadata(self.lrf) - self.opf = OPFCreator(self.output_dir, mi) - - def create_page_styles(self): - self.page_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, PageAttr): - selector = 'body.'+str(obj.id) - self.page_css = selector + ' {\n' - # TODO: Headers and footers - self.page_css += '}\n' - - def create_block_styles(self): - self.block_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, BlockAttr): - self.block_css += str(BlockStyle(obj)) - - def create_text_styles(self): - self.text_css = '' - for obj in self.lrf.objects.values(): - if isinstance(obj, TextAttr): - self.text_css += str(TextStyle(obj)) - print(self.text_css) - - def create_styles(self): - self.logger.info('Creating CSS stylesheet...') - self.create_page_styles() - self.create_block_styles() - - -def option_parser(): - parser = OptionParser(usage='%prog book.lrf') - parser.add_option('--output-dir', '-o', default=None, help=( - 'Output directory in which to store created HTML files. If it does not exist, it is created. By default the current directory is used.'), dest='out') - parser.add_option('--verbose', default=False, action='store_true', dest='verbose') - return parser - - -def process_file(lrfpath, opts, logger=None): - if logger is None: - level = logging.DEBUG if opts.verbose else logging.INFO - logger = logging.getLogger('lrf2html') - setup_cli_handlers(logger, level) - if opts.out is None: - opts.out = os.getcwdu() - else: - opts.out = os.path.abspath(opts.out) - if not os.path.isdir(opts.out): - raise ConversionError(opts.out + ' is not a directory') - if not os.path.exists(opts.out): - os.makedirs(opts.out) - - document = LRFDocument(open(lrfpath, 'rb')) - LRFConverter(document, opts, logger) - - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) != 2: - parser.print_help() - return 1 - process_file(args[1], opts) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/pdf/from_comic.py b/src/calibre/ebooks/pdf/from_comic.py deleted file mode 100644 index c39b660ef4..0000000000 --- a/src/calibre/ebooks/pdf/from_comic.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -'Convert a comic in CBR/CBZ format to pdf' - -import sys -from functools import partial -from calibre.ebooks.lrf.comic.convert_from import do_convert, option_parser, config, main as _main - -convert = partial(do_convert, output_format='pdf') -main = partial(_main, output_format='pdf') - -if __name__ == '__main__': - sys.exit(main()) - -if False: - option_parser - config - From 97ab4acce5a3dd9e4f79ef217bd9db3c281e97b9 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 12:46:40 -0400 Subject: [PATCH 14/19] python3: reduce use of cmp when key can be used instead cmp does not exist on python3 --- manual/custom.py | 4 ++-- src/calibre/customize/profiles.py | 4 ++-- src/calibre/customize/ui.py | 4 ++-- src/calibre/db/cli/cmd_list_categories.py | 4 +--- src/calibre/devices/__init__.py | 3 +-- src/calibre/devices/prs505/sony_cache.py | 2 +- src/calibre/devices/usbms/device.py | 2 +- src/calibre/devices/winusb.py | 3 +-- src/calibre/ebooks/conversion/plumber.py | 2 +- src/calibre/ebooks/lit/writer.py | 3 +-- src/calibre/ebooks/metadata/lit.py | 2 +- src/calibre/ebooks/metadata/meta.py | 4 +--- src/calibre/ebooks/oeb/transforms/guide.py | 3 +-- src/calibre/ebooks/pdf/reflow.py | 10 +++++----- src/calibre/gui2/actions/choose_library.py | 3 +-- src/calibre/gui2/dialogs/catalog.py | 2 +- src/calibre/gui2/library/models.py | 2 +- src/calibre/gui2/library/views.py | 2 +- src/calibre/gui2/preferences/columns.py | 10 ++++------ src/calibre/gui2/preferences/look_feel.py | 2 +- src/calibre/gui2/preferences/main.py | 4 ++-- src/calibre/gui2/preferences/plugboard.py | 13 +------------ src/calibre/gui2/preferences/plugins.py | 2 +- src/calibre/gui2/wizard/__init__.py | 4 ++-- src/calibre/library/custom_columns.py | 4 ++-- src/calibre/library/database.py | 2 +- src/calibre/utils/smtp.py | 3 +-- src/calibre/web/feeds/news.py | 2 +- 28 files changed, 41 insertions(+), 64 deletions(-) diff --git a/manual/custom.py b/manual/custom.py index 2c9b31d514..b35ff29234 100644 --- a/manual/custom.py +++ b/manual/custom.py @@ -179,7 +179,7 @@ def generate_ebook_convert_help(preamble, app): raw += '\n\n.. contents::\n :local:' raw += '\n\n' + options - for pl in sorted(input_format_plugins(), key=lambda x:x.name): + for pl in sorted(input_format_plugins(), key=lambda x: x.name): parser, plumber = create_option_parser(['ebook-convert', 'dummyi.'+sorted(pl.file_types)[0], 'dummyo.epub', '-h'], default_log) groups = [(pl.name+ ' Options', '', g.option_list) for g in @@ -279,7 +279,7 @@ def cli_docs(app): info(bold('creating CLI documentation...')) documented_cmds, undocumented_cmds = get_cli_docs() - documented_cmds.sort(cmp=lambda x, y: cmp(x[0], y[0])) + documented_cmds.sort(key=lambda x: x[0]) undocumented_cmds.sort() documented = [' '*4 + c[0] for c in documented_cmds] diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 2c7fead69f..1038d448be 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -233,7 +233,7 @@ class NookInput(InputProfile): HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput, IRexDR1000Input, IRexDR800Input, NookInput] -input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) +input_profiles.sort(key=lambda x: x.name.lower()) # }}} @@ -870,4 +870,4 @@ class PocketBookPro912Output(OutputProfile): KindlePaperWhite3Output, KindleOasisOutput ] -output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) +output_profiles.sort(key=lambda x: x.name.lower()) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 1229cbf712..8bb9b5814f 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -727,9 +727,9 @@ def initialize_plugins(perf=False): # ipython sys.stdout, sys.stderr = ostdout, ostderr if perf: - for x in sorted(times, key=lambda x:times[x]): + for x in sorted(times, key=lambda x: times[x]): print('%50s: %.3f'%(x, times[x])) - _initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) + _initialized_plugins.sort(key=lambda x: x.priority, reverse=True) reread_filetype_plugins() reread_metadata_plugins() diff --git a/src/calibre/db/cli/cmd_list_categories.py b/src/calibre/db/cli/cmd_list_categories.py index 0280455f66..6165c50877 100644 --- a/src/calibre/db/cli/cmd_list_categories.py +++ b/src/calibre/db/cli/cmd_list_categories.py @@ -149,9 +149,7 @@ def category_metadata(k): (not report_on or k in report_on) ] - categories.sort( - cmp=lambda x, y: cmp(x if x[0] != '#' else x[1:], y if y[0] != '#' else y[1:]) - ) + categories.sort(key=lambda x: x if x[0] != '#' else x[1:]) def fmtr(v): v = v or 0 diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index d3ee2b0f19..5ab3b38dee 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -82,8 +82,7 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None, out = partial(prints, file=buf) devplugins = device_plugins() if plugins is None else plugins - devplugins = list(sorted(devplugins, cmp=lambda - x,y:cmp(x.__class__.__name__, y.__class__.__name__))) + devplugins = list(sorted(devplugins, key=lambda x: x.__class__.__name__)) if plugins is None: for d in devplugins: try: diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index c8db25b73a..0e5a90aaee 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -321,7 +321,7 @@ def rebase_ids(root, base, sourceid, pl_sourceid): # Only rebase ids of nodes that are immediate children of the # record root (that way playlist/itemnodes are unaffected items = root.xpath('child::*[@id]') - items.sort(cmp=lambda x,y:cmp(int(x.get('id')), int(y.get('id')))) + items.sort(key=lambda x: int(x.get('id'))) idmap = {} for i, item in enumerate(items): old = int(item.get('id')) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index a7eb0b3567..b025ca7abf 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -537,7 +537,7 @@ def find_largest_partition(self, path): if sz > 0: nodes.append((x.split('/')[-1], sz)) - nodes.sort(cmp=lambda x, y: cmp(x[1], y[1])) + nodes.sort(key=lambda x: x[1]) if not nodes: return node return nodes[-1][0] diff --git a/src/calibre/devices/winusb.py b/src/calibre/devices/winusb.py index 0a7647cce0..dd927e5df4 100644 --- a/src/calibre/devices/winusb.py +++ b/src/calibre/devices/winusb.py @@ -995,8 +995,7 @@ def develop(): # {{{ drive_letters = set() pprint(usb_devices) print() - devplugins = list(sorted(device_plugins(), cmp=lambda - x,y:cmp(x.__class__.__name__, y.__class__.__name__))) + devplugins = list(sorted(device_plugins(), key=lambda x: x.__class__.__name__)) for dev in devplugins: dev.startup() for dev in devplugins: diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 25ad577c8b..5d8e46fd67 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -821,7 +821,7 @@ def find_html_index(self, files): if not html_files: raise ValueError(_('Could not find an e-book inside the archive')) html_files = [(f, os.stat(f).st_size) for f in html_files] - html_files.sort(cmp=lambda x, y: cmp(x[1], y[1])) + html_files.sort(key=lambda x: x[1]) html_files = [f[0] for f in html_files] for q in ('toc', 'index'): for f in html_files: diff --git a/src/calibre/ebooks/lit/writer.py b/src/calibre/ebooks/lit/writer.py index fd292f4a03..490c06b42f 100644 --- a/src/calibre/ebooks/lit/writer.py +++ b/src/calibre/ebooks/lit/writer.py @@ -670,8 +670,7 @@ def _calculate_deskey(self, hashdata): def _build_dchunks(self): ddata = [] directory = list(self._directory) - directory.sort(cmp=lambda x, y: - cmp(x.name.lower(), y.name.lower())) + directory.sort(key=lambda x: x.name.lower()) qrn = 1 + (1 << 2) dchunk = io.BytesIO() dcount = 0 diff --git a/src/calibre/ebooks/metadata/lit.py b/src/calibre/ebooks/metadata/lit.py index 330bf8cc86..d2f078487d 100644 --- a/src/calibre/ebooks/metadata/lit.py +++ b/src/calibre/ebooks/metadata/lit.py @@ -32,7 +32,7 @@ def get_metadata(stream): except: pass break - covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True) + covers.sort(key=lambda x: len(x[0]), reverse=True) idx = 0 if len(covers) > 1: if covers[1][1] == covers[0][1]+'-standard': diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index de773ec8c6..7a8299dc19 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -41,8 +41,7 @@ def metadata_from_formats(formats, force_read_metadata=False, pattern=None): def _metadata_from_formats(formats, force_read_metadata=False, pattern=None): mi = MetaInformation(None, None) - formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)], - METADATA_PRIORITIES[path_to_ext(y)])) + formats.sort(key=lambda x: METADATA_PRIORITIES[path_to_ext(x)]) extensions = list(map(path_to_ext, formats)) if 'opf' in extensions: opf = formats[extensions.index('opf')] @@ -241,4 +240,3 @@ def forked_read_metadata(path, tdir): opf = metadata_to_opf(mi, default_lang='und') with lopen(os.path.join(tdir, 'metadata.opf'), 'wb') as f: f.write(opf) - diff --git a/src/calibre/ebooks/oeb/transforms/guide.py b/src/calibre/ebooks/oeb/transforms/guide.py index 29c5f2a93a..104a2fc1d5 100644 --- a/src/calibre/ebooks/oeb/transforms/guide.py +++ b/src/calibre/ebooks/oeb/transforms/guide.py @@ -28,7 +28,7 @@ def __call__(self, oeb, opts): else: covers.append([self.oeb.guide[x], len(item.data)]) - covers.sort(cmp=lambda x,y:cmp(x[1], y[1]), reverse=True) + covers.sort(key=lambda x: x[1], reverse=True) if covers: ref = covers[0][0] if len(covers) > 1: @@ -53,4 +53,3 @@ def __call__(self, oeb, opts): if item.title and item.title.lower() == 'start': continue self.oeb.guide.remove(x) - diff --git a/src/calibre/ebooks/pdf/reflow.py b/src/calibre/ebooks/pdf/reflow.py index a7708ea8ce..337147d8a7 100644 --- a/src/calibre/ebooks/pdf/reflow.py +++ b/src/calibre/ebooks/pdf/reflow.py @@ -169,7 +169,7 @@ def prepend(self, elem): self._post_add() def _post_add(self): - self.elements.sort(cmp=lambda x,y:cmp(x.bottom,y.bottom)) + self.elements.sort(key=lambda x: x.bottom) self.top = self.elements[0].top self.bottom = self.elements[-1].bottom self.left, self.right = sys.maxint, 0 @@ -260,7 +260,7 @@ def __init__(self, opts, log): def add(self, columns): if not self.columns: - for x in sorted(columns, cmp=lambda x,y: cmp(x.left, y.left)): + for x in sorted(columns, key=lambda x: x.left): self.columns.append(x) else: for i in range(len(columns)): @@ -458,7 +458,7 @@ def __init__(self, page, font_map, opts, log, idc): self.elements = list(self.texts) for img in page.xpath('descendant::img'): self.elements.append(Image(img, self.opts, self.log, idc)) - self.elements.sort(cmp=lambda x,y:cmp(x.top, y.top)) + self.elements.sort(key=lambda x: x.top) def coalesce_fragments(self): @@ -580,7 +580,7 @@ def coalesce_regions(self): def sort_into_columns(self, elem, neighbors): neighbors.add(elem) - neighbors = sorted(neighbors, cmp=lambda x,y:cmp(x.left, y.left)) + neighbors = sorted(neighbors, key=lambda x: x.left) if self.opts.verbose > 3: self.log.debug('Neighbors:', [x.to_html() for x in neighbors]) columns = [Column()] @@ -595,7 +595,7 @@ def sort_into_columns(self, elem, neighbors): if not added: columns.append(Column()) columns[-1].add(x) - columns.sort(cmp=lambda x,y:cmp(x.left, y.left)) + columns.sort(key=lambda x: x.left) return columns def find_elements_in_row_of(self, x): diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index a8adb593e4..a562958470 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -51,8 +51,7 @@ def read_stats(self): def write_stats(self): locs = list(self.stats.keys()) - locs.sort(cmp=lambda x, y: cmp(self.stats[x], self.stats[y]), - reverse=True) + locs.sort(key=lambda x: self.stats[x], reverse=True) for key in locs[500:]: self.stats.pop(key) gprefs.set('library_usage_stats', self.stats) diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index 1f32406b76..f10eab4384 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -93,7 +93,7 @@ def __init__(self, parent, dbspec, ids, db): else: info("No dynamic tab resources found for %s" % name) - self.widgets = sorted(self.widgets, cmp=lambda x,y:cmp(x.TITLE, y.TITLE)) + self.widgets = sorted(self.widgets, key=lambda x: x.TITLE) # Generate a sorted list of installed catalog formats/sync_enabled pairs fmts = sorted([x[0] for x in self.fmts]) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index b36f91798c..d058e396e0 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -323,7 +323,7 @@ def col_idx(name): return 100000 return self.db.field_metadata[name]['rec_index'] - self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y))) + self.column_map.sort(key=lambda x: col_idx(x)) for col in self.column_map: if col in self.orig_headers: self.headers[col] = self.orig_headers[col] diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index a49f7c778f..a5ce76815d 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -1031,7 +1031,7 @@ def set_current_row(self, row=0, select=True, for_sync=False): h.visualIndex(x) > -1] if not pairs: pairs = [(0, 0)] - pairs.sort(cmp=lambda x,y:cmp(x[1], y[1])) + pairs.sort(key=lambda x: x[1]) i = pairs[0][0] index = self.model().index(row, i) if for_sync: diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index 0cd0fe5bea..3d346c41fb 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -69,7 +69,7 @@ def init_columns(self, defaults=False): state = self.columns_state(defaults) self.hidden_cols = state['hidden_columns'] positions = state['column_positions'] - colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y])) + colmap.sort(key=lambda x: positions[x]) self.opt_columns.clear() db = model.db @@ -248,12 +248,10 @@ def apply_custom_column_changes(self): if 'ondevice' in hidden_cols: hidden_cols.remove('ondevice') - def col_pos(x, y): - xidx = config_cols.index(x) if x in config_cols else sys.maxint - yidx = config_cols.index(y) if y in config_cols else sys.maxint - return cmp(xidx, yidx) + def col_pos(x): + return config_cols.index(x) if x in config_cols else sys.maxint positions = {} - for i, col in enumerate((sorted(model.column_map, cmp=col_pos))): + for i, col in enumerate((sorted(model.column_map, key=col_pos))): positions[col] = i state = {'hidden_columns': hidden_cols, 'column_positions':positions} self.gui.library_view.apply_state(state) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 0fb52d9a97..258ffcdc15 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -456,7 +456,7 @@ def get_esc_lang(l): if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) - items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower())) + items.sort(key=lambda x: x[1].lower()) choices = [(y, x) for x, y in items] # Default language is the autodetected one choices = [(get_language(lang), lang)] + choices diff --git a/src/calibre/gui2/preferences/main.py b/src/calibre/gui2/preferences/main.py index 3b8cc99c45..43f4386d5b 100644 --- a/src/calibre/gui2/preferences/main.py +++ b/src/calibre/gui2/preferences/main.py @@ -171,7 +171,7 @@ def __init__(self, parent=None): self.category_names = category_names categories = list(category_map.keys()) - categories.sort(cmp=lambda x, y: cmp(category_map[x], category_map[y])) + categories.sort(key=lambda x: category_map[x]) self.category_map = OrderedDict() for c in categories: @@ -181,7 +181,7 @@ def __init__(self, parent=None): self.category_map[plugin.category].append(plugin) for plugins in self.category_map.values(): - plugins.sort(cmp=lambda x, y: cmp(x.name_order, y.name_order)) + plugins.sort(key=lambda x: x.name_order) self.widgets = [] self._layout = QVBoxLayout() diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index 5005e87735..65dea1c87d 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -34,17 +34,6 @@ def genesis(self, gui): self.db = gui.library_view.model().db def initialize(self): - def field_cmp(x, y): - if x.startswith('#'): - if y.startswith('#'): - return cmp(x.lower(), y.lower()) - else: - return 1 - elif y.startswith('#'): - return -1 - else: - return cmp(x.lower(), y.lower()) - ConfigWidgetBase.initialize(self) self.current_plugboards = copy.deepcopy(self.db.prefs.get('plugboards',{})) @@ -73,7 +62,7 @@ def field_cmp(x, y): if n not in self.disabled_devices: self.disabled_devices.append(n) - self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) + self.devices.sort(key=lambda x: x.lower()) self.devices.insert(1, plugboard_save_to_disk_value) self.devices.insert(1, plugboard_content_server_value) self.device_to_formats_map[plugboard_content_server_value] = \ diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index a683eccf4d..c3c777f1e9 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -61,7 +61,7 @@ def populate(self): self.categories = sorted(self._data.keys()) for plugins in self._data.values(): - plugins.sort(cmp=lambda x, y: cmp(x.name.lower(), y.name.lower())) + plugins.sort(key=lambda x: x.name.lower()) def universal_set(self): ans = set([]) diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py index ec61d6c0fe..e137babbbe 100644 --- a/src/calibre/gui2/wizard/__init__.py +++ b/src/calibre/gui2/wizard/__init__.py @@ -427,7 +427,7 @@ def get_manufacturers(): def get_devices_of(manufacturer): ans = [d for d in get_devices() if d.manufacturer == manufacturer] - return sorted(ans, cmp=lambda x,y:cmp(x.name, y.name)) + return sorted(ans, key=lambda x: x.name) class ManufacturerModel(QAbstractListModel): @@ -682,7 +682,7 @@ def get_esc_lang(l): if l != lang] if lang != 'en': items.append(('en', get_esc_lang('en'))) - items.sort(cmp=lambda x, y: cmp(x[1], y[1])) + items.sort(key=lambda x: x[1]) for item in items: self.language.addItem(item[1], (item[0])) self.language.blockSignals(False) diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 56e53ee91b..ad2ae31e39 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -216,7 +216,7 @@ def get_custom(self, idx, label=None, num=None, index_is_id=False): if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + ans.sort(key=lambda x:x.lower()) return ans def get_custom_extra(self, idx, label=None, num=None, index_is_id=False): @@ -243,7 +243,7 @@ def get_custom_and_extra(self, idx, label=None, num=None, index_is_id=False): if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + ans.sort(key=lambda x: x.lower()) if data['datatype'] != 'series': return (ans, None) ign,lt = self.custom_table_names(data['num']) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index b303833439..8ce65a023e 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1018,7 +1018,7 @@ def books_in_series(self, series_id): if not ans: return [] ans = [id[0] for id in ans] - ans.sort(cmp=lambda x, y: cmp(self.series_index(x, True), self.series_index(y, True))) + ans.sort(key=lambda x: self.series_index(x, True)) return ans def books_in_series_of(self, index, index_is_id=False): diff --git a/src/calibre/utils/smtp.py b/src/calibre/utils/smtp.py index d840cc4c37..8da8cc7bd2 100644 --- a/src/calibre/utils/smtp.py +++ b/src/calibre/utils/smtp.py @@ -98,8 +98,7 @@ def get_mx(host, verbose=0): if verbose: print('Find mail exchanger for', host) answers = list(dns.resolver.query(host, 'MX')) - answers.sort(cmp=lambda x, y: cmp(int(getattr(x, 'preference', sys.maxint)), - int(getattr(y, 'preference', sys.maxint)))) + answers.sort(key=lambda x: int(getattr(x, 'preference', sys.maxint))) return [str(x.exchange) for x in answers if hasattr(x, 'exchange')] diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index f51f7e5a21..101e68b510 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -760,7 +760,7 @@ def sort_index_by(self, index, weights): in index are not in weights, they are assumed to have a weight of 0. ''' weights = defaultdict(lambda: 0, weights) - index.sort(cmp=lambda x, y: cmp(weights[x], weights[y])) + index.sort(key=lambda x: weights[x]) return index def parse_index(self): From 9023ea8947f228ec2ef1e72f254930ff8a4d8145 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 13:00:31 -0400 Subject: [PATCH 15/19] python3: add Cookie wrapper to polyglot --- src/calibre/srv/tests/auth.py | 5 +++-- src/calibre/srv/utils.py | 2 +- src/calibre/utils/browser.py | 2 +- src/polyglot/http_cookie.py | 12 ++++++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 src/polyglot/http_cookie.py diff --git a/src/calibre/srv/tests/auth.py b/src/calibre/srv/tests/auth.py index 9407516cf0..45764253e6 100644 --- a/src/calibre/srv/tests/auth.py +++ b/src/calibre/srv/tests/auth.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import base64, subprocess, os, cookielib, time +import base64, subprocess, os, time from collections import namedtuple try: from distutils.spawn import find_executable @@ -19,6 +19,7 @@ from calibre.srv.routes import endpoint, Router from polyglot.builtins import iteritems, itervalues from polyglot import http_client +from polyglot.http_cookie import CookieJar from polyglot.urllib import (build_opener, HTTPBasicAuthHandler, HTTPCookieProcessor, HTTPDigestAuthHandler, HTTPError) @@ -283,7 +284,7 @@ def test_android_auth_workaround(self): # {{{ auth_handler = HTTPDigestAuthHandler() url = 'http://localhost:%d%s' % (server.address[1], '/android') auth_handler.add_password(realm=REALM, uri=url, user='testuser', passwd='testpw') - cj = cookielib.CookieJar() + cj = CookieJar() cookie_handler = HTTPCookieProcessor(cj) r = build_opener(auth_handler, cookie_handler).open(url) self.ae(r.getcode(), http_client.OK) diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index 0037e4e8a7..22f9d2b777 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -7,7 +7,6 @@ __copyright__ = '2015, Kovid Goyal ' import errno, socket, select, os, time -from Cookie import SimpleCookie from contextlib import closing from email.utils import formatdate from operator import itemgetter @@ -23,6 +22,7 @@ from calibre.utils.shared_file import share_open, raise_winerror from polyglot.builtins import iteritems, map, unicode_type, range from polyglot import reprlib +from polyglot.http_cookie import SimpleCookie from polyglot.urllib import parse_qs, quote as urlquote HTTP1 = 'HTTP/1.0' diff --git a/src/calibre/utils/browser.py b/src/calibre/utils/browser.py index 47835dec1e..1228747ee8 100644 --- a/src/calibre/utils/browser.py +++ b/src/calibre/utils/browser.py @@ -6,11 +6,11 @@ __docformat__ = 'restructuredtext en' import copy, ssl -from cookielib import CookieJar, Cookie from mechanize import Browser as B, HTTPSHandler from polyglot import http_client +from polyglot.http_cookie import CookieJar, Cookie class ModernHTTPSHandler(HTTPSHandler): diff --git a/src/polyglot/http_cookie.py b/src/polyglot/http_cookie.py new file mode 100644 index 0000000000..645b8b6a3a --- /dev/null +++ b/src/polyglot/http_cookie.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Eli Schwartz + +from polyglot.builtins import is_py3 + +if is_py3: + from http.cookies import SimpleCookie # noqa + from http.cookiejar import CookieJar, Cookie # noqa +else: + from Cookie import SimpleCookie # noqa + from cookielib import CookieJar, Cookie # noqa From 491937360891d60b820a0d7e05aaa3bd25811e95 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 13:13:46 -0400 Subject: [PATCH 16/19] python3: do not decode uuid4 hex, as it is always a str() --- src/calibre/srv/http_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index b5b16fbe8f..8313d18e32 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -29,7 +29,7 @@ from polyglot import http_client, reprlib Range = namedtuple('Range', 'start stop size') -MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') +MULTIPART_SEPARATOR = uuid.uuid4().hex COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'} if is_py3: import zlib From 24f504f16bf4320c77d058687cc529a8419d3749 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 14:26:32 -0400 Subject: [PATCH 17/19] python3: remove more cStringIO --- src/calibre/devices/__init__.py | 4 ++-- src/calibre/ebooks/conversion/plugins/tcr_input.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 5ab3b38dee..83ad041ff6 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -8,7 +8,7 @@ import sys, time, pprint from functools import partial -from StringIO import StringIO +from io import BytesIO DAY_MAP = dict(Sun=0, Mon=1, Tue=2, Wed=3, Thu=4, Fri=5, Sat=6) MONTH_MAP = dict(Jan=1, Feb=2, Mar=3, Apr=4, May=5, Jun=6, Jul=7, Aug=8, Sep=9, Oct=10, Nov=11, Dec=12) @@ -77,7 +77,7 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None, oldo, olde = sys.stdout, sys.stderr if buf is None: - buf = StringIO() + buf = BytesIO() sys.stdout = sys.stderr = buf out = partial(prints, file=buf) diff --git a/src/calibre/ebooks/conversion/plugins/tcr_input.py b/src/calibre/ebooks/conversion/plugins/tcr_input.py index 45903e2c56..0578eb3aac 100644 --- a/src/calibre/ebooks/conversion/plugins/tcr_input.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_input.py @@ -4,7 +4,7 @@ __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -from cStringIO import StringIO +from io import BytesIO from calibre.customize.conversion import InputFormatPlugin @@ -24,7 +24,7 @@ def convert(self, stream, options, file_ext, log, accelerators): raw_txt = decompress(stream) log.info('Converting text to OEB...') - stream = StringIO(raw_txt) + stream = BytesIO(raw_txt) from calibre.customize.ui import plugin_for_input_format From a75b0730a7190f5720af6a3a41ad2e3896fdd06f Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 14:37:29 -0400 Subject: [PATCH 18/19] python3: do not use types.*Type which is an alias for str Just compare to (str, unicode_type) directly, and handle the related python3 reclassification. --- src/calibre/library/catalogs/bibtex.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index 4a6e8a4e27..d21d9a6fa7 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -7,7 +7,6 @@ import re, codecs, os, numbers from collections import namedtuple -from types import StringType, UnicodeType from calibre import (strftime) from calibre.customize import CatalogPlugin @@ -15,7 +14,7 @@ from calibre.customize.conversion import DummyReporter from calibre.constants import preferred_encoding from calibre.ebooks.metadata import format_isbn -from polyglot.builtins import string_or_bytes +from polyglot.builtins import string_or_bytes, unicode_type class BIBTEX(CatalogPlugin): @@ -351,7 +350,7 @@ def tpl_replace(objtplname) : bibtexc.ascii_bibtex = True # Check citation choice and go to default in case of bad CLI - if isinstance(opts.impcit, (StringType, UnicodeType)) : + if isinstance(opts.impcit, (str, unicode_type)) : if opts.impcit == 'False' : citation_bibtex= False elif opts.impcit == 'True' : From 02be4edee24ecc94fccfd9a6165903a60901a9f3 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 26 Mar 2019 16:46:44 -0400 Subject: [PATCH 19/19] test: try to make test_import work better without assuming cwd Allows it to run properly on Windows. --- src/calibre/test_build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 397299264b..50fb71430d 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -82,7 +82,9 @@ def test_imports(self): exclude.append('osx') if not islinux: exclude.extend(['dbus', 'linux']) - for root, dirs, files in os.walk('src/calibre'): + base = os.path.dirname(__file__) + trimpath = len(os.path.dirname(base)) + 1 + for root, dirs, files in os.walk(base): for dir in dirs: if not os.path.isfile(os.path.join(root, dir, '__init__.py')): dirs.remove(dir) @@ -90,7 +92,7 @@ def test_imports(self): file, ext = os.path.splitext(file) if ext != '.py': continue - name = '.'.join(root.split(os.path.sep)[1:] + [file]) + name = '.'.join(root[trimpath:].split(os.path.sep) + [file]) if not any(x for x in exclude if x in name): importlib.import_module(name)