From a8b1489233d6cbc9bbf83fd28ecb731a0c3078af Mon Sep 17 00:00:00 2001 From: Jim Miller Date: Sat, 18 Jan 2025 08:17:39 -0600 Subject: [PATCH] Strip out unused parts of requests_toolbelt to avoid dependency issues. #1145 --- .../requests_toolbelt/__init__.py | 22 +- .../requests_toolbelt/_compat.py | 324 --------- .../requests_toolbelt/adapters/__init__.py | 15 - .../requests_toolbelt/adapters/appengine.py | 206 ------ .../requests_toolbelt/adapters/fingerprint.py | 48 -- .../adapters/host_header_ssl.py | 43 -- .../adapters/socket_options.py | 129 ---- .../requests_toolbelt/adapters/source.py | 67 -- .../requests_toolbelt/adapters/ssl.py | 66 -- .../requests_toolbelt/adapters/x509.py | 178 ----- .../requests_toolbelt/auth/__init__.py | 0 .../auth/_digest_auth_compat.py | 29 - .../requests_toolbelt/auth/guess.py | 146 ---- .../requests_toolbelt/auth/handler.py | 142 ---- .../auth/http_proxy_digest.py | 103 --- .../requests_toolbelt/cookies/__init__.py | 0 .../requests_toolbelt/cookies/forgetful.py | 7 - .../downloadutils/__init__.py | 0 .../requests_toolbelt/downloadutils/stream.py | 177 ----- .../requests_toolbelt/downloadutils/tee.py | 123 ---- .../requests_toolbelt/exceptions.py | 37 - .../requests_toolbelt/multipart/__init__.py | 31 - .../requests_toolbelt/multipart/decoder.py | 156 ----- .../requests_toolbelt/multipart/encoder.py | 655 ------------------ .../requests_toolbelt/sessions.py | 70 -- .../requests_toolbelt/streaming_iterator.py | 116 ---- .../requests_toolbelt/threaded/__init__.py | 97 --- .../requests_toolbelt/threaded/pool.py | 211 ------ .../requests_toolbelt/threaded/thread.py | 53 -- .../requests_toolbelt/utils/deprecated.py | 91 --- .../requests_toolbelt/utils/formdata.py | 108 --- .../requests_toolbelt/utils/user_agent.py | 143 ---- 32 files changed, 5 insertions(+), 3588 deletions(-) delete mode 100644 included_dependencies/requests_toolbelt/_compat.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/__init__.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/appengine.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/fingerprint.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/host_header_ssl.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/socket_options.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/source.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/ssl.py delete mode 100644 included_dependencies/requests_toolbelt/adapters/x509.py delete mode 100644 included_dependencies/requests_toolbelt/auth/__init__.py delete mode 100644 included_dependencies/requests_toolbelt/auth/_digest_auth_compat.py delete mode 100644 included_dependencies/requests_toolbelt/auth/guess.py delete mode 100644 included_dependencies/requests_toolbelt/auth/handler.py delete mode 100644 included_dependencies/requests_toolbelt/auth/http_proxy_digest.py delete mode 100644 included_dependencies/requests_toolbelt/cookies/__init__.py delete mode 100644 included_dependencies/requests_toolbelt/cookies/forgetful.py delete mode 100644 included_dependencies/requests_toolbelt/downloadutils/__init__.py delete mode 100644 included_dependencies/requests_toolbelt/downloadutils/stream.py delete mode 100644 included_dependencies/requests_toolbelt/downloadutils/tee.py delete mode 100644 included_dependencies/requests_toolbelt/exceptions.py delete mode 100644 included_dependencies/requests_toolbelt/multipart/__init__.py delete mode 100644 included_dependencies/requests_toolbelt/multipart/decoder.py delete mode 100644 included_dependencies/requests_toolbelt/multipart/encoder.py delete mode 100644 included_dependencies/requests_toolbelt/sessions.py delete mode 100644 included_dependencies/requests_toolbelt/streaming_iterator.py delete mode 100644 included_dependencies/requests_toolbelt/threaded/__init__.py delete mode 100644 included_dependencies/requests_toolbelt/threaded/pool.py delete mode 100644 included_dependencies/requests_toolbelt/threaded/thread.py delete mode 100644 included_dependencies/requests_toolbelt/utils/deprecated.py delete mode 100644 included_dependencies/requests_toolbelt/utils/formdata.py delete mode 100644 included_dependencies/requests_toolbelt/utils/user_agent.py diff --git a/included_dependencies/requests_toolbelt/__init__.py b/included_dependencies/requests_toolbelt/__init__.py index 888b77ec..fff0d349 100644 --- a/included_dependencies/requests_toolbelt/__init__.py +++ b/included_dependencies/requests_toolbelt/__init__.py @@ -7,16 +7,12 @@ See http://toolbelt.rtfd.org/ for documentation :copyright: (c) 2014 by Ian Cordasco and Cory Benfield :license: Apache v2.0, see LICENSE for more details -""" -from .adapters import SSLAdapter, SourceAddressAdapter -from .auth.guess import GuessAuth -from .multipart import ( - MultipartEncoder, MultipartEncoderMonitor, MultipartDecoder, - ImproperBodyPartContentException, NonMultipartContentTypeException - ) -from .streaming_iterator import StreamingIterator -from .utils.user_agent import user_agent +Modified 2025 for FanFicFare including in Calibre plugin for old +python2 versions of Calibre. + +FFF only needs utils.dump for cloudscraper debug output. +""" __title__ = 'requests-toolbelt' __authors__ = 'Ian Cordasco, Cory Benfield' @@ -24,11 +20,3 @@ __license__ = 'Apache v2.0' __copyright__ = 'Copyright 2014 Ian Cordasco, Cory Benfield' __version__ = '0.9.1' __version_info__ = tuple(int(i) for i in __version__.split('.')) - -__all__ = [ - 'GuessAuth', 'MultipartEncoder', 'MultipartEncoderMonitor', - 'MultipartDecoder', 'SSLAdapter', 'SourceAddressAdapter', - 'StreamingIterator', 'user_agent', 'ImproperBodyPartContentException', - 'NonMultipartContentTypeException', '__title__', '__authors__', - '__license__', '__copyright__', '__version__', '__version_info__', -] diff --git a/included_dependencies/requests_toolbelt/_compat.py b/included_dependencies/requests_toolbelt/_compat.py deleted file mode 100644 index 622e77fa..00000000 --- a/included_dependencies/requests_toolbelt/_compat.py +++ /dev/null @@ -1,324 +0,0 @@ -"""Private module full of compatibility hacks. - -Primarily this is for downstream redistributions of requests that unvendor -urllib3 without providing a shim. - -.. warning:: - - This module is private. If you use it, and something breaks, you were - warned -""" -import sys - -import requests - -try: - from requests.packages.urllib3 import fields - from requests.packages.urllib3 import filepost - from requests.packages.urllib3 import poolmanager -except ImportError: - from urllib3 import fields - from urllib3 import filepost - from urllib3 import poolmanager - -try: - from requests.packages.urllib3.connection import HTTPConnection - from requests.packages.urllib3 import connection -except ImportError: - try: - from urllib3.connection import HTTPConnection - from urllib3 import connection - except ImportError: - HTTPConnection = None - connection = None - - -if requests.__build__ < 0x020300: - timeout = None -else: - try: - from requests.packages.urllib3.util import timeout - except ImportError: - from urllib3.util import timeout - -if requests.__build__ < 0x021000: - gaecontrib = None -else: - try: - from requests.packages.urllib3.contrib import appengine as gaecontrib - except ImportError: - from urllib3.contrib import appengine as gaecontrib - -if requests.__build__ < 0x021200: - PyOpenSSLContext = None -else: - try: - from requests.packages.urllib3.contrib.pyopenssl \ - import PyOpenSSLContext - except ImportError: - try: - from urllib3.contrib.pyopenssl import PyOpenSSLContext - except ImportError: - PyOpenSSLContext = None - -PY3 = sys.version_info > (3, 0) - -if PY3: - from collections.abc import Mapping, MutableMapping - import queue - from urllib.parse import urlencode, urljoin -else: - from collections import Mapping, MutableMapping - import Queue as queue - from urllib import urlencode - from urlparse import urljoin - -try: - basestring = basestring -except NameError: - basestring = (str, bytes) - - -class HTTPHeaderDict(MutableMapping): - """ - :param headers: - An iterable of field-value pairs. Must not contain multiple field names - when compared case-insensitively. - - :param kwargs: - Additional field-value pairs to pass in to ``dict.update``. - - A ``dict`` like container for storing HTTP Headers. - - Field names are stored and compared case-insensitively in compliance with - RFC 7230. Iteration provides the first case-sensitive key seen for each - case-insensitive pair. - - Using ``__setitem__`` syntax overwrites fields that compare equal - case-insensitively in order to maintain ``dict``'s api. For fields that - compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add`` - in a loop. - - If multiple fields that are equal case-insensitively are passed to the - constructor or ``.update``, the behavior is undefined and some will be - lost. - - >>> headers = HTTPHeaderDict() - >>> headers.add('Set-Cookie', 'foo=bar') - >>> headers.add('set-cookie', 'baz=quxx') - >>> headers['content-length'] = '7' - >>> headers['SET-cookie'] - 'foo=bar, baz=quxx' - >>> headers['Content-Length'] - '7' - """ - - def __init__(self, headers=None, **kwargs): - super(HTTPHeaderDict, self).__init__() - self._container = {} - if headers is not None: - if isinstance(headers, HTTPHeaderDict): - self._copy_from(headers) - else: - self.extend(headers) - if kwargs: - self.extend(kwargs) - - def __setitem__(self, key, val): - self._container[key.lower()] = (key, val) - return self._container[key.lower()] - - def __getitem__(self, key): - val = self._container[key.lower()] - return ', '.join(val[1:]) - - def __delitem__(self, key): - del self._container[key.lower()] - - def __contains__(self, key): - return key.lower() in self._container - - def __eq__(self, other): - if not isinstance(other, Mapping) and not hasattr(other, 'keys'): - return False - if not isinstance(other, type(self)): - other = type(self)(other) - return (dict((k.lower(), v) for k, v in self.itermerged()) == - dict((k.lower(), v) for k, v in other.itermerged())) - - def __ne__(self, other): - return not self.__eq__(other) - - if not PY3: # Python 2 - iterkeys = MutableMapping.iterkeys - itervalues = MutableMapping.itervalues - - __marker = object() - - def __len__(self): - return len(self._container) - - def __iter__(self): - # Only provide the originally cased names - for vals in self._container.values(): - yield vals[0] - - def pop(self, key, default=__marker): - """D.pop(k[,d]) -> v, remove specified key and return its value. - - If key is not found, d is returned if given, otherwise KeyError is - raised. - """ - # Using the MutableMapping function directly fails due to the private - # marker. - # Using ordinary dict.pop would expose the internal structures. - # So let's reinvent the wheel. - try: - value = self[key] - except KeyError: - if default is self.__marker: - raise - return default - else: - del self[key] - return value - - def discard(self, key): - try: - del self[key] - except KeyError: - pass - - def add(self, key, val): - """Adds a (name, value) pair, doesn't overwrite the value if it already - exists. - - >>> headers = HTTPHeaderDict(foo='bar') - >>> headers.add('Foo', 'baz') - >>> headers['foo'] - 'bar, baz' - """ - key_lower = key.lower() - new_vals = key, val - # Keep the common case aka no item present as fast as possible - vals = self._container.setdefault(key_lower, new_vals) - if new_vals is not vals: - # new_vals was not inserted, as there was a previous one - if isinstance(vals, list): - # If already several items got inserted, we have a list - vals.append(val) - else: - # vals should be a tuple then, i.e. only one item so far - # Need to convert the tuple to list for further extension - self._container[key_lower] = [vals[0], vals[1], val] - - def extend(self, *args, **kwargs): - """Generic import function for any type of header-like object. - Adapted version of MutableMapping.update in order to insert items - with self.add instead of self.__setitem__ - """ - if len(args) > 1: - raise TypeError("extend() takes at most 1 positional " - "arguments ({} given)".format(len(args))) - other = args[0] if len(args) >= 1 else () - - if isinstance(other, HTTPHeaderDict): - for key, val in other.iteritems(): - self.add(key, val) - elif isinstance(other, Mapping): - for key in other: - self.add(key, other[key]) - elif hasattr(other, "keys"): - for key in other.keys(): - self.add(key, other[key]) - else: - for key, value in other: - self.add(key, value) - - for key, value in kwargs.items(): - self.add(key, value) - - def getlist(self, key): - """Returns a list of all the values for the named field. Returns an - empty list if the key doesn't exist.""" - try: - vals = self._container[key.lower()] - except KeyError: - return [] - else: - if isinstance(vals, tuple): - return [vals[1]] - else: - return vals[1:] - - # Backwards compatibility for httplib - getheaders = getlist - getallmatchingheaders = getlist - iget = getlist - - def __repr__(self): - return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) - - def _copy_from(self, other): - for key in other: - val = other.getlist(key) - if isinstance(val, list): - # Don't need to convert tuples - val = list(val) - self._container[key.lower()] = [key] + val - - def copy(self): - clone = type(self)() - clone._copy_from(self) - return clone - - def iteritems(self): - """Iterate over all header lines, including duplicate ones.""" - for key in self: - vals = self._container[key.lower()] - for val in vals[1:]: - yield vals[0], val - - def itermerged(self): - """Iterate over all headers, merging duplicate ones together.""" - for key in self: - val = self._container[key.lower()] - yield val[0], ', '.join(val[1:]) - - def items(self): - return list(self.iteritems()) - - @classmethod - def from_httplib(cls, message): # Python 2 - """Read headers from a Python 2 httplib message object.""" - # python2.7 does not expose a proper API for exporting multiheaders - # efficiently. This function re-reads raw lines from the message - # object and extracts the multiheaders properly. - headers = [] - - for line in message.headers: - if line.startswith((' ', '\t')): - key, value = headers[-1] - headers[-1] = (key, value + '\r\n' + line.rstrip()) - continue - - key, value = line.split(':', 1) - headers.append((key, value.strip())) - - return cls(headers) - - -__all__ = ( - 'basestring', - 'connection', - 'fields', - 'filepost', - 'poolmanager', - 'timeout', - 'HTTPHeaderDict', - 'queue', - 'urlencode', - 'gaecontrib', - 'urljoin', - 'PyOpenSSLContext', -) diff --git a/included_dependencies/requests_toolbelt/adapters/__init__.py b/included_dependencies/requests_toolbelt/adapters/__init__.py deleted file mode 100644 index ddfe4fe2..00000000 --- a/included_dependencies/requests_toolbelt/adapters/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -""" -requests-toolbelt.adapters -========================== - -See http://toolbelt.rtfd.org/ for documentation - -:copyright: (c) 2014 by Ian Cordasco and Cory Benfield -:license: Apache v2.0, see LICENSE for more details -""" - -from .ssl import SSLAdapter -from .source import SourceAddressAdapter - -__all__ = ['SSLAdapter', 'SourceAddressAdapter'] diff --git a/included_dependencies/requests_toolbelt/adapters/appengine.py b/included_dependencies/requests_toolbelt/adapters/appengine.py deleted file mode 100644 index 2af2bbbd..00000000 --- a/included_dependencies/requests_toolbelt/adapters/appengine.py +++ /dev/null @@ -1,206 +0,0 @@ -# -*- coding: utf-8 -*- -"""The App Engine Transport Adapter for requests. - -.. versionadded:: 0.6.0 - -This requires a version of requests >= 2.10.0 and Python 2. - -There are two ways to use this library: - -#. If you're using requests directly, you can use code like: - - .. code-block:: python - - >>> import requests - >>> import ssl - >>> import requests.packages.urllib3.contrib.appengine as ul_appengine - >>> from requests_toolbelt.adapters import appengine - >>> s = requests.Session() - >>> if ul_appengine.is_appengine_sandbox(): - ... s.mount('http://', appengine.AppEngineAdapter()) - ... s.mount('https://', appengine.AppEngineAdapter()) - -#. If you depend on external libraries which use requests, you can use code - like: - - .. code-block:: python - - >>> from requests_toolbelt.adapters import appengine - >>> appengine.monkeypatch() - -which will ensure all requests.Session objects use AppEngineAdapter properly. - -You are also able to :ref:`disable certificate validation ` -when monkey-patching. -""" -import requests -import warnings -from requests import adapters -from requests import sessions - -from .. import exceptions as exc -from .._compat import gaecontrib -from .._compat import timeout - - -class AppEngineMROHack(adapters.HTTPAdapter): - """Resolves infinite recursion when monkeypatching. - - This works by injecting itself as the base class of both the - :class:`AppEngineAdapter` and Requests' default HTTPAdapter, which needs to - be done because default HTTPAdapter's MRO is recompiled when we - monkeypatch, at which point this class becomes HTTPAdapter's base class. - In addition, we use an instantiation flag to avoid infinite recursion. - """ - _initialized = False - - def __init__(self, *args, **kwargs): - if not self._initialized: - self._initialized = True - super(AppEngineMROHack, self).__init__(*args, **kwargs) - - -class AppEngineAdapter(AppEngineMROHack, adapters.HTTPAdapter): - """The transport adapter for Requests to use urllib3's GAE support. - - Implements Requests's HTTPAdapter API. - - When deploying to Google's App Engine service, some of Requests' - functionality is broken. There is underlying support for GAE in urllib3. - This functionality, however, is opt-in and needs to be enabled explicitly - for Requests to be able to use it. - """ - - __attrs__ = adapters.HTTPAdapter.__attrs__ + ['_validate_certificate'] - - def __init__(self, validate_certificate=True, *args, **kwargs): - _check_version() - self._validate_certificate = validate_certificate - super(AppEngineAdapter, self).__init__(*args, **kwargs) - - def init_poolmanager(self, connections, maxsize, block=False): - self.poolmanager = _AppEnginePoolManager(self._validate_certificate) - - -class InsecureAppEngineAdapter(AppEngineAdapter): - """An always-insecure GAE adapter for Requests. - - This is a variant of the the transport adapter for Requests to use - urllib3's GAE support that does not validate certificates. Use with - caution! - - .. note:: - The ``validate_certificate`` keyword argument will not be honored here - and is not part of the signature because we always force it to - ``False``. - - See :class:`AppEngineAdapter` for further details. - """ - - def __init__(self, *args, **kwargs): - if kwargs.pop("validate_certificate", False): - warnings.warn("Certificate validation cannot be specified on the " - "InsecureAppEngineAdapter, but was present. This " - "will be ignored and certificate validation will " - "remain off.", exc.IgnoringGAECertificateValidation) - - super(InsecureAppEngineAdapter, self).__init__( - validate_certificate=False, *args, **kwargs) - - -class _AppEnginePoolManager(object): - """Implements urllib3's PoolManager API expected by requests. - - While a real PoolManager map hostnames to reusable Connections, - AppEngine has no concept of a reusable connection to a host. - So instead, this class constructs a small Connection per request, - that is returned to the Adapter and used to access the URL. - """ - - def __init__(self, validate_certificate=True): - self.appengine_manager = gaecontrib.AppEngineManager( - validate_certificate=validate_certificate) - - def connection_from_url(self, url): - return _AppEngineConnection(self.appengine_manager, url) - - def clear(self): - pass - - -class _AppEngineConnection(object): - """Implements urllib3's HTTPConnectionPool API's urlopen(). - - This Connection's urlopen() is called with a host-relative path, - so in order to properly support opening the URL, we need to store - the full URL when this Connection is constructed from the PoolManager. - - This code wraps AppEngineManager.urlopen(), which exposes a different - API than in the original urllib3 urlopen(), and thus needs this adapter. - """ - - def __init__(self, appengine_manager, url): - self.appengine_manager = appengine_manager - self.url = url - - def urlopen(self, method, url, body=None, headers=None, retries=None, - redirect=True, assert_same_host=True, - timeout=timeout.Timeout.DEFAULT_TIMEOUT, - pool_timeout=None, release_conn=None, **response_kw): - # This function's url argument is a host-relative URL, - # but the AppEngineManager expects an absolute URL. - # So we saved out the self.url when the AppEngineConnection - # was constructed, which we then can use down below instead. - - # We once tried to verify our assumptions here, but sometimes the - # passed-in URL differs on url fragments, or "http://a.com" vs "/". - - # urllib3's App Engine adapter only uses Timeout.total, not read or - # connect. - if not timeout.total: - timeout.total = timeout._read or timeout._connect - - # Jump through the hoops necessary to call AppEngineManager's API. - return self.appengine_manager.urlopen( - method, - self.url, - body=body, - headers=headers, - retries=retries, - redirect=redirect, - timeout=timeout, - **response_kw) - - -def monkeypatch(validate_certificate=True): - """Sets up all Sessions to use AppEngineAdapter by default. - - If you don't want to deal with configuring your own Sessions, - or if you use libraries that use requests directly (ie requests.post), - then you may prefer to monkeypatch and auto-configure all Sessions. - - .. warning: : - - If ``validate_certificate`` is ``False``, certification validation will - effectively be disabled for all requests. - """ - _check_version() - # HACK: We should consider modifying urllib3 to support this cleanly, - # so that we can set a module-level variable in the sessions module, - # instead of overriding an imported HTTPAdapter as is done here. - adapter = AppEngineAdapter - if not validate_certificate: - adapter = InsecureAppEngineAdapter - - sessions.HTTPAdapter = adapter - adapters.HTTPAdapter = adapter - - -def _check_version(): - if gaecontrib is None: - raise exc.VersionMismatchError( - "The toolbelt requires at least Requests 2.10.0 to be " - "installed. Version {0} was found instead.".format( - requests.__version__ - ) - ) diff --git a/included_dependencies/requests_toolbelt/adapters/fingerprint.py b/included_dependencies/requests_toolbelt/adapters/fingerprint.py deleted file mode 100644 index 6645d349..00000000 --- a/included_dependencies/requests_toolbelt/adapters/fingerprint.py +++ /dev/null @@ -1,48 +0,0 @@ -# -*- coding: utf-8 -*- -"""Submodule containing the implementation for the FingerprintAdapter. - -This file contains an implementation of a Transport Adapter that validates -the fingerprints of SSL certificates presented upon connection. -""" -from requests.adapters import HTTPAdapter - -from .._compat import poolmanager - - -class FingerprintAdapter(HTTPAdapter): - """ - A HTTPS Adapter for Python Requests that verifies certificate fingerprints, - instead of certificate hostnames. - - Example usage: - - .. code-block:: python - - import requests - import ssl - from requests_toolbelt.adapters.fingerprint import FingerprintAdapter - - twitter_fingerprint = '...' - s = requests.Session() - s.mount( - 'https://twitter.com', - FingerprintAdapter(twitter_fingerprint) - ) - - The fingerprint should be provided as a hexadecimal string, optionally - containing colons. - """ - - __attrs__ = HTTPAdapter.__attrs__ + ['fingerprint'] - - def __init__(self, fingerprint, **kwargs): - self.fingerprint = fingerprint - - super(FingerprintAdapter, self).__init__(**kwargs) - - def init_poolmanager(self, connections, maxsize, block=False): - self.poolmanager = poolmanager.PoolManager( - num_pools=connections, - maxsize=maxsize, - block=block, - assert_fingerprint=self.fingerprint) diff --git a/included_dependencies/requests_toolbelt/adapters/host_header_ssl.py b/included_dependencies/requests_toolbelt/adapters/host_header_ssl.py deleted file mode 100644 index f34ed1aa..00000000 --- a/included_dependencies/requests_toolbelt/adapters/host_header_ssl.py +++ /dev/null @@ -1,43 +0,0 @@ -# -*- coding: utf-8 -*- -""" -requests_toolbelt.adapters.host_header_ssl -========================================== - -This file contains an implementation of the HostHeaderSSLAdapter. -""" - -from requests.adapters import HTTPAdapter - - -class HostHeaderSSLAdapter(HTTPAdapter): - """ - A HTTPS Adapter for Python Requests that sets the hostname for certificate - verification based on the Host header. - - This allows requesting the IP address directly via HTTPS without getting - a "hostname doesn't match" exception. - - Example usage: - - >>> s.mount('https://', HostHeaderSSLAdapter()) - >>> s.get("https://93.184.216.34", headers={"Host": "example.org"}) - - """ - - def send(self, request, **kwargs): - # HTTP headers are case-insensitive (RFC 7230) - host_header = None - for header in request.headers: - if header.lower() == "host": - host_header = request.headers[header] - break - - connection_pool_kwargs = self.poolmanager.connection_pool_kw - - if host_header: - connection_pool_kwargs["assert_hostname"] = host_header - elif "assert_hostname" in connection_pool_kwargs: - # an assert_hostname from a previous request may have been left - connection_pool_kwargs.pop("assert_hostname", None) - - return super(HostHeaderSSLAdapter, self).send(request, **kwargs) diff --git a/included_dependencies/requests_toolbelt/adapters/socket_options.py b/included_dependencies/requests_toolbelt/adapters/socket_options.py deleted file mode 100644 index f8aef5dd..00000000 --- a/included_dependencies/requests_toolbelt/adapters/socket_options.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -"""The implementation of the SocketOptionsAdapter.""" -import socket -import warnings -import sys - -import requests -from requests import adapters - -from .._compat import connection -from .._compat import poolmanager -from .. import exceptions as exc - - -class SocketOptionsAdapter(adapters.HTTPAdapter): - """An adapter for requests that allows users to specify socket options. - - Since version 2.4.0 of requests, it is possible to specify a custom list - of socket options that need to be set before establishing the connection. - - Example usage:: - - >>> import socket - >>> import requests - >>> from requests_toolbelt.adapters import socket_options - >>> s = requests.Session() - >>> opts = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 0)] - >>> adapter = socket_options.SocketOptionsAdapter(socket_options=opts) - >>> s.mount('http://', adapter) - - You can also take advantage of the list of default options on this class - to keep using the original options in addition to your custom options. In - that case, ``opts`` might look like:: - - >>> opts = socket_options.SocketOptionsAdapter.default_options + opts - - """ - - if connection is not None: - default_options = getattr( - connection.HTTPConnection, - 'default_socket_options', - [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] - ) - else: - default_options = [] - warnings.warn(exc.RequestsVersionTooOld, - "This version of Requests is only compatible with a " - "version of urllib3 which is too old to support " - "setting options on a socket. This adapter is " - "functionally useless.") - - def __init__(self, **kwargs): - self.socket_options = kwargs.pop('socket_options', - self.default_options) - - super(SocketOptionsAdapter, self).__init__(**kwargs) - - def init_poolmanager(self, connections, maxsize, block=False): - if requests.__build__ >= 0x020400: - # NOTE(Ian): Perhaps we should raise a warning - self.poolmanager = poolmanager.PoolManager( - num_pools=connections, - maxsize=maxsize, - block=block, - socket_options=self.socket_options - ) - else: - super(SocketOptionsAdapter, self).init_poolmanager( - connections, maxsize, block - ) - - -class TCPKeepAliveAdapter(SocketOptionsAdapter): - """An adapter for requests that turns on TCP Keep-Alive by default. - - The adapter sets 4 socket options: - - - ``SOL_SOCKET`` ``SO_KEEPALIVE`` - This turns on TCP Keep-Alive - - ``IPPROTO_TCP`` ``TCP_KEEPINTVL`` 20 - Sets the keep alive interval - - ``IPPROTO_TCP`` ``TCP_KEEPCNT`` 5 - Sets the number of keep alive probes - - ``IPPROTO_TCP`` ``TCP_KEEPIDLE`` 60 - Sets the keep alive time if the - socket library has the ``TCP_KEEPIDLE`` constant - - The latter three can be overridden by keyword arguments (respectively): - - - ``idle`` - - ``interval`` - - ``count`` - - You can use this adapter like so:: - - >>> from requests_toolbelt.adapters import socket_options - >>> tcp = socket_options.TCPKeepAliveAdapter(idle=120, interval=10) - >>> s = requests.Session() - >>> s.mount('http://', tcp) - - """ - - def __init__(self, **kwargs): - socket_options = kwargs.pop('socket_options', - SocketOptionsAdapter.default_options) - idle = kwargs.pop('idle', 60) - interval = kwargs.pop('interval', 20) - count = kwargs.pop('count', 5) - socket_options = socket_options + [ - (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - ] - - # NOTE(Ian): OSX does not have these constants defined, so we - # set them conditionally. - if getattr(socket, 'TCP_KEEPINTVL', None) is not None: - socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, - interval)] - elif sys.platform == 'darwin': - # On OSX, TCP_KEEPALIVE from netinet/tcp.h is not exported - # by python's socket module - TCP_KEEPALIVE = getattr(socket, 'TCP_KEEPALIVE', 0x10) - socket_options += [(socket.IPPROTO_TCP, TCP_KEEPALIVE, interval)] - - if getattr(socket, 'TCP_KEEPCNT', None) is not None: - socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, count)] - - if getattr(socket, 'TCP_KEEPIDLE', None) is not None: - socket_options += [(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle)] - - super(TCPKeepAliveAdapter, self).__init__( - socket_options=socket_options, **kwargs - ) diff --git a/included_dependencies/requests_toolbelt/adapters/source.py b/included_dependencies/requests_toolbelt/adapters/source.py deleted file mode 100644 index d3dda797..00000000 --- a/included_dependencies/requests_toolbelt/adapters/source.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -""" -requests_toolbelt.source_adapter -================================ - -This file contains an implementation of the SourceAddressAdapter originally -demonstrated on the Requests GitHub page. -""" -from requests.adapters import HTTPAdapter - -from .._compat import poolmanager, basestring - - -class SourceAddressAdapter(HTTPAdapter): - """ - A Source Address Adapter for Python Requests that enables you to choose the - local address to bind to. This allows you to send your HTTP requests from a - specific interface and IP address. - - Two address formats are accepted. The first is a string: this will set the - local IP address to the address given in the string, and will also choose a - semi-random high port for the local port number. - - The second is a two-tuple of the form (ip address, port): for example, - ``('10.10.10.10', 8999)``. This will set the local IP address to the first - element, and the local port to the second element. If ``0`` is used as the - port number, a semi-random high port will be selected. - - .. warning:: Setting an explicit local port can have negative interactions - with connection-pooling in Requests: in particular, it risks - the possibility of getting "Address in use" errors. The - string-only argument is generally preferred to the tuple-form. - - Example usage: - - .. code-block:: python - - import requests - from requests_toolbelt.adapters.source import SourceAddressAdapter - - s = requests.Session() - s.mount('http://', SourceAddressAdapter('10.10.10.10')) - s.mount('https://', SourceAddressAdapter(('10.10.10.10', 8999))) - """ - def __init__(self, source_address, **kwargs): - if isinstance(source_address, basestring): - self.source_address = (source_address, 0) - elif isinstance(source_address, tuple): - self.source_address = source_address - else: - raise TypeError( - "source_address must be IP address string or (ip, port) tuple" - ) - - super(SourceAddressAdapter, self).__init__(**kwargs) - - def init_poolmanager(self, connections, maxsize, block=False): - self.poolmanager = poolmanager.PoolManager( - num_pools=connections, - maxsize=maxsize, - block=block, - source_address=self.source_address) - - def proxy_manager_for(self, *args, **kwargs): - kwargs['source_address'] = self.source_address - return super(SourceAddressAdapter, self).proxy_manager_for( - *args, **kwargs) diff --git a/included_dependencies/requests_toolbelt/adapters/ssl.py b/included_dependencies/requests_toolbelt/adapters/ssl.py deleted file mode 100644 index c4a76ae4..00000000 --- a/included_dependencies/requests_toolbelt/adapters/ssl.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -""" - -requests_toolbelt.ssl_adapter -============================= - -This file contains an implementation of the SSLAdapter originally demonstrated -in this blog post: -https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/ - -""" -import requests - -from requests.adapters import HTTPAdapter - -from .._compat import poolmanager - - -class SSLAdapter(HTTPAdapter): - """ - A HTTPS Adapter for Python Requests that allows the choice of the SSL/TLS - version negotiated by Requests. This can be used either to enforce the - choice of high-security TLS versions (where supported), or to work around - misbehaving servers that fail to correctly negotiate the default TLS - version being offered. - - Example usage: - - >>> import requests - >>> import ssl - >>> from requests_toolbelt import SSLAdapter - >>> s = requests.Session() - >>> s.mount('https://', SSLAdapter(ssl.PROTOCOL_TLSv1)) - - You can replace the chosen protocol with any that are available in the - default Python SSL module. All subsequent requests that match the adapter - prefix will use the chosen SSL version instead of the default. - - This adapter will also attempt to change the SSL/TLS version negotiated by - Requests when using a proxy. However, this may not always be possible: - prior to Requests v2.4.0 the adapter did not have access to the proxy setup - code. In earlier versions of Requests, this adapter will not function - properly when used with proxies. - """ - - __attrs__ = HTTPAdapter.__attrs__ + ['ssl_version'] - - def __init__(self, ssl_version=None, **kwargs): - self.ssl_version = ssl_version - - super(SSLAdapter, self).__init__(**kwargs) - - def init_poolmanager(self, connections, maxsize, block=False): - self.poolmanager = poolmanager.PoolManager( - num_pools=connections, - maxsize=maxsize, - block=block, - ssl_version=self.ssl_version) - - if requests.__build__ >= 0x020400: - # Earlier versions of requests either don't have this method or, worse, - # don't allow passing arbitrary keyword arguments. As a result, only - # conditionally define this method. - def proxy_manager_for(self, *args, **kwargs): - kwargs['ssl_version'] = self.ssl_version - return super(SSLAdapter, self).proxy_manager_for(*args, **kwargs) diff --git a/included_dependencies/requests_toolbelt/adapters/x509.py b/included_dependencies/requests_toolbelt/adapters/x509.py deleted file mode 100644 index dc9d5dc0..00000000 --- a/included_dependencies/requests_toolbelt/adapters/x509.py +++ /dev/null @@ -1,178 +0,0 @@ -# -*- coding: utf-8 -*- -"""A X509Adapter for use with the requests library. - -This file contains an implementation of the X509Adapter that will -allow users to authenticate a request using an arbitrary -X.509 certificate without needing to convert it to a .pem file - -""" - -from OpenSSL.crypto import PKey, X509 -from cryptography import x509 -from cryptography.hazmat.primitives.serialization import (load_pem_private_key, - load_der_private_key) -from cryptography.hazmat.primitives.serialization import Encoding -from cryptography.hazmat.backends import default_backend - -from datetime import datetime -from requests.adapters import HTTPAdapter -import requests - -from .._compat import PyOpenSSLContext -from .. import exceptions as exc - -""" -importing the protocol constants from _ssl instead of ssl because only the -constants are needed and to handle issues caused by importing from ssl on -the 2.7.x line. -""" -try: - from _ssl import PROTOCOL_TLS as PROTOCOL -except ImportError: - from _ssl import PROTOCOL_SSLv23 as PROTOCOL - - -class X509Adapter(HTTPAdapter): - r"""Adapter for use with X.509 certificates. - - Provides an interface for Requests sessions to contact HTTPS urls and - authenticate with an X.509 cert by implementing the Transport Adapter - interface. This class will need to be manually instantiated and mounted - to the session - - :param pool_connections: The number of urllib3 connection pools to - cache. - :param pool_maxsize: The maximum number of connections to save in the - pool. - :param max_retries: The maximum number of retries each connection - should attempt. Note, this applies only to failed DNS lookups, - socket connections and connection timeouts, never to requests where - data has made it to the server. By default, Requests does not retry - failed connections. If you need granular control over the - conditions under which we retry a request, import urllib3's - ``Retry`` class and pass that instead. - :param pool_block: Whether the connection pool should block for - connections. - - :param bytes cert_bytes: - bytes object containing contents of a cryptography.x509Certificate - object using the encoding specified by the ``encoding`` parameter. - :param bytes pk_bytes: - bytes object containing contents of a object that implements - ``cryptography.hazmat.primitives.serialization.PrivateFormat`` - using the encoding specified by the ``encoding`` parameter. - :param password: - string or utf8 encoded bytes containing the passphrase used for the - private key. None if unencrypted. Defaults to None. - :param encoding: - Enumeration detailing the encoding method used on the ``cert_bytes`` - parameter. Can be either PEM or DER. Defaults to PEM. - :type encoding: - :class: `cryptography.hazmat.primitives.serialization.Encoding` - - Usage:: - - >>> import requests - >>> from requests_toolbelt.adapters.x509 import X509Adapter - >>> s = requests.Session() - >>> a = X509Adapter(max_retries=3, - cert_bytes=b'...', pk_bytes=b'...', encoding='...' - >>> s.mount('https://', a) - """ - - def __init__(self, *args, **kwargs): - self._check_version() - cert_bytes = kwargs.pop('cert_bytes', None) - pk_bytes = kwargs.pop('pk_bytes', None) - password = kwargs.pop('password', None) - encoding = kwargs.pop('encoding', Encoding.PEM) - - password_bytes = None - - if cert_bytes is None or not isinstance(cert_bytes, bytes): - raise ValueError('Invalid cert content provided. ' - 'You must provide an X.509 cert ' - 'formatted as a byte array.') - if pk_bytes is None or not isinstance(pk_bytes, bytes): - raise ValueError('Invalid private key content provided. ' - 'You must provide a private key ' - 'formatted as a byte array.') - - if isinstance(password, bytes): - password_bytes = password - elif password: - password_bytes = password.encode('utf8') - - self.ssl_context = create_ssl_context(cert_bytes, pk_bytes, - password_bytes, encoding) - - super(X509Adapter, self).__init__(*args, **kwargs) - - def init_poolmanager(self, *args, **kwargs): - if self.ssl_context: - kwargs['ssl_context'] = self.ssl_context - return super(X509Adapter, self).init_poolmanager(*args, **kwargs) - - def proxy_manager_for(self, *args, **kwargs): - if self.ssl_context: - kwargs['ssl_context'] = self.ssl_context - return super(X509Adapter, self).proxy_manager_for(*args, **kwargs) - - def _check_version(self): - if PyOpenSSLContext is None: - raise exc.VersionMismatchError( - "The X509Adapter requires at least Requests 2.12.0 to be " - "installed. Version {0} was found instead.".format( - requests.__version__ - ) - ) - - -def check_cert_dates(cert): - """Verify that the supplied client cert is not invalid.""" - - now = datetime.utcnow() - if cert.not_valid_after < now or cert.not_valid_before > now: - raise ValueError('Client certificate expired: Not After: ' - '{0:%Y-%m-%d %H:%M:%SZ} ' - 'Not Before: {1:%Y-%m-%d %H:%M:%SZ}' - .format(cert.not_valid_after, cert.not_valid_before)) - - -def create_ssl_context(cert_byes, pk_bytes, password=None, - encoding=Encoding.PEM): - """Create an SSL Context with the supplied cert/password. - - :param cert_bytes array of bytes containing the cert encoded - using the method supplied in the ``encoding`` parameter - :param pk_bytes array of bytes containing the private key encoded - using the method supplied in the ``encoding`` parameter - :param password array of bytes containing the passphrase to be used - with the supplied private key. None if unencrypted. - Defaults to None. - :param encoding ``cryptography.hazmat.primitives.serialization.Encoding`` - details the encoding method used on the ``cert_bytes`` and - ``pk_bytes`` parameters. Can be either PEM or DER. - Defaults to PEM. - """ - backend = default_backend() - - cert = None - key = None - if encoding == Encoding.PEM: - cert = x509.load_pem_x509_certificate(cert_byes, backend) - key = load_pem_private_key(pk_bytes, password, backend) - elif encoding == Encoding.DER: - cert = x509.load_der_x509_certificate(cert_byes, backend) - key = load_der_private_key(pk_bytes, password, backend) - else: - raise ValueError('Invalid encoding provided: Must be PEM or DER') - - if not (cert and key): - raise ValueError('Cert and key could not be parsed from ' - 'provided data') - check_cert_dates(cert) - ssl_context = PyOpenSSLContext(PROTOCOL) - ssl_context._ctx.use_certificate(X509.from_cryptography(cert)) - ssl_context._ctx.use_privatekey(PKey.from_cryptography_key(key)) - return ssl_context diff --git a/included_dependencies/requests_toolbelt/auth/__init__.py b/included_dependencies/requests_toolbelt/auth/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/included_dependencies/requests_toolbelt/auth/_digest_auth_compat.py b/included_dependencies/requests_toolbelt/auth/_digest_auth_compat.py deleted file mode 100644 index 285a6a76..00000000 --- a/included_dependencies/requests_toolbelt/auth/_digest_auth_compat.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Provide a compatibility layer for requests.auth.HTTPDigestAuth.""" -import requests - - -class _ThreadingDescriptor(object): - def __init__(self, prop, default): - self.prop = prop - self.default = default - - def __get__(self, obj, objtype=None): - return getattr(obj._thread_local, self.prop, self.default) - - def __set__(self, obj, value): - setattr(obj._thread_local, self.prop, value) - - -class _HTTPDigestAuth(requests.auth.HTTPDigestAuth): - init = _ThreadingDescriptor('init', True) - last_nonce = _ThreadingDescriptor('last_nonce', '') - nonce_count = _ThreadingDescriptor('nonce_count', 0) - chal = _ThreadingDescriptor('chal', {}) - pos = _ThreadingDescriptor('pos', None) - num_401_calls = _ThreadingDescriptor('num_401_calls', 1) - - -if requests.__build__ < 0x020800: - HTTPDigestAuth = requests.auth.HTTPDigestAuth -else: - HTTPDigestAuth = _HTTPDigestAuth diff --git a/included_dependencies/requests_toolbelt/auth/guess.py b/included_dependencies/requests_toolbelt/auth/guess.py deleted file mode 100644 index ba6de504..00000000 --- a/included_dependencies/requests_toolbelt/auth/guess.py +++ /dev/null @@ -1,146 +0,0 @@ -# -*- coding: utf-8 -*- -"""The module containing the code for GuessAuth.""" -from requests import auth -from requests import cookies - -from . import _digest_auth_compat as auth_compat, http_proxy_digest - - -class GuessAuth(auth.AuthBase): - """Guesses the auth type by the WWW-Authentication header.""" - def __init__(self, username, password): - self.username = username - self.password = password - self.auth = None - self.pos = None - - def _handle_basic_auth_401(self, r, kwargs): - if self.pos is not None: - r.request.body.seek(self.pos) - - # Consume content and release the original connection - # to allow our new request to reuse the same one. - r.content - r.raw.release_conn() - prep = r.request.copy() - if not hasattr(prep, '_cookies'): - prep._cookies = cookies.RequestsCookieJar() - cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) - prep.prepare_cookies(prep._cookies) - - self.auth = auth.HTTPBasicAuth(self.username, self.password) - prep = self.auth(prep) - _r = r.connection.send(prep, **kwargs) - _r.history.append(r) - _r.request = prep - - return _r - - def _handle_digest_auth_401(self, r, kwargs): - self.auth = auth_compat.HTTPDigestAuth(self.username, self.password) - try: - self.auth.init_per_thread_state() - except AttributeError: - # If we're not on requests 2.8.0+ this method does not exist and - # is not relevant. - pass - - # Check that the attr exists because much older versions of requests - # set this attribute lazily. For example: - # https://github.com/kennethreitz/requests/blob/33735480f77891754304e7f13e3cdf83aaaa76aa/requests/auth.py#L59 - if (hasattr(self.auth, 'num_401_calls') and - self.auth.num_401_calls is None): - self.auth.num_401_calls = 1 - # Digest auth would resend the request by itself. We can take a - # shortcut here. - return self.auth.handle_401(r, **kwargs) - - def handle_401(self, r, **kwargs): - """Resends a request with auth headers, if needed.""" - - www_authenticate = r.headers.get('www-authenticate', '').lower() - - if 'basic' in www_authenticate: - return self._handle_basic_auth_401(r, kwargs) - - if 'digest' in www_authenticate: - return self._handle_digest_auth_401(r, kwargs) - - def __call__(self, request): - if self.auth is not None: - return self.auth(request) - - try: - self.pos = request.body.tell() - except AttributeError: - pass - - request.register_hook('response', self.handle_401) - return request - - -class GuessProxyAuth(GuessAuth): - """ - Guesses the auth type by WWW-Authentication and Proxy-Authentication - headers - """ - def __init__(self, username=None, password=None, - proxy_username=None, proxy_password=None): - super(GuessProxyAuth, self).__init__(username, password) - self.proxy_username = proxy_username - self.proxy_password = proxy_password - self.proxy_auth = None - - def _handle_basic_auth_407(self, r, kwargs): - if self.pos is not None: - r.request.body.seek(self.pos) - - r.content - r.raw.release_conn() - prep = r.request.copy() - if not hasattr(prep, '_cookies'): - prep._cookies = cookies.RequestsCookieJar() - cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) - prep.prepare_cookies(prep._cookies) - - self.proxy_auth = auth.HTTPProxyAuth(self.proxy_username, - self.proxy_password) - prep = self.proxy_auth(prep) - _r = r.connection.send(prep, **kwargs) - _r.history.append(r) - _r.request = prep - - return _r - - def _handle_digest_auth_407(self, r, kwargs): - self.proxy_auth = http_proxy_digest.HTTPProxyDigestAuth( - username=self.proxy_username, - password=self.proxy_password) - - try: - self.auth.init_per_thread_state() - except AttributeError: - pass - - return self.proxy_auth.handle_407(r, **kwargs) - - def handle_407(self, r, **kwargs): - proxy_authenticate = r.headers.get('Proxy-Authenticate', '').lower() - - if 'basic' in proxy_authenticate: - return self._handle_basic_auth_407(r, kwargs) - - if 'digest' in proxy_authenticate: - return self._handle_digest_auth_407(r, kwargs) - - def __call__(self, request): - if self.proxy_auth is not None: - request = self.proxy_auth(request) - - try: - self.pos = request.body.tell() - except AttributeError: - pass - - request.register_hook('response', self.handle_407) - return super(GuessProxyAuth, self).__call__(request) diff --git a/included_dependencies/requests_toolbelt/auth/handler.py b/included_dependencies/requests_toolbelt/auth/handler.py deleted file mode 100644 index 7729c11b..00000000 --- a/included_dependencies/requests_toolbelt/auth/handler.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -""" - -requests_toolbelt.auth.handler -============================== - -This holds all of the implementation details of the Authentication Handler. - -""" - -from requests.auth import AuthBase, HTTPBasicAuth -from requests.compat import urlparse, urlunparse - - -class AuthHandler(AuthBase): - - """ - - The ``AuthHandler`` object takes a dictionary of domains paired with - authentication strategies and will use this to determine which credentials - to use when making a request. For example, you could do the following: - - .. code-block:: python - - from requests import HTTPDigestAuth - from requests_toolbelt.auth.handler import AuthHandler - - import requests - - auth = AuthHandler({ - 'https://api.github.com': ('sigmavirus24', 'fakepassword'), - 'https://example.com': HTTPDigestAuth('username', 'password') - }) - - r = requests.get('https://api.github.com/user', auth=auth) - # => - r = requests.get('https://example.com/some/path', auth=auth) - # => - - s = requests.Session() - s.auth = auth - r = s.get('https://api.github.com/user') - # => - - .. warning:: - - :class:`requests.auth.HTTPDigestAuth` is not yet thread-safe. If you - use :class:`AuthHandler` across multiple threads you should - instantiate a new AuthHandler for each thread with a new - HTTPDigestAuth instance for each thread. - - """ - - def __init__(self, strategies): - self.strategies = dict(strategies) - self._make_uniform() - - def __call__(self, request): - auth = self.get_strategy_for(request.url) - return auth(request) - - def __repr__(self): - return ''.format(self.strategies) - - def _make_uniform(self): - existing_strategies = list(self.strategies.items()) - self.strategies = {} - - for (k, v) in existing_strategies: - self.add_strategy(k, v) - - @staticmethod - def _key_from_url(url): - parsed = urlparse(url) - return urlunparse((parsed.scheme.lower(), - parsed.netloc.lower(), - '', '', '', '')) - - def add_strategy(self, domain, strategy): - """Add a new domain and authentication strategy. - - :param str domain: The domain you wish to match against. For example: - ``'https://api.github.com'`` - :param str strategy: The authentication strategy you wish to use for - that domain. For example: ``('username', 'password')`` or - ``requests.HTTPDigestAuth('username', 'password')`` - - .. code-block:: python - - a = AuthHandler({}) - a.add_strategy('https://api.github.com', ('username', 'password')) - - """ - # Turn tuples into Basic Authentication objects - if isinstance(strategy, tuple): - strategy = HTTPBasicAuth(*strategy) - - key = self._key_from_url(domain) - self.strategies[key] = strategy - - def get_strategy_for(self, url): - """Retrieve the authentication strategy for a specified URL. - - :param str url: The full URL you will be making a request against. For - example, ``'https://api.github.com/user'`` - :returns: Callable that adds authentication to a request. - - .. code-block:: python - - import requests - a = AuthHandler({'example.com', ('foo', 'bar')}) - strategy = a.get_strategy_for('http://example.com/example') - assert isinstance(strategy, requests.auth.HTTPBasicAuth) - - """ - key = self._key_from_url(url) - return self.strategies.get(key, NullAuthStrategy()) - - def remove_strategy(self, domain): - """Remove the domain and strategy from the collection of strategies. - - :param str domain: The domain you wish remove. For example, - ``'https://api.github.com'``. - - .. code-block:: python - - a = AuthHandler({'example.com', ('foo', 'bar')}) - a.remove_strategy('example.com') - assert a.strategies == {} - - """ - key = self._key_from_url(domain) - if key in self.strategies: - del self.strategies[key] - - -class NullAuthStrategy(AuthBase): - def __repr__(self): - return '' - - def __call__(self, r): - return r diff --git a/included_dependencies/requests_toolbelt/auth/http_proxy_digest.py b/included_dependencies/requests_toolbelt/auth/http_proxy_digest.py deleted file mode 100644 index d4143808..00000000 --- a/included_dependencies/requests_toolbelt/auth/http_proxy_digest.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- -"""The module containing HTTPProxyDigestAuth.""" -import re - -from requests import cookies, utils - -from . import _digest_auth_compat as auth - - -class HTTPProxyDigestAuth(auth.HTTPDigestAuth): - """HTTP digest authentication between proxy - - :param stale_rejects: The number of rejects indicate that: - the client may wish to simply retry the request - with a new encrypted response, without reprompting the user for a - new username and password. i.e., retry build_digest_header - :type stale_rejects: int - """ - _pat = re.compile(r'digest ', flags=re.IGNORECASE) - - def __init__(self, *args, **kwargs): - super(HTTPProxyDigestAuth, self).__init__(*args, **kwargs) - self.stale_rejects = 0 - - self.init_per_thread_state() - - @property - def stale_rejects(self): - thread_local = getattr(self, '_thread_local', None) - if thread_local is None: - return self._stale_rejects - return thread_local.stale_rejects - - @stale_rejects.setter - def stale_rejects(self, value): - thread_local = getattr(self, '_thread_local', None) - if thread_local is None: - self._stale_rejects = value - else: - thread_local.stale_rejects = value - - def init_per_thread_state(self): - try: - super(HTTPProxyDigestAuth, self).init_per_thread_state() - except AttributeError: - # If we're not on requests 2.8.0+ this method does not exist - pass - - def handle_407(self, r, **kwargs): - """Handle HTTP 407 only once, otherwise give up - - :param r: current response - :returns: responses, along with the new response - """ - if r.status_code == 407 and self.stale_rejects < 2: - s_auth = r.headers.get("proxy-authenticate") - if s_auth is None: - raise IOError( - "proxy server violated RFC 7235:" - "407 response MUST contain header proxy-authenticate") - elif not self._pat.match(s_auth): - return r - - self.chal = utils.parse_dict_header( - self._pat.sub('', s_auth, count=1)) - - # if we present the user/passwd and still get rejected - # http://tools.ietf.org/html/rfc2617#section-3.2.1 - if ('Proxy-Authorization' in r.request.headers and - 'stale' in self.chal): - if self.chal['stale'].lower() == 'true': # try again - self.stale_rejects += 1 - # wrong user/passwd - elif self.chal['stale'].lower() == 'false': - raise IOError("User or password is invalid") - - # Consume content and release the original connection - # to allow our new request to reuse the same one. - r.content - r.close() - prep = r.request.copy() - cookies.extract_cookies_to_jar(prep._cookies, r.request, r.raw) - prep.prepare_cookies(prep._cookies) - - prep.headers['Proxy-Authorization'] = self.build_digest_header( - prep.method, prep.url) - _r = r.connection.send(prep, **kwargs) - _r.history.append(r) - _r.request = prep - - return _r - else: # give up authenticate - return r - - def __call__(self, r): - self.init_per_thread_state() - # if we have nonce, then just use it, otherwise server will tell us - if self.last_nonce: - r.headers['Proxy-Authorization'] = self.build_digest_header( - r.method, r.url - ) - r.register_hook('response', self.handle_407) - return r diff --git a/included_dependencies/requests_toolbelt/cookies/__init__.py b/included_dependencies/requests_toolbelt/cookies/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/included_dependencies/requests_toolbelt/cookies/forgetful.py b/included_dependencies/requests_toolbelt/cookies/forgetful.py deleted file mode 100644 index 33203638..00000000 --- a/included_dependencies/requests_toolbelt/cookies/forgetful.py +++ /dev/null @@ -1,7 +0,0 @@ -"""The module containing the code for ForgetfulCookieJar.""" -from requests.cookies import RequestsCookieJar - - -class ForgetfulCookieJar(RequestsCookieJar): - def set_cookie(self, *args, **kwargs): - return diff --git a/included_dependencies/requests_toolbelt/downloadutils/__init__.py b/included_dependencies/requests_toolbelt/downloadutils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/included_dependencies/requests_toolbelt/downloadutils/stream.py b/included_dependencies/requests_toolbelt/downloadutils/stream.py deleted file mode 100644 index eed60a72..00000000 --- a/included_dependencies/requests_toolbelt/downloadutils/stream.py +++ /dev/null @@ -1,177 +0,0 @@ -# -*- coding: utf-8 -*- -"""Utilities for dealing with streamed requests.""" -import collections -import os.path -import re - -from .. import exceptions as exc - -# Regular expressions stolen from werkzeug/http.py -# cd2c97bb0a076da2322f11adce0b2731f9193396 L62-L64 -_QUOTED_STRING_RE = r'"[^"\\]*(?:\\.[^"\\]*)*"' -_OPTION_HEADER_PIECE_RE = re.compile( - r';\s*(%s|[^\s;=]+)\s*(?:=\s*(%s|[^;]+))?\s*' % (_QUOTED_STRING_RE, - _QUOTED_STRING_RE) -) -_DEFAULT_CHUNKSIZE = 512 - - -def _get_filename(content_disposition): - for match in _OPTION_HEADER_PIECE_RE.finditer(content_disposition): - k, v = match.groups() - if k == 'filename': - # ignore any directory paths in the filename - return os.path.split(v)[1] - return None - - -def get_download_file_path(response, path): - """ - Given a response and a path, return a file path for a download. - - If a ``path`` parameter is a directory, this function will parse the - ``Content-Disposition`` header on the response to determine the name of the - file as reported by the server, and return a file path in the specified - directory. - - If ``path`` is empty or None, this function will return a path relative - to the process' current working directory. - - If path is a full file path, return it. - - :param response: A Response object from requests - :type response: requests.models.Response - :param str path: Directory or file path. - :returns: full file path to download as - :rtype: str - :raises: :class:`requests_toolbelt.exceptions.StreamingError` - """ - path_is_dir = path and os.path.isdir(path) - - if path and not path_is_dir: - # fully qualified file path - filepath = path - else: - response_filename = _get_filename( - response.headers.get('content-disposition', '') - ) - if not response_filename: - raise exc.StreamingError('No filename given to stream response to') - - if path_is_dir: - # directory to download to - filepath = os.path.join(path, response_filename) - else: - # fallback to downloading to current working directory - filepath = response_filename - - return filepath - - -def stream_response_to_file(response, path=None, chunksize=_DEFAULT_CHUNKSIZE): - """Stream a response body to the specified file. - - Either use the ``path`` provided or use the name provided in the - ``Content-Disposition`` header. - - .. warning:: - - If you pass this function an open file-like object as the ``path`` - parameter, the function will not close that file for you. - - .. warning:: - - This function will not automatically close the response object - passed in as the ``response`` parameter. - - If a ``path`` parameter is a directory, this function will parse the - ``Content-Disposition`` header on the response to determine the name of the - file as reported by the server, and return a file path in the specified - directory. If no ``path`` parameter is supplied, this function will default - to the process' current working directory. - - .. code-block:: python - - import requests - from requests_toolbelt import exceptions - from requests_toolbelt.downloadutils import stream - - r = requests.get(url, stream=True) - try: - filename = stream.stream_response_to_file(r) - except exceptions.StreamingError as e: - # The toolbelt could not find the filename in the - # Content-Disposition - print(e.message) - - You can also specify the filename as a string. This will be passed to - the built-in :func:`open` and we will read the content into the file. - - .. code-block:: python - - import requests - from requests_toolbelt.downloadutils import stream - - r = requests.get(url, stream=True) - filename = stream.stream_response_to_file(r, path='myfile') - - If the calculated download file path already exists, this function will - raise a StreamingError. - - Instead, if you want to manage the file object yourself, you need to - provide either a :class:`io.BytesIO` object or a file opened with the - `'b'` flag. See the two examples below for more details. - - .. code-block:: python - - import requests - from requests_toolbelt.downloadutils import stream - - with open('myfile', 'wb') as fd: - r = requests.get(url, stream=True) - filename = stream.stream_response_to_file(r, path=fd) - - print('{0} saved to {1}'.format(url, filename)) - - .. code-block:: python - - import io - import requests - from requests_toolbelt.downloadutils import stream - - b = io.BytesIO() - r = requests.get(url, stream=True) - filename = stream.stream_response_to_file(r, path=b) - assert filename is None - - :param response: A Response object from requests - :type response: requests.models.Response - :param path: *(optional)*, Either a string with the path to the location - to save the response content, or a file-like object expecting bytes. - :type path: :class:`str`, or object with a :meth:`write` - :param int chunksize: (optional), Size of chunk to attempt to stream - (default 512B). - :returns: The name of the file, if one can be determined, else None - :rtype: str - :raises: :class:`requests_toolbelt.exceptions.StreamingError` - """ - pre_opened = False - fd = None - filename = None - if path and isinstance(getattr(path, 'write', None), collections.Callable): - pre_opened = True - fd = path - filename = getattr(fd, 'name', None) - else: - filename = get_download_file_path(response, path) - if os.path.exists(filename): - raise exc.StreamingError("File already exists: %s" % filename) - fd = open(filename, 'wb') - - for chunk in response.iter_content(chunk_size=chunksize): - fd.write(chunk) - - if not pre_opened: - fd.close() - - return filename diff --git a/included_dependencies/requests_toolbelt/downloadutils/tee.py b/included_dependencies/requests_toolbelt/downloadutils/tee.py deleted file mode 100644 index ecc7d0cd..00000000 --- a/included_dependencies/requests_toolbelt/downloadutils/tee.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Tee function implementations.""" -import io - -_DEFAULT_CHUNKSIZE = 65536 - -__all__ = ['tee', 'tee_to_file', 'tee_to_bytearray'] - - -def _tee(response, callback, chunksize, decode_content): - for chunk in response.raw.stream(amt=chunksize, - decode_content=decode_content): - callback(chunk) - yield chunk - - -def tee(response, fileobject, chunksize=_DEFAULT_CHUNKSIZE, - decode_content=None): - """Stream the response both to the generator and a file. - - This will stream the response body while writing the bytes to - ``fileobject``. - - Example usage: - - .. code-block:: python - - resp = requests.get(url, stream=True) - with open('save_file', 'wb') as save_file: - for chunk in tee(resp, save_file): - # do stuff with chunk - - .. code-block:: python - - import io - - resp = requests.get(url, stream=True) - fileobject = io.BytesIO() - - for chunk in tee(resp, fileobject): - # do stuff with chunk - - :param response: Response from requests. - :type response: requests.Response - :param fileobject: Writable file-like object. - :type fileobject: file, io.BytesIO - :param int chunksize: (optional), Size of chunk to attempt to stream. - :param bool decode_content: (optional), If True, this will decode the - compressed content of the response. - :raises: TypeError if the fileobject wasn't opened with the right mode - or isn't a BytesIO object. - """ - # We will be streaming the raw bytes from over the wire, so we need to - # ensure that writing to the fileobject will preserve those bytes. On - # Python3, if the user passes an io.StringIO, this will fail, so we need - # to check for BytesIO instead. - if not ('b' in getattr(fileobject, 'mode', '') or - isinstance(fileobject, io.BytesIO)): - raise TypeError('tee() will write bytes directly to this fileobject' - ', it must be opened with the "b" flag if it is a file' - ' or inherit from io.BytesIO.') - - return _tee(response, fileobject.write, chunksize, decode_content) - - -def tee_to_file(response, filename, chunksize=_DEFAULT_CHUNKSIZE, - decode_content=None): - """Stream the response both to the generator and a file. - - This will open a file named ``filename`` and stream the response body - while writing the bytes to the opened file object. - - Example usage: - - .. code-block:: python - - resp = requests.get(url, stream=True) - for chunk in tee_to_file(resp, 'save_file'): - # do stuff with chunk - - :param response: Response from requests. - :type response: requests.Response - :param str filename: Name of file in which we write the response content. - :param int chunksize: (optional), Size of chunk to attempt to stream. - :param bool decode_content: (optional), If True, this will decode the - compressed content of the response. - """ - with open(filename, 'wb') as fd: - for chunk in tee(response, fd, chunksize, decode_content): - yield chunk - - -def tee_to_bytearray(response, bytearr, chunksize=_DEFAULT_CHUNKSIZE, - decode_content=None): - """Stream the response both to the generator and a bytearray. - - This will stream the response provided to the function, add them to the - provided :class:`bytearray` and yield them to the user. - - .. note:: - - This uses the :meth:`bytearray.extend` by default instead of passing - the bytearray into the ``readinto`` method. - - Example usage: - - .. code-block:: python - - b = bytearray() - resp = requests.get(url, stream=True) - for chunk in tee_to_bytearray(resp, b): - # do stuff with chunk - - :param response: Response from requests. - :type response: requests.Response - :param bytearray bytearr: Array to add the streamed bytes to. - :param int chunksize: (optional), Size of chunk to attempt to stream. - :param bool decode_content: (optional), If True, this will decode the - compressed content of the response. - """ - if not isinstance(bytearr, bytearray): - raise TypeError('tee_to_bytearray() expects bytearr to be a ' - 'bytearray') - return _tee(response, bytearr.extend, chunksize, decode_content) diff --git a/included_dependencies/requests_toolbelt/exceptions.py b/included_dependencies/requests_toolbelt/exceptions.py deleted file mode 100644 index 32ade215..00000000 --- a/included_dependencies/requests_toolbelt/exceptions.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- -"""Collection of exceptions raised by requests-toolbelt.""" - - -class StreamingError(Exception): - """Used in :mod:`requests_toolbelt.downloadutils.stream`.""" - pass - - -class VersionMismatchError(Exception): - """Used to indicate a version mismatch in the version of requests required. - - The feature in use requires a newer version of Requests to function - appropriately but the version installed is not sufficient. - """ - pass - - -class RequestsVersionTooOld(Warning): - """Used to indicate that the Requests version is too old. - - If the version of Requests is too old to support a feature, we will issue - this warning to the user. - """ - pass - - -class IgnoringGAECertificateValidation(Warning): - """Used to indicate that given GAE validation behavior will be ignored. - - If the user has tried to specify certificate validation when using the - insecure AppEngine adapter, it will be ignored (certificate validation will - remain off), so we will issue this warning to the user. - - In :class:`requests_toolbelt.adapters.appengine.InsecureAppEngineAdapter`. - """ - pass diff --git a/included_dependencies/requests_toolbelt/multipart/__init__.py b/included_dependencies/requests_toolbelt/multipart/__init__.py deleted file mode 100644 index 4bc49660..00000000 --- a/included_dependencies/requests_toolbelt/multipart/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -requests_toolbelt.multipart -=========================== - -See http://toolbelt.rtfd.org/ for documentation - -:copyright: (c) 2014 by Ian Cordasco and Cory Benfield -:license: Apache v2.0, see LICENSE for more details -""" - -from .encoder import MultipartEncoder, MultipartEncoderMonitor -from .decoder import MultipartDecoder -from .decoder import ImproperBodyPartContentException -from .decoder import NonMultipartContentTypeException - -__title__ = 'requests-toolbelt' -__authors__ = 'Ian Cordasco, Cory Benfield' -__license__ = 'Apache v2.0' -__copyright__ = 'Copyright 2014 Ian Cordasco, Cory Benfield' - -__all__ = [ - 'MultipartEncoder', - 'MultipartEncoderMonitor', - 'MultipartDecoder', - 'ImproperBodyPartContentException', - 'NonMultipartContentTypeException', - '__title__', - '__authors__', - '__license__', - '__copyright__', -] diff --git a/included_dependencies/requests_toolbelt/multipart/decoder.py b/included_dependencies/requests_toolbelt/multipart/decoder.py deleted file mode 100644 index 19d86af9..00000000 --- a/included_dependencies/requests_toolbelt/multipart/decoder.py +++ /dev/null @@ -1,156 +0,0 @@ -# -*- coding: utf-8 -*- -""" - -requests_toolbelt.multipart.decoder -=================================== - -This holds all the implementation details of the MultipartDecoder - -""" - -import sys -import email.parser -from .encoder import encode_with -from requests.structures import CaseInsensitiveDict - - -def _split_on_find(content, bound): - point = content.find(bound) - return content[:point], content[point + len(bound):] - - -class ImproperBodyPartContentException(Exception): - pass - - -class NonMultipartContentTypeException(Exception): - pass - - -def _header_parser(string, encoding): - major = sys.version_info[0] - if major == 3: - string = string.decode(encoding) - headers = email.parser.HeaderParser().parsestr(string).items() - return ( - (encode_with(k, encoding), encode_with(v, encoding)) - for k, v in headers - ) - - -class BodyPart(object): - """ - - The ``BodyPart`` object is a ``Response``-like interface to an individual - subpart of a multipart response. It is expected that these will - generally be created by objects of the ``MultipartDecoder`` class. - - Like ``Response``, there is a ``CaseInsensitiveDict`` object named headers, - ``content`` to access bytes, ``text`` to access unicode, and ``encoding`` - to access the unicode codec. - - """ - - def __init__(self, content, encoding): - self.encoding = encoding - headers = {} - # Split into header section (if any) and the content - if b'\r\n\r\n' in content: - first, self.content = _split_on_find(content, b'\r\n\r\n') - if first != b'': - headers = _header_parser(first.lstrip(), encoding) - else: - raise ImproperBodyPartContentException( - 'content does not contain CR-LF-CR-LF' - ) - self.headers = CaseInsensitiveDict(headers) - - @property - def text(self): - """Content of the ``BodyPart`` in unicode.""" - return self.content.decode(self.encoding) - - -class MultipartDecoder(object): - """ - - The ``MultipartDecoder`` object parses the multipart payload of - a bytestring into a tuple of ``Response``-like ``BodyPart`` objects. - - The basic usage is:: - - import requests - from requests_toolbelt import MultipartDecoder - - response = request.get(url) - decoder = MultipartDecoder.from_response(response) - for part in decoder.parts: - print(part.headers['content-type']) - - If the multipart content is not from a response, basic usage is:: - - from requests_toolbelt import MultipartDecoder - - decoder = MultipartDecoder(content, content_type) - for part in decoder.parts: - print(part.headers['content-type']) - - For both these usages, there is an optional ``encoding`` parameter. This is - a string, which is the name of the unicode codec to use (default is - ``'utf-8'``). - - """ - def __init__(self, content, content_type, encoding='utf-8'): - #: Original Content-Type header - self.content_type = content_type - #: Response body encoding - self.encoding = encoding - #: Parsed parts of the multipart response body - self.parts = tuple() - self._find_boundary() - self._parse_body(content) - - def _find_boundary(self): - ct_info = tuple(x.strip() for x in self.content_type.split(';')) - mimetype = ct_info[0] - if mimetype.split('/')[0].lower() != 'multipart': - raise NonMultipartContentTypeException( - "Unexpected mimetype in content-type: '{0}'".format(mimetype) - ) - for item in ct_info[1:]: - attr, value = _split_on_find( - item, - '=' - ) - if attr.lower() == 'boundary': - self.boundary = encode_with(value.strip('"'), self.encoding) - - @staticmethod - def _fix_first_part(part, boundary_marker): - bm_len = len(boundary_marker) - if boundary_marker == part[:bm_len]: - return part[bm_len:] - else: - return part - - def _parse_body(self, content): - boundary = b''.join((b'--', self.boundary)) - - def body_part(part): - fixed = MultipartDecoder._fix_first_part(part, boundary) - return BodyPart(fixed, self.encoding) - - def test_part(part): - return (part != b'' and - part != b'\r\n' and - part[:4] != b'--\r\n' and - part != b'--') - - parts = content.split(b''.join((b'\r\n', boundary))) - self.parts = tuple(body_part(x) for x in parts if test_part(x)) - - @classmethod - def from_response(cls, response, encoding='utf-8'): - content = response.content - content_type = response.headers.get('content-type', None) - return cls(content, content_type, encoding) diff --git a/included_dependencies/requests_toolbelt/multipart/encoder.py b/included_dependencies/requests_toolbelt/multipart/encoder.py deleted file mode 100644 index e4c7cd7a..00000000 --- a/included_dependencies/requests_toolbelt/multipart/encoder.py +++ /dev/null @@ -1,655 +0,0 @@ -# -*- coding: utf-8 -*- -""" - -requests_toolbelt.multipart.encoder -=================================== - -This holds all of the implementation details of the MultipartEncoder - -""" -import contextlib -import io -import os -from uuid import uuid4 - -import requests - -from .._compat import fields - - -class FileNotSupportedError(Exception): - """File not supported error.""" - - -class MultipartEncoder(object): - - """ - - The ``MultipartEncoder`` object is a generic interface to the engine that - will create a ``multipart/form-data`` body for you. - - The basic usage is: - - .. code-block:: python - - import requests - from requests_toolbelt import MultipartEncoder - - encoder = MultipartEncoder({'field': 'value', - 'other_field', 'other_value'}) - r = requests.post('https://httpbin.org/post', data=encoder, - headers={'Content-Type': encoder.content_type}) - - If you do not need to take advantage of streaming the post body, you can - also do: - - .. code-block:: python - - r = requests.post('https://httpbin.org/post', - data=encoder.to_string(), - headers={'Content-Type': encoder.content_type}) - - If you want the encoder to use a specific order, you can use an - OrderedDict or more simply, a list of tuples: - - .. code-block:: python - - encoder = MultipartEncoder([('field', 'value'), - ('other_field', 'other_value')]) - - .. versionchanged:: 0.4.0 - - You can also provide tuples as part values as you would provide them to - requests' ``files`` parameter. - - .. code-block:: python - - encoder = MultipartEncoder({ - 'field': ('file_name', b'{"a": "b"}', 'application/json', - {'X-My-Header': 'my-value'}) - ]) - - .. warning:: - - This object will end up directly in :mod:`httplib`. Currently, - :mod:`httplib` has a hard-coded read size of **8192 bytes**. This - means that it will loop until the file has been read and your upload - could take a while. This is **not** a bug in requests. A feature is - being considered for this object to allow you, the user, to specify - what size should be returned on a read. If you have opinions on this, - please weigh in on `this issue`_. - - .. _this issue: - https://github.com/requests/toolbelt/issues/75 - - """ - - def __init__(self, fields, boundary=None, encoding='utf-8'): - #: Boundary value either passed in by the user or created - self.boundary_value = boundary or uuid4().hex - - # Computed boundary - self.boundary = '--{0}'.format(self.boundary_value) - - #: Encoding of the data being passed in - self.encoding = encoding - - # Pre-encoded boundary - self._encoded_boundary = b''.join([ - encode_with(self.boundary, self.encoding), - encode_with('\r\n', self.encoding) - ]) - - #: Fields provided by the user - self.fields = fields - - #: Whether or not the encoder is finished - self.finished = False - - #: Pre-computed parts of the upload - self.parts = [] - - # Pre-computed parts iterator - self._iter_parts = iter([]) - - # The part we're currently working with - self._current_part = None - - # Cached computation of the body's length - self._len = None - - # Our buffer - self._buffer = CustomBytesIO(encoding=encoding) - - # Pre-compute each part's headers - self._prepare_parts() - - # Load boundary into buffer - self._write_boundary() - - @property - def len(self): - """Length of the multipart/form-data body. - - requests will first attempt to get the length of the body by calling - ``len(body)`` and then by checking for the ``len`` attribute. - - On 32-bit systems, the ``__len__`` method cannot return anything - larger than an integer (in C) can hold. If the total size of the body - is even slightly larger than 4GB users will see an OverflowError. This - manifested itself in `bug #80`_. - - As such, we now calculate the length lazily as a property. - - .. _bug #80: - https://github.com/requests/toolbelt/issues/80 - """ - # If _len isn't already calculated, calculate, return, and set it - return self._len or self._calculate_length() - - def __repr__(self): - return ''.format(self.fields) - - def _calculate_length(self): - """ - This uses the parts to calculate the length of the body. - - This returns the calculated length so __len__ can be lazy. - """ - boundary_len = len(self.boundary) # Length of --{boundary} - # boundary length + header length + body length + len('\r\n') * 2 - self._len = sum( - (boundary_len + total_len(p) + 4) for p in self.parts - ) + boundary_len + 4 - return self._len - - def _calculate_load_amount(self, read_size): - """This calculates how many bytes need to be added to the buffer. - - When a consumer read's ``x`` from the buffer, there are two cases to - satisfy: - - 1. Enough data in the buffer to return the requested amount - 2. Not enough data - - This function uses the amount of unread bytes in the buffer and - determines how much the Encoder has to load before it can return the - requested amount of bytes. - - :param int read_size: the number of bytes the consumer requests - :returns: int -- the number of bytes that must be loaded into the - buffer before the read can be satisfied. This will be strictly - non-negative - """ - amount = read_size - total_len(self._buffer) - return amount if amount > 0 else 0 - - def _load(self, amount): - """Load ``amount`` number of bytes into the buffer.""" - self._buffer.smart_truncate() - part = self._current_part or self._next_part() - while amount == -1 or amount > 0: - written = 0 - if part and not part.bytes_left_to_write(): - written += self._write(b'\r\n') - written += self._write_boundary() - part = self._next_part() - - if not part: - written += self._write_closing_boundary() - self.finished = True - break - - written += part.write_to(self._buffer, amount) - - if amount != -1: - amount -= written - - def _next_part(self): - try: - p = self._current_part = next(self._iter_parts) - except StopIteration: - p = None - return p - - def _iter_fields(self): - _fields = self.fields - if hasattr(self.fields, 'items'): - _fields = list(self.fields.items()) - for k, v in _fields: - file_name = None - file_type = None - file_headers = None - if isinstance(v, (list, tuple)): - if len(v) == 2: - file_name, file_pointer = v - elif len(v) == 3: - file_name, file_pointer, file_type = v - else: - file_name, file_pointer, file_type, file_headers = v - else: - file_pointer = v - - field = fields.RequestField(name=k, data=file_pointer, - filename=file_name, - headers=file_headers) - field.make_multipart(content_type=file_type) - yield field - - def _prepare_parts(self): - """This uses the fields provided by the user and creates Part objects. - - It populates the `parts` attribute and uses that to create a - generator for iteration. - """ - enc = self.encoding - self.parts = [Part.from_field(f, enc) for f in self._iter_fields()] - self._iter_parts = iter(self.parts) - - def _write(self, bytes_to_write): - """Write the bytes to the end of the buffer. - - :param bytes bytes_to_write: byte-string (or bytearray) to append to - the buffer - :returns: int -- the number of bytes written - """ - return self._buffer.append(bytes_to_write) - - def _write_boundary(self): - """Write the boundary to the end of the buffer.""" - return self._write(self._encoded_boundary) - - def _write_closing_boundary(self): - """Write the bytes necessary to finish a multipart/form-data body.""" - with reset(self._buffer): - self._buffer.seek(-2, 2) - self._buffer.write(b'--\r\n') - return 2 - - def _write_headers(self, headers): - """Write the current part's headers to the buffer.""" - return self._write(encode_with(headers, self.encoding)) - - @property - def content_type(self): - return str( - 'multipart/form-data; boundary={0}'.format(self.boundary_value) - ) - - def to_string(self): - """Return the entirety of the data in the encoder. - - .. note:: - - This simply reads all of the data it can. If you have started - streaming or reading data from the encoder, this method will only - return whatever data is left in the encoder. - - .. note:: - - This method affects the internal state of the encoder. Calling - this method will exhaust the encoder. - - :returns: the multipart message - :rtype: bytes - """ - - return self.read() - - def read(self, size=-1): - """Read data from the streaming encoder. - - :param int size: (optional), If provided, ``read`` will return exactly - that many bytes. If it is not provided, it will return the - remaining bytes. - :returns: bytes - """ - if self.finished: - return self._buffer.read(size) - - bytes_to_load = size - if bytes_to_load != -1 and bytes_to_load is not None: - bytes_to_load = self._calculate_load_amount(int(size)) - - self._load(bytes_to_load) - return self._buffer.read(size) - - -def IDENTITY(monitor): - return monitor - - -class MultipartEncoderMonitor(object): - - """ - An object used to monitor the progress of a :class:`MultipartEncoder`. - - The :class:`MultipartEncoder` should only be responsible for preparing and - streaming the data. For anyone who wishes to monitor it, they shouldn't be - using that instance to manage that as well. Using this class, they can - monitor an encoder and register a callback. The callback receives the - instance of the monitor. - - To use this monitor, you construct your :class:`MultipartEncoder` as you - normally would. - - .. code-block:: python - - from requests_toolbelt import (MultipartEncoder, - MultipartEncoderMonitor) - import requests - - def callback(monitor): - # Do something with this information - pass - - m = MultipartEncoder(fields={'field0': 'value0'}) - monitor = MultipartEncoderMonitor(m, callback) - headers = {'Content-Type': monitor.content_type} - r = requests.post('https://httpbin.org/post', data=monitor, - headers=headers) - - Alternatively, if your use case is very simple, you can use the following - pattern. - - .. code-block:: python - - from requests_toolbelt import MultipartEncoderMonitor - import requests - - def callback(monitor): - # Do something with this information - pass - - monitor = MultipartEncoderMonitor.from_fields( - fields={'field0': 'value0'}, callback - ) - headers = {'Content-Type': montior.content_type} - r = requests.post('https://httpbin.org/post', data=monitor, - headers=headers) - - """ - - def __init__(self, encoder, callback=None): - #: Instance of the :class:`MultipartEncoder` being monitored - self.encoder = encoder - - #: Optionally function to call after a read - self.callback = callback or IDENTITY - - #: Number of bytes already read from the :class:`MultipartEncoder` - #: instance - self.bytes_read = 0 - - #: Avoid the same problem in bug #80 - self.len = self.encoder.len - - @classmethod - def from_fields(cls, fields, boundary=None, encoding='utf-8', - callback=None): - encoder = MultipartEncoder(fields, boundary, encoding) - return cls(encoder, callback) - - @property - def content_type(self): - return self.encoder.content_type - - def to_string(self): - return self.read() - - def read(self, size=-1): - string = self.encoder.read(size) - self.bytes_read += len(string) - self.callback(self) - return string - - -def encode_with(string, encoding): - """Encoding ``string`` with ``encoding`` if necessary. - - :param str string: If string is a bytes object, it will not encode it. - Otherwise, this function will encode it with the provided encoding. - :param str encoding: The encoding with which to encode string. - :returns: encoded bytes object - """ - if not (string is None or isinstance(string, bytes)): - return string.encode(encoding) - return string - - -def readable_data(data, encoding): - """Coerce the data to an object with a ``read`` method.""" - if hasattr(data, 'read'): - return data - - return CustomBytesIO(data, encoding) - - -def total_len(o): - if hasattr(o, '__len__'): - return len(o) - - if hasattr(o, 'len'): - return o.len - - if hasattr(o, 'fileno'): - try: - fileno = o.fileno() - except io.UnsupportedOperation: - pass - else: - return os.fstat(fileno).st_size - - if hasattr(o, 'getvalue'): - # e.g. BytesIO, cStringIO.StringIO - return len(o.getvalue()) - - -@contextlib.contextmanager -def reset(buffer): - """Keep track of the buffer's current position and write to the end. - - This is a context manager meant to be used when adding data to the buffer. - It eliminates the need for every function to be concerned with the - position of the cursor in the buffer. - """ - original_position = buffer.tell() - buffer.seek(0, 2) - yield - buffer.seek(original_position, 0) - - -def coerce_data(data, encoding): - """Ensure that every object's __len__ behaves uniformly.""" - if not isinstance(data, CustomBytesIO): - if hasattr(data, 'getvalue'): - return CustomBytesIO(data.getvalue(), encoding) - - if hasattr(data, 'fileno'): - return FileWrapper(data) - - if not hasattr(data, 'read'): - return CustomBytesIO(data, encoding) - - return data - - -def to_list(fields): - if hasattr(fields, 'items'): - return list(fields.items()) - return list(fields) - - -class Part(object): - def __init__(self, headers, body): - self.headers = headers - self.body = body - self.headers_unread = True - self.len = len(self.headers) + total_len(self.body) - - @classmethod - def from_field(cls, field, encoding): - """Create a part from a Request Field generated by urllib3.""" - headers = encode_with(field.render_headers(), encoding) - body = coerce_data(field.data, encoding) - return cls(headers, body) - - def bytes_left_to_write(self): - """Determine if there are bytes left to write. - - :returns: bool -- ``True`` if there are bytes left to write, otherwise - ``False`` - """ - to_read = 0 - if self.headers_unread: - to_read += len(self.headers) - - return (to_read + total_len(self.body)) > 0 - - def write_to(self, buffer, size): - """Write the requested amount of bytes to the buffer provided. - - The number of bytes written may exceed size on the first read since we - load the headers ambitiously. - - :param CustomBytesIO buffer: buffer we want to write bytes to - :param int size: number of bytes requested to be written to the buffer - :returns: int -- number of bytes actually written - """ - written = 0 - if self.headers_unread: - written += buffer.append(self.headers) - self.headers_unread = False - - while total_len(self.body) > 0 and (size == -1 or written < size): - amount_to_read = size - if size != -1: - amount_to_read = size - written - written += buffer.append(self.body.read(amount_to_read)) - - return written - - -class CustomBytesIO(io.BytesIO): - def __init__(self, buffer=None, encoding='utf-8'): - buffer = encode_with(buffer, encoding) - super(CustomBytesIO, self).__init__(buffer) - - def _get_end(self): - current_pos = self.tell() - self.seek(0, 2) - length = self.tell() - self.seek(current_pos, 0) - return length - - @property - def len(self): - length = self._get_end() - return length - self.tell() - - def append(self, bytes): - with reset(self): - written = self.write(bytes) - return written - - def smart_truncate(self): - to_be_read = total_len(self) - already_read = self._get_end() - to_be_read - - if already_read >= to_be_read: - old_bytes = self.read() - self.seek(0, 0) - self.truncate() - self.write(old_bytes) - self.seek(0, 0) # We want to be at the beginning - - -class FileWrapper(object): - def __init__(self, file_object): - self.fd = file_object - - @property - def len(self): - return total_len(self.fd) - self.fd.tell() - - def read(self, length=-1): - return self.fd.read(length) - - -class FileFromURLWrapper(object): - """File from URL wrapper. - - The :class:`FileFromURLWrapper` object gives you the ability to stream file - from provided URL in chunks by :class:`MultipartEncoder`. - Provide a stateless solution for streaming file from one server to another. - You can use the :class:`FileFromURLWrapper` without a session or with - a session as demonstated by the examples below: - - .. code-block:: python - # no session - - import requests - from requests_toolbelt import MultipartEncoder, FileFromURLWrapper - - url = 'https://httpbin.org/image/png' - streaming_encoder = MultipartEncoder( - fields={ - 'file': FileFromURLWrapper(url) - } - ) - r = requests.post( - 'https://httpbin.org/post', data=streaming_encoder, - headers={'Content-Type': streaming_encoder.content_type} - ) - - .. code-block:: python - # using a session - - import requests - from requests_toolbelt import MultipartEncoder, FileFromURLWrapper - - session = requests.Session() - url = 'https://httpbin.org/image/png' - streaming_encoder = MultipartEncoder( - fields={ - 'file': FileFromURLWrapper(url, session=session) - } - ) - r = session.post( - 'https://httpbin.org/post', data=streaming_encoder, - headers={'Content-Type': streaming_encoder.content_type} - ) - - """ - - def __init__(self, file_url, session=None): - self.session = session or requests.Session() - requested_file = self._request_for_file(file_url) - self.len = int(requested_file.headers['content-length']) - self.raw_data = requested_file.raw - - def _request_for_file(self, file_url): - """Make call for file under provided URL.""" - response = self.session.get(file_url, stream=True) - content_length = response.headers.get('content-length', None) - if content_length is None: - error_msg = ( - "Data from provided URL {url} is not supported. Lack of " - "content-length Header in requested file response.".format( - url=file_url) - ) - raise FileNotSupportedError(error_msg) - elif not content_length.isdigit(): - error_msg = ( - "Data from provided URL {url} is not supported. content-length" - " header value is not a digit.".format(url=file_url) - ) - raise FileNotSupportedError(error_msg) - return response - - def read(self, chunk_size): - """Read file in chunks.""" - chunk_size = chunk_size if chunk_size >= 0 else self.len - chunk = self.raw_data.read(chunk_size) or b'' - self.len -= len(chunk) if chunk else 0 # left to read - return chunk diff --git a/included_dependencies/requests_toolbelt/sessions.py b/included_dependencies/requests_toolbelt/sessions.py deleted file mode 100644 index 7beac968..00000000 --- a/included_dependencies/requests_toolbelt/sessions.py +++ /dev/null @@ -1,70 +0,0 @@ -import requests - -from ._compat import urljoin - - -class BaseUrlSession(requests.Session): - """A Session with a URL that all requests will use as a base. - - Let's start by looking at an example: - - .. code-block:: python - - >>> from requests_toolbelt import sessions - >>> s = sessions.BaseUrlSession( - ... base_url='https://example.com/resource/') - >>> r = s.get('sub-resource/' params={'foo': 'bar'}) - >>> print(r.request.url) - https://example.com/resource/sub-resource/?foo=bar - - Our call to the ``get`` method will make a request to the URL passed in - when we created the Session and the partial resource name we provide. - - We implement this by overriding the ``request`` method so most uses of a - Session are covered. (This, however, precludes the use of PreparedRequest - objects). - - .. note:: - - The base URL that you provide and the path you provide are **very** - important. - - Let's look at another *similar* example - - .. code-block:: python - - >>> from requests_toolbelt import sessions - >>> s = sessions.BaseUrlSession( - ... base_url='https://example.com/resource/') - >>> r = s.get('/sub-resource/' params={'foo': 'bar'}) - >>> print(r.request.url) - https://example.com/sub-resource/?foo=bar - - The key difference here is that we called ``get`` with ``/sub-resource/``, - i.e., there was a leading ``/``. This changes how we create the URL - because we rely on :mod:`urllib.parse.urljoin`. - - To override how we generate the URL, sub-class this method and override the - ``create_url`` method. - - Based on implementation from - https://github.com/kennethreitz/requests/issues/2554#issuecomment-109341010 - """ - - base_url = None - - def __init__(self, base_url=None): - if base_url: - self.base_url = base_url - super(BaseUrlSession, self).__init__() - - def request(self, method, url, *args, **kwargs): - """Send the request after generating the complete URL.""" - url = self.create_url(url) - return super(BaseUrlSession, self).request( - method, url, *args, **kwargs - ) - - def create_url(self, url): - """Create the URL based off this partial path.""" - return urljoin(self.base_url, url) diff --git a/included_dependencies/requests_toolbelt/streaming_iterator.py b/included_dependencies/requests_toolbelt/streaming_iterator.py deleted file mode 100644 index 64fd75f1..00000000 --- a/included_dependencies/requests_toolbelt/streaming_iterator.py +++ /dev/null @@ -1,116 +0,0 @@ -# -*- coding: utf-8 -*- -""" - -requests_toolbelt.streaming_iterator -==================================== - -This holds the implementation details for the :class:`StreamingIterator`. It -is designed for the case where you, the user, know the size of the upload but -need to provide the data as an iterator. This class will allow you to specify -the size and stream the data without using a chunked transfer-encoding. - -""" -from requests.utils import super_len - -from .multipart.encoder import CustomBytesIO, encode_with - - -class StreamingIterator(object): - - """ - This class provides a way of allowing iterators with a known size to be - streamed instead of chunked. - - In requests, if you pass in an iterator it assumes you want to use - chunked transfer-encoding to upload the data, which not all servers - support well. Additionally, you may want to set the content-length - yourself to avoid this but that will not work. The only way to preempt - requests using a chunked transfer-encoding and forcing it to stream the - uploads is to mimic a very specific interace. Instead of having to know - these details you can instead just use this class. You simply provide the - size and iterator and pass the instance of StreamingIterator to requests - via the data parameter like so: - - .. code-block:: python - - from requests_toolbelt import StreamingIterator - - import requests - - # Let iterator be some generator that you already have and size be - # the size of the data produced by the iterator - - r = requests.post(url, data=StreamingIterator(size, iterator)) - - You can also pass file-like objects to :py:class:`StreamingIterator` in - case requests can't determize the filesize itself. This is the case with - streaming file objects like ``stdin`` or any sockets. Wrapping e.g. files - that are on disk with ``StreamingIterator`` is unnecessary, because - requests can determine the filesize itself. - - Naturally, you should also set the `Content-Type` of your upload - appropriately because the toolbelt will not attempt to guess that for you. - """ - - def __init__(self, size, iterator, encoding='utf-8'): - #: The expected size of the upload - self.size = int(size) - - if self.size < 0: - raise ValueError( - 'The size of the upload must be a positive integer' - ) - - #: Attribute that requests will check to determine the length of the - #: body. See bug #80 for more details - self.len = self.size - - #: Encoding the input data is using - self.encoding = encoding - - #: The iterator used to generate the upload data - self.iterator = iterator - - if hasattr(iterator, 'read'): - self._file = iterator - else: - self._file = _IteratorAsBinaryFile(iterator, encoding) - - def read(self, size=-1): - return encode_with(self._file.read(size), self.encoding) - - -class _IteratorAsBinaryFile(object): - def __init__(self, iterator, encoding='utf-8'): - #: The iterator used to generate the upload data - self.iterator = iterator - - #: Encoding the iterator is using - self.encoding = encoding - - # The buffer we use to provide the correct number of bytes requested - # during a read - self._buffer = CustomBytesIO() - - def _get_bytes(self): - try: - return encode_with(next(self.iterator), self.encoding) - except StopIteration: - return b'' - - def _load_bytes(self, size): - self._buffer.smart_truncate() - amount_to_load = size - super_len(self._buffer) - bytes_to_append = True - - while amount_to_load > 0 and bytes_to_append: - bytes_to_append = self._get_bytes() - amount_to_load -= self._buffer.append(bytes_to_append) - - def read(self, size=-1): - size = int(size) - if size == -1: - return b''.join(self.iterator) - - self._load_bytes(size) - return self._buffer.read(size) diff --git a/included_dependencies/requests_toolbelt/threaded/__init__.py b/included_dependencies/requests_toolbelt/threaded/__init__.py deleted file mode 100644 index 984f1e80..00000000 --- a/included_dependencies/requests_toolbelt/threaded/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -This module provides the API for ``requests_toolbelt.threaded``. - -The module provides a clean and simple API for making requests via a thread -pool. The thread pool will use sessions for increased performance. - -A simple use-case is: - -.. code-block:: python - - from requests_toolbelt import threaded - - urls_to_get = [{ - 'url': 'https://api.github.com/users/sigmavirus24', - 'method': 'GET', - }, { - 'url': 'https://api.github.com/repos/requests/toolbelt', - 'method': 'GET', - }, { - 'url': 'https://google.com', - 'method': 'GET', - }] - responses, errors = threaded.map(urls_to_get) - -By default, the threaded submodule will detect the number of CPUs your -computer has and use that if no other number of processes is selected. To -change this, always use the keyword argument ``num_processes``. Using the -above example, we would expand it like so: - -.. code-block:: python - - responses, errors = threaded.map(urls_to_get, num_processes=10) - -You can also customize how a :class:`requests.Session` is initialized by -creating a callback function: - -.. code-block:: python - - from requests_toolbelt import user_agent - - def initialize_session(session): - session.headers['User-Agent'] = user_agent('my-scraper', '0.1') - session.headers['Accept'] = 'application/json' - - responses, errors = threaded.map(urls_to_get, - initializer=initialize_session) - -.. autofunction:: requests_toolbelt.threaded.map - -Inspiration is blatantly drawn from the standard library's multiprocessing -library. See the following references: - -- multiprocessing's `pool source`_ - -- map and map_async `inspiration`_ - -.. _pool source: - https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py -.. _inspiration: - https://hg.python.org/cpython/file/8ef4f75a8018/Lib/multiprocessing/pool.py#l340 -""" -from . import pool -from .._compat import queue - - -def map(requests, **kwargs): - r"""Simple interface to the threaded Pool object. - - This function takes a list of dictionaries representing requests to make - using Sessions in threads and returns a tuple where the first item is - a generator of successful responses and the second is a generator of - exceptions. - - :param list requests: - Collection of dictionaries representing requests to make with the Pool - object. - :param \*\*kwargs: - Keyword arguments that are passed to the - :class:`~requests_toolbelt.threaded.pool.Pool` object. - :returns: Tuple of responses and exceptions from the pool - :rtype: (:class:`~requests_toolbelt.threaded.pool.ThreadResponse`, - :class:`~requests_toolbelt.threaded.pool.ThreadException`) - """ - if not (requests and all(isinstance(r, dict) for r in requests)): - raise ValueError('map expects a list of dictionaries.') - - # Build our queue of requests - job_queue = queue.Queue() - for request in requests: - job_queue.put(request) - - # Ensure the user doesn't try to pass their own job_queue - kwargs['job_queue'] = job_queue - - threadpool = pool.Pool(**kwargs) - threadpool.join_all() - return threadpool.responses(), threadpool.exceptions() diff --git a/included_dependencies/requests_toolbelt/threaded/pool.py b/included_dependencies/requests_toolbelt/threaded/pool.py deleted file mode 100644 index 1fe81461..00000000 --- a/included_dependencies/requests_toolbelt/threaded/pool.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Module implementing the Pool for :mod:``requests_toolbelt.threaded``.""" -import multiprocessing -import requests - -from . import thread -from .._compat import queue - - -class Pool(object): - """Pool that manages the threads containing sessions. - - :param queue: - The queue you're expected to use to which you should add items. - :type queue: queue.Queue - :param initializer: - Function used to initialize an instance of ``session``. - :type initializer: collections.Callable - :param auth_generator: - Function used to generate new auth credentials for the session. - :type auth_generator: collections.Callable - :param int num_process: - Number of threads to create. - :param session: - :type session: requests.Session - """ - - def __init__(self, job_queue, initializer=None, auth_generator=None, - num_processes=None, session=requests.Session): - if num_processes is None: - num_processes = multiprocessing.cpu_count() or 1 - - if num_processes < 1: - raise ValueError("Number of processes should at least be 1.") - - self._job_queue = job_queue - self._response_queue = queue.Queue() - self._exc_queue = queue.Queue() - self._processes = num_processes - self._initializer = initializer or _identity - self._auth = auth_generator or _identity - self._session = session - self._pool = [ - thread.SessionThread(self._new_session(), self._job_queue, - self._response_queue, self._exc_queue) - for _ in range(self._processes) - ] - - def _new_session(self): - return self._auth(self._initializer(self._session())) - - @classmethod - def from_exceptions(cls, exceptions, **kwargs): - r"""Create a :class:`~Pool` from an :class:`~ThreadException`\ s. - - Provided an iterable that provides :class:`~ThreadException` objects, - this classmethod will generate a new pool to retry the requests that - caused the exceptions. - - :param exceptions: - Iterable that returns :class:`~ThreadException` - :type exceptions: iterable - :param kwargs: - Keyword arguments passed to the :class:`~Pool` initializer. - :returns: An initialized :class:`~Pool` object. - :rtype: :class:`~Pool` - """ - job_queue = queue.Queue() - for exc in exceptions: - job_queue.put(exc.request_kwargs) - - return cls(job_queue=job_queue, **kwargs) - - @classmethod - def from_urls(cls, urls, request_kwargs=None, **kwargs): - """Create a :class:`~Pool` from an iterable of URLs. - - :param urls: - Iterable that returns URLs with which we create a pool. - :type urls: iterable - :param dict request_kwargs: - Dictionary of other keyword arguments to provide to the request - method. - :param kwargs: - Keyword arguments passed to the :class:`~Pool` initializer. - :returns: An initialized :class:`~Pool` object. - :rtype: :class:`~Pool` - """ - request_dict = {'method': 'GET'} - request_dict.update(request_kwargs or {}) - job_queue = queue.Queue() - for url in urls: - job = request_dict.copy() - job.update({'url': url}) - job_queue.put(job) - - return cls(job_queue=job_queue, **kwargs) - - def exceptions(self): - """Iterate over all the exceptions in the pool. - - :returns: Generator of :class:`~ThreadException` - """ - while True: - exc = self.get_exception() - if exc is None: - break - yield exc - - def get_exception(self): - """Get an exception from the pool. - - :rtype: :class:`~ThreadException` - """ - try: - (request, exc) = self._exc_queue.get_nowait() - except queue.Empty: - return None - else: - return ThreadException(request, exc) - - def get_response(self): - """Get a response from the pool. - - :rtype: :class:`~ThreadResponse` - """ - try: - (request, response) = self._response_queue.get_nowait() - except queue.Empty: - return None - else: - return ThreadResponse(request, response) - - def responses(self): - """Iterate over all the responses in the pool. - - :returns: Generator of :class:`~ThreadResponse` - """ - while True: - resp = self.get_response() - if resp is None: - break - yield resp - - def join_all(self): - """Join all the threads to the master thread.""" - for session_thread in self._pool: - session_thread.join() - - -class ThreadProxy(object): - proxied_attr = None - - def __getattr__(self, attr): - """Proxy attribute accesses to the proxied object.""" - get = object.__getattribute__ - if attr not in self.attrs: - response = get(self, self.proxied_attr) - return getattr(response, attr) - else: - return get(self, attr) - - -class ThreadResponse(ThreadProxy): - """A wrapper around a requests Response object. - - This will proxy most attribute access actions to the Response object. For - example, if you wanted the parsed JSON from the response, you might do: - - .. code-block:: python - - thread_response = pool.get_response() - json = thread_response.json() - - """ - proxied_attr = 'response' - attrs = frozenset(['request_kwargs', 'response']) - - def __init__(self, request_kwargs, response): - #: The original keyword arguments provided to the queue - self.request_kwargs = request_kwargs - #: The wrapped response - self.response = response - - -class ThreadException(ThreadProxy): - """A wrapper around an exception raised during a request. - - This will proxy most attribute access actions to the exception object. For - example, if you wanted the message from the exception, you might do: - - .. code-block:: python - - thread_exc = pool.get_exception() - msg = thread_exc.message - - """ - proxied_attr = 'exception' - attrs = frozenset(['request_kwargs', 'exception']) - - def __init__(self, request_kwargs, exception): - #: The original keyword arguments provided to the queue - self.request_kwargs = request_kwargs - #: The captured and wrapped exception - self.exception = exception - - -def _identity(session_obj): - return session_obj - - -__all__ = ['ThreadException', 'ThreadResponse', 'Pool'] diff --git a/included_dependencies/requests_toolbelt/threaded/thread.py b/included_dependencies/requests_toolbelt/threaded/thread.py deleted file mode 100644 index 542813c1..00000000 --- a/included_dependencies/requests_toolbelt/threaded/thread.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Module containing the SessionThread class.""" -import threading -import uuid - -import requests.exceptions as exc - -from .._compat import queue - - -class SessionThread(object): - def __init__(self, initialized_session, job_queue, response_queue, - exception_queue): - self._session = initialized_session - self._jobs = job_queue - self._create_worker() - self._responses = response_queue - self._exceptions = exception_queue - - def _create_worker(self): - self._worker = threading.Thread( - target=self._make_request, - name=uuid.uuid4(), - ) - self._worker.daemon = True - self._worker._state = 0 - self._worker.start() - - def _handle_request(self, kwargs): - try: - response = self._session.request(**kwargs) - except exc.RequestException as e: - self._exceptions.put((kwargs, e)) - else: - self._responses.put((kwargs, response)) - finally: - self._jobs.task_done() - - def _make_request(self): - while True: - try: - kwargs = self._jobs.get_nowait() - except queue.Empty: - break - - self._handle_request(kwargs) - - def is_alive(self): - """Proxy to the thread's ``is_alive`` method.""" - return self._worker.is_alive() - - def join(self): - """Join this thread to the master thread.""" - self._worker.join() diff --git a/included_dependencies/requests_toolbelt/utils/deprecated.py b/included_dependencies/requests_toolbelt/utils/deprecated.py deleted file mode 100644 index c935783b..00000000 --- a/included_dependencies/requests_toolbelt/utils/deprecated.py +++ /dev/null @@ -1,91 +0,0 @@ -# -*- coding: utf-8 -*- -"""A collection of functions deprecated in requests.utils.""" -import re -import sys - -from requests import utils - -find_charset = re.compile( - br']', flags=re.I -).findall - -find_pragma = re.compile( - br']', flags=re.I -).findall - -find_xml = re.compile( - br'^<\?xml.*?encoding=["\']*(.+?)["\'>]' -).findall - - -def get_encodings_from_content(content): - """Return encodings from given content string. - - .. code-block:: python - - import requests - from requests_toolbelt.utils import deprecated - - r = requests.get(url) - encodings = deprecated.get_encodings_from_content(r) - - :param content: bytestring to extract encodings from - :type content: bytes - :return: encodings detected in the provided content - :rtype: list(str) - """ - encodings = (find_charset(content) + find_pragma(content) - + find_xml(content)) - if (3, 0) <= sys.version_info < (4, 0): - encodings = [encoding.decode('utf8') for encoding in encodings] - return encodings - - -def get_unicode_from_response(response): - """Return the requested content back in unicode. - - This will first attempt to retrieve the encoding from the response - headers. If that fails, it will use - :func:`requests_toolbelt.utils.deprecated.get_encodings_from_content` - to determine encodings from HTML elements. - - .. code-block:: python - - import requests - from requests_toolbelt.utils import deprecated - - r = requests.get(url) - text = deprecated.get_unicode_from_response(r) - - :param response: Response object to get unicode content from. - :type response: requests.models.Response - """ - tried_encodings = set() - - # Try charset from content-type - encoding = utils.get_encoding_from_headers(response.headers) - - if encoding: - try: - return str(response.content, encoding) - except UnicodeError: - tried_encodings.add(encoding.lower()) - - encodings = get_encodings_from_content(response.content) - - for _encoding in encodings: - _encoding = _encoding.lower() - if _encoding in tried_encodings: - continue - try: - return str(response.content, _encoding) - except UnicodeError: - tried_encodings.add(_encoding) - - # Fall back: - if encoding: - try: - return str(response.content, encoding, errors='replace') - except TypeError: - pass - return response.text diff --git a/included_dependencies/requests_toolbelt/utils/formdata.py b/included_dependencies/requests_toolbelt/utils/formdata.py deleted file mode 100644 index b0a909d2..00000000 --- a/included_dependencies/requests_toolbelt/utils/formdata.py +++ /dev/null @@ -1,108 +0,0 @@ -# -*- coding: utf-8 -*- -"""Implementation of nested form-data encoding function(s).""" -from .._compat import basestring -from .._compat import urlencode as _urlencode - - -__all__ = ('urlencode',) - - -def urlencode(query, *args, **kwargs): - """Handle nested form-data queries and serialize them appropriately. - - There are times when a website expects a nested form data query to be sent - but, the standard library's urlencode function does not appropriately - handle the nested structures. In that case, you need this function which - will flatten the structure first and then properly encode it for you. - - When using this to send data in the body of a request, make sure you - specify the appropriate Content-Type header for the request. - - .. code-block:: python - - import requests - from requests_toolbelt.utils import formdata - - query = { - 'my_dict': { - 'foo': 'bar', - 'biz': 'baz", - }, - 'a': 'b', - } - - resp = requests.get(url, params=formdata.urlencode(query)) - # or - resp = requests.post( - url, - data=formdata.urlencode(query), - headers={ - 'Content-Type': 'application/x-www-form-urlencoded' - }, - ) - - Similarly, you can specify a list of nested tuples, e.g., - - .. code-block:: python - - import requests - from requests_toolbelt.utils import formdata - - query = [ - ('my_list', [ - ('foo', 'bar'), - ('biz', 'baz'), - ]), - ('a', 'b'), - ] - - resp = requests.get(url, params=formdata.urlencode(query)) - # or - resp = requests.post( - url, - data=formdata.urlencode(query), - headers={ - 'Content-Type': 'application/x-www-form-urlencoded' - }, - ) - - For additional parameter and return information, see the official - `urlencode`_ documentation. - - .. _urlencode: - https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode - """ - expand_classes = (dict, list, tuple) - original_query_list = _to_kv_list(query) - - if not all(_is_two_tuple(i) for i in original_query_list): - raise ValueError("Expected query to be able to be converted to a " - "list comprised of length 2 tuples.") - - query_list = original_query_list - while any(isinstance(v, expand_classes) for _, v in query_list): - query_list = _expand_query_values(query_list) - - return _urlencode(query_list, *args, **kwargs) - - -def _to_kv_list(dict_or_list): - if hasattr(dict_or_list, 'items'): - return list(dict_or_list.items()) - return dict_or_list - - -def _is_two_tuple(item): - return isinstance(item, (list, tuple)) and len(item) == 2 - - -def _expand_query_values(original_query_list): - query_list = [] - for key, value in original_query_list: - if isinstance(value, basestring): - query_list.append((key, value)) - else: - key_fmt = key + '[%s]' - value_list = _to_kv_list(value) - query_list.extend((key_fmt % k, v) for k, v in value_list) - return query_list diff --git a/included_dependencies/requests_toolbelt/utils/user_agent.py b/included_dependencies/requests_toolbelt/utils/user_agent.py deleted file mode 100644 index e9636a41..00000000 --- a/included_dependencies/requests_toolbelt/utils/user_agent.py +++ /dev/null @@ -1,143 +0,0 @@ -# -*- coding: utf-8 -*- -import collections -import platform -import sys - - -def user_agent(name, version, extras=None): - """Return an internet-friendly user_agent string. - - The majority of this code has been wilfully stolen from the equivalent - function in Requests. - - :param name: The intended name of the user-agent, e.g. "python-requests". - :param version: The version of the user-agent, e.g. "0.0.1". - :param extras: List of two-item tuples that are added to the user-agent - string. - :returns: Formatted user-agent string - :rtype: str - """ - if extras is None: - extras = [] - - return UserAgentBuilder( - name, version - ).include_extras( - extras - ).include_implementation( - ).include_system().build() - - -class UserAgentBuilder(object): - """Class to provide a greater level of control than :func:`user_agent`. - - This is used by :func:`user_agent` to build its User-Agent string. - - .. code-block:: python - - user_agent_str = UserAgentBuilder( - name='requests-toolbelt', - version='17.4.0', - ).include_implementation( - ).include_system( - ).include_extras([ - ('requests', '2.14.2'), - ('urllib3', '1.21.2'), - ]).build() - - """ - - format_string = '%s/%s' - - def __init__(self, name, version): - """Initialize our builder with the name and version of our user agent. - - :param str name: - Name of our user-agent. - :param str version: - The version string for user-agent. - """ - self._pieces = collections.deque([(name, version)]) - - def build(self): - """Finalize the User-Agent string. - - :returns: - Formatted User-Agent string. - :rtype: - str - """ - return " ".join([self.format_string % piece for piece in self._pieces]) - - def include_extras(self, extras): - """Include extra portions of the User-Agent. - - :param list extras: - list of tuples of extra-name and extra-version - """ - if any(len(extra) != 2 for extra in extras): - raise ValueError('Extras should be a sequence of two item tuples.') - - self._pieces.extend(extras) - return self - - def include_implementation(self): - """Append the implementation string to the user-agent string. - - This adds the the information that you're using CPython 2.7.13 to the - User-Agent. - """ - self._pieces.append(_implementation_tuple()) - return self - - def include_system(self): - """Append the information about the Operating System.""" - self._pieces.append(_platform_tuple()) - return self - - -def _implementation_tuple(): - """Return the tuple of interpreter name and version. - - Returns a string that provides both the name and the version of the Python - implementation currently running. For example, on CPython 2.7.5 it will - return "CPython/2.7.5". - - This function works best on CPython and PyPy: in particular, it probably - doesn't work for Jython or IronPython. Future investigation should be done - to work out the correct shape of the code for those platforms. - """ - implementation = platform.python_implementation() - - if implementation == 'CPython': - implementation_version = platform.python_version() - elif implementation == 'PyPy': - implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro) - if sys.pypy_version_info.releaselevel != 'final': - implementation_version = ''.join([ - implementation_version, sys.pypy_version_info.releaselevel - ]) - elif implementation == 'Jython': - implementation_version = platform.python_version() # Complete Guess - elif implementation == 'IronPython': - implementation_version = platform.python_version() # Complete Guess - else: - implementation_version = 'Unknown' - - return (implementation, implementation_version) - - -def _implementation_string(): - return "%s/%s" % _implementation_tuple() - - -def _platform_tuple(): - try: - p_system = platform.system() - p_release = platform.release() - except IOError: - p_system = 'Unknown' - p_release = 'Unknown' - return (p_system, p_release)