diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24652ef --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +SOURCES/urllib3-1.10.2.tar.gz diff --git a/.python-urllib3.metadata b/.python-urllib3.metadata new file mode 100644 index 0000000..63ac330 --- /dev/null +++ b/.python-urllib3.metadata @@ -0,0 +1 @@ +9b123cf0032f7a7ba43b9837c7a57a399f9d42b6 SOURCES/urllib3-1.10.2.tar.gz diff --git a/SOURCES/Add-support-for-IP-address-SAN-fields.patch b/SOURCES/Add-support-for-IP-address-SAN-fields.patch new file mode 100644 index 0000000..387c55d --- /dev/null +++ b/SOURCES/Add-support-for-IP-address-SAN-fields.patch @@ -0,0 +1,161 @@ +diff --git a/CHANGES.rst b/CHANGES.rst +index b2b8ae6..0150d85 100644 +--- a/CHANGES.rst ++++ b/CHANGES.rst +@@ -1,6 +1,9 @@ + Changes + ======= + ++* Accept ``iPAddress`` subject alternative name fields in TLS certificates. ++ (Issue #258) ++ + 1.10.2 (2015-02-25) + +++++++++++++++++++ + +diff --git a/dummyserver/certs/server.ip_san.crt b/dummyserver/certs/server.ip_san.crt +new file mode 100644 +index 000000000..58689d64d +--- /dev/null ++++ b/dummyserver/certs/server.ip_san.crt +@@ -0,0 +1,21 @@ ++-----BEGIN CERTIFICATE----- ++MIIDeTCCAuKgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBgTELMAkGA1UEBhMCRkkx ++DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQHDAVkdW1teTEOMAwGA1UECgwFZHVtbXkx ++DjAMBgNVBAsMBWR1bW15MREwDwYDVQQDDAhTbmFrZU9pbDEfMB0GCSqGSIb3DQEJ ++ARYQZHVtbXlAdGVzdC5sb2NhbDAeFw0xMTEyMjIwNzU4NDBaFw0yMTEyMTgwNzU4 ++NDBaMGExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UEBwwFZHVt ++bXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQLDAVkdW1teTESMBAGA1UEAwwJbG9j ++YWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDXe3FqmCWvP8XPxqtT +++0bfL1Tvzvebi46k0WIcUV8bP3vyYiSRXG9ALmyzZH4GHY9UVs4OEDkCMDOBSezB ++0y9ai/9doTNcaictdEBu8nfdXKoTtzrn+VX4UPrkH5hm7NQ1fTQuj1MR7yBCmYqN ++3Q2Q+Efuujyx0FwBzAuy1aKYuwIDAQABo4IBHjCCARowCQYDVR0TBAIwADAdBgNV ++HQ4EFgQUG+dK5Uos08QUwAWofDb3a8YcYlIwgbYGA1UdIwSBrjCBq4AUGXd/I2Ji ++QllF+3Wdx3NyBLszCi2hgYekgYQwgYExCzAJBgNVBAYTAkZJMQ4wDAYDVQQIDAVk ++dW1teTEOMAwGA1UEBwwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MQ4wDAYDVQQLDAVk ++dW1teTERMA8GA1UEAwwIU25ha2VPaWwxHzAdBgkqhkiG9w0BCQEWEGR1bW15QHRl ++c3QubG9jYWyCCQCz67HKL+G/4zAJBgNVHRIEAjAAMCoGA1UdEQQjMCGBDnJvb3RA ++bG9jYWxob3N0gglsb2NhbGhvc3SHBH8AAAEwDQYJKoZIhvcNAQEFBQADgYEAFEAy ++O9rxM14W0pVJWHTZkWBcDTqp8A8OB3JFVxeuCNcbtyfyYLWs2juv4YMmo1EKBOQe ++7LYfGuIvtIzT7KBa2QAPmX9JR+F6yl0IVSrYYt9hS7w9Cqr8+jK9QRpNwm3k25hp ++BmmoT5b9Q+AYcLMtdMu3uFjLmQBI2XobI/9vCT4= ++-----END CERTIFICATE----- +diff --git a/dummyserver/server.py b/dummyserver/server.py +index 18d81e1..3190835 100755 +--- a/dummyserver/server.py ++++ b/dummyserver/server.py +@@ -32,6 +32,10 @@ NO_SAN_CERTS = { + 'certfile': os.path.join(CERTS_PATH, 'server.no_san.crt'), + 'keyfile': DEFAULT_CERTS['keyfile'] + } ++IP_SAN_CERTS = { ++ 'certfile': os.path.join(CERTS_PATH, 'server.ip_san.crt'), ++ 'keyfile': DEFAULT_CERTS['keyfile'] ++} + DEFAULT_CA = os.path.join(CERTS_PATH, 'cacert.pem') + DEFAULT_CA_BAD = os.path.join(CERTS_PATH, 'client_bad.pem') + NO_SAN_CA = os.path.join(CERTS_PATH, 'cacert.no_san.pem') +diff --git a/urllib3/packages/ssl_match_hostname/__init__.py b/urllib3/packages/ssl_match_hostname/__init__.py +index dd59a75fd..d6594eb26 100644 +--- a/urllib3/packages/ssl_match_hostname/__init__.py ++++ b/urllib3/packages/ssl_match_hostname/__init__.py +@@ -1,5 +1,11 @@ ++import sys ++ + try: +- # Python 3.2+ ++ # Our match_hostname function is the same as 3.5's, so we only want to ++ # import the match_hostname function if it's at least that good. ++ if sys.version_info < (3, 5): ++ raise ImportError("Fallback to vendored code") ++ + from ssl import CertificateError, match_hostname + except ImportError: + try: +diff --git a/urllib3/packages/ssl_match_hostname/_implementation.py b/urllib3/packages/ssl_match_hostname/_implementation.py +index 52f428733..1fd42f38a 100644 +--- a/urllib3/packages/ssl_match_hostname/_implementation.py ++++ b/urllib3/packages/ssl_match_hostname/_implementation.py +@@ -4,8 +4,20 @@ + # stdlib. http://docs.python.org/3/license.html + + import re ++import sys ++ ++# ipaddress has been backported to 2.6+ in pypi. If it is installed on the ++# system, use it to handle IPAddress ServerAltnames (this was added in ++# python-3.5) otherwise only do DNS matching. This allows ++# backports.ssl_match_hostname to continue to be used all the way back to ++# python-2.4. ++try: ++ import ipaddress ++except ImportError: ++ ipaddress = None ++ ++__version__ = '3.5.0.1' + +-__version__ = '3.4.0.2' + + class CertificateError(ValueError): + pass +@@ -64,6 +76,23 @@ def _dnsname_match(dn, hostname, max_wildcards=1): + return pat.match(hostname) + + ++def _to_unicode(obj): ++ if isinstance(obj, str) and sys.version_info < (3,): ++ obj = unicode(obj, encoding='ascii', errors='strict') ++ return obj ++ ++def _ipaddress_match(ipname, host_ip): ++ """Exact matching of IP addresses. ++ ++ RFC 6125 explicitly doesn't define an algorithm for this ++ (section 1.7.2 - "Out of Scope"). ++ """ ++ # OpenSSL may add a trailing newline to a subjectAltName's IP address ++ # Divergence from upstream: ipaddress can't handle byte str ++ ip = ipaddress.ip_address(_to_unicode(ipname).rstrip()) ++ return ip == host_ip ++ ++ + def match_hostname(cert, hostname): + """Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 +@@ -73,12 +102,35 @@ def match_hostname(cert, hostname): + returns nothing. + """ + if not cert: +- raise ValueError("empty or no certificate") ++ raise ValueError("empty or no certificate, match_hostname needs a " ++ "SSL socket or SSL context with either " ++ "CERT_OPTIONAL or CERT_REQUIRED") ++ try: ++ # Divergence from upstream: ipaddress can't handle byte str ++ host_ip = ipaddress.ip_address(_to_unicode(hostname)) ++ except ValueError: ++ # Not an IP address (common case) ++ host_ip = None ++ except UnicodeError: ++ # Divergence from upstream: Have to deal with ipaddress not taking ++ # byte strings. addresses should be all ascii, so we consider it not ++ # an ipaddress in this case ++ host_ip = None ++ except AttributeError: ++ # Divergence from upstream: Make ipaddress library optional ++ if ipaddress is None: ++ host_ip = None ++ else: ++ raise + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': +- if _dnsname_match(value, hostname): ++ if host_ip is None and _dnsname_match(value, hostname): ++ return ++ dnsnames.append(value) ++ elif key == 'IP Address': ++ if host_ip is not None and _ipaddress_match(value, host_ip): + return + dnsnames.append(value) + if not dnsnames: diff --git a/SOURCES/CVE-2018-20060.patch b/SOURCES/CVE-2018-20060.patch new file mode 100644 index 0000000..f9e6c60 --- /dev/null +++ b/SOURCES/CVE-2018-20060.patch @@ -0,0 +1,108 @@ +From 73474e1b518ce91481b7101d68d8a75c7ad67029 Mon Sep 17 00:00:00 2001 +From: Lumir Balhar +Date: Mon, 4 Mar 2019 12:45:09 +0100 +Subject: [PATCH] CVE-2018-20060 + +Cross-host redirect does not remove Authorization header allow +for credential exposure. +--- + CHANGES.rst | 4 ++++ + urllib3/poolmanager.py | 11 ++++++++++- + urllib3/util/retry.py | 12 +++++++++++- + 3 files changed, 25 insertions(+), 2 deletions(-) + +diff --git a/CHANGES.rst b/CHANGES.rst +index 0150d85..34ad6b0 100644 +--- a/CHANGES.rst ++++ b/CHANGES.rst +@@ -1,6 +1,10 @@ + Changes + ======= + ++* Allow providing a list of headers to strip from requests when redirecting ++ to a different host. Defaults to the ``Authorization`` header. Different ++ headers can be set via ``Retry.remove_headers_on_redirect``. (Issue #1316) ++ + * Accept ``iPAddress`` subject alternative name fields in TLS certificates. + (Issue #258) + +diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py +index 4fdae8d..b43b235 100644 +--- a/urllib3/poolmanager.py ++++ b/urllib3/poolmanager.py +@@ -238,8 +238,9 @@ class PoolManager(RequestMethods): + + kw['assert_same_host'] = False + kw['redirect'] = False ++ + if 'headers' not in kw: +- kw['headers'] = self.headers ++ kw['headers'] = self.headers.copy() + + if self.proxy is not None and u.scheme == "http": + response = conn.urlopen(method, url, **kw) +@@ -261,6 +262,14 @@ class PoolManager(RequestMethods): + if not isinstance(retries, Retry): + retries = Retry.from_int(retries, redirect=redirect) + ++ # Strip headers marked as unsafe to forward to the redirected location. ++ # Check remove_headers_on_redirect to avoid a potential network call within ++ # conn.is_same_host() which may use socket.gethostbyname() in the future. ++ if (retries.remove_headers_on_redirect ++ and not conn.is_same_host(redirect_location)): ++ for header in retries.remove_headers_on_redirect: ++ kw['headers'].pop(header, None) ++ + try: + retries = retries.increment(method, url, response=response, _pool=conn) + except MaxRetryError: +diff --git a/urllib3/util/retry.py b/urllib3/util/retry.py +index 7e0959d..0cf8eed 100644 +--- a/urllib3/util/retry.py ++++ b/urllib3/util/retry.py +@@ -101,17 +101,25 @@ class Retry(object): + :param bool raise_on_redirect: Whether, if the number of redirects is + exhausted, to raise a MaxRetryError, or to return a response with a + response code in the 3xx range. ++ ++ :param iterable remove_headers_on_redirect: ++ Sequence of headers to remove from the request when a response ++ indicating a redirect is returned before firing off the redirected ++ request. + """ + + DEFAULT_METHOD_WHITELIST = frozenset([ + 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) + ++ DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) ++ + #: Maximum backoff time. + BACKOFF_MAX = 120 + + def __init__(self, total=10, connect=None, read=None, redirect=None, + method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, +- backoff_factor=0, raise_on_redirect=True, _observed_errors=0): ++ backoff_factor=0, raise_on_redirect=True, _observed_errors=0, ++ remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): + + self.total = total + self.connect = connect +@@ -127,6 +135,7 @@ class Retry(object): + self.backoff_factor = backoff_factor + self.raise_on_redirect = raise_on_redirect + self._observed_errors = _observed_errors # TODO: use .history instead? ++ self.remove_headers_on_redirect = remove_headers_on_redirect + + def new(self, **kw): + params = dict( +@@ -137,6 +146,7 @@ class Retry(object): + backoff_factor=self.backoff_factor, + raise_on_redirect=self.raise_on_redirect, + _observed_errors=self._observed_errors, ++ remove_headers_on_redirect=self.remove_headers_on_redirect, + ) + params.update(kw) + return type(self)(**params) +-- +2.20.1 + diff --git a/SOURCES/CVE-2019-9740.patch b/SOURCES/CVE-2019-9740.patch new file mode 100644 index 0000000..8583077 --- /dev/null +++ b/SOURCES/CVE-2019-9740.patch @@ -0,0 +1,107 @@ +From 374bf2ea399cd1f2abbc0890fe796d570416adea Mon Sep 17 00:00:00 2001 +From: Ryan Petrello +Date: Tue, 30 Apr 2019 12:36:48 -0400 +Subject: [PATCH 1/2] prevent CVE-2019-9740 in 1.24.x + +adapted from https://github.com/python/cpython/pull/12755 +--- + test/test_util.py | 5 +++++ + urllib3/util/url.py | 8 ++++++++ + 2 files changed, 13 insertions(+) + +diff --git a/test/test_util.py b/test/test_util.py +index c850d91..5fa49bb 100644 +--- a/test/test_util.py ++++ b/test/test_util.py +@@ -142,6 +142,11 @@ class TestUtil(unittest.TestCase): + def test_parse_url_invalid_IPv6(self): + self.assertRaises(ValueError, parse_url, '[::1') + ++ def test_parse_url_contains_control_characters(self): ++ # see CVE-2019-9740 ++ with self.assertRaises(LocationParseError): ++ parse_url('http://localhost:8000/ HTTP/1.1\r\nHEADER: INJECTED\r\nIgnore:') ++ + def test_Url_str(self): + U = Url('http', host='google.com') + self.assertEqual(str(U), U.url) +diff --git a/urllib3/util/url.py b/urllib3/util/url.py +index b2ec834..7878892 100644 +--- a/urllib3/util/url.py ++++ b/urllib3/util/url.py +@@ -1,10 +1,13 @@ + from collections import namedtuple ++import re + + from ..exceptions import LocationParseError + + + url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] + ++_contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]') ++ + + class Url(namedtuple('Url', url_attrs)): + """ +@@ -142,6 +145,11 @@ def parse_url(url): + # Empty + return Url() + ++ # Prevent CVE-2019-9740. ++ # adapted from https://github.com/python/cpython/pull/12755 ++ if _contains_disallowed_url_pchar_re.search(url): ++ raise LocationParseError("URL can't contain control characters. {!r}".format(url)) ++ + scheme = None + auth = None + host = None + +From 5661da387242337e09a939120a1e154e7335d18b Mon Sep 17 00:00:00 2001 +From: Ryan Petrello +Date: Wed, 1 May 2019 16:46:44 -0400 +Subject: [PATCH 2/2] avoid CVE-2019-9740 by percent-encoding invalid path + characters + +this is to avoid breaking changes in downstream libraries like requests +--- + test/test_util.py | 4 ++-- + urllib3/util/url.py | 4 ++-- + 2 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/test/test_util.py b/test/test_util.py +index 5fa49bb..122e00b 100644 +--- a/test/test_util.py ++++ b/test/test_util.py +@@ -144,8 +144,8 @@ class TestUtil(unittest.TestCase): + + def test_parse_url_contains_control_characters(self): + # see CVE-2019-9740 +- with self.assertRaises(LocationParseError): +- parse_url('http://localhost:8000/ HTTP/1.1\r\nHEADER: INJECTED\r\nIgnore:') ++ url = parse_url('http://localhost:8000/ HTTP/1.1\r\nHEADER: INJECTED\r\nIgnore:') ++ self.assertEqual(url.path, '/%20HTTP/1.1%0D%0AHEADER:%20INJECTED%0D%0AIgnore:') + + def test_Url_str(self): + U = Url('http', host='google.com') +diff --git a/urllib3/util/url.py b/urllib3/util/url.py +index 7878892..cc15b6b 100644 +--- a/urllib3/util/url.py ++++ b/urllib3/util/url.py +@@ -2,6 +2,7 @@ from collections import namedtuple + import re + + from ..exceptions import LocationParseError ++from six.moves.urllib.parse import quote + + + url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] +@@ -147,8 +148,7 @@ def parse_url(url): + + # Prevent CVE-2019-9740. + # adapted from https://github.com/python/cpython/pull/12755 +- if _contains_disallowed_url_pchar_re.search(url): +- raise LocationParseError("URL can't contain control characters. {!r}".format(url)) ++ url = _contains_disallowed_url_pchar_re.sub(lambda match: quote(match.group()), url) + + scheme = None + auth = None diff --git a/SOURCES/key-connection-pools-off-custom-keys.patch b/SOURCES/key-connection-pools-off-custom-keys.patch new file mode 100644 index 0000000..c00dfc6 --- /dev/null +++ b/SOURCES/key-connection-pools-off-custom-keys.patch @@ -0,0 +1,488 @@ +From 99274b9e7524be7e8f041e327be1dd7ba3fdc8a4 Mon Sep 17 00:00:00 2001 +From: Jeremy Cline +Date: Thu, 21 Apr 2016 10:56:28 -0700 +Subject: [PATCH] Key connection pools off custom keys + +--- + CONTRIBUTORS.txt | 3 + + docs/managers.rst | 44 +++++++++ + test/test_poolmanager.py | 230 ++++++++++++++++++++++++++++++++++++++++++++++- + urllib3/poolmanager.py | 104 +++++++++++++++++++-- + 4 files changed, 371 insertions(+), 10 deletions(-) + +diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt +index 5141a50..f32faaa 100644 +--- a/CONTRIBUTORS.txt ++++ b/CONTRIBUTORS.txt +@@ -142,5 +142,8 @@ In chronological order: + * Alex Gaynor + * Updates to the default SSL configuration + ++* Jeremy Cline ++ * Added connection pool keys by scheme ++ + * [Your name or handle] <[email or website]> + * [Brief summary of your changes] +diff --git a/docs/managers.rst b/docs/managers.rst +index 6c841b7..ef91ca4 100644 +--- a/docs/managers.rst ++++ b/docs/managers.rst +@@ -28,6 +28,44 @@ so you don't have to. + >>> conn.num_requests + 3 + ++A :class:`.PoolManager` will create a new :doc:`ConnectionPool ` ++when no :doc:`ConnectionPools ` exist with a matching pool key. ++The pool key is derived using the requested URL and the current values ++of the ``connection_pool_kw`` instance variable on :class:`.PoolManager`. ++ ++The keys in ``connection_pool_kw`` used when deriving the key are ++configurable. For example, by default the ``my_field`` key is not ++considered. ++ ++.. doctest :: ++ ++ >>> from urllib3.poolmanager import PoolManager ++ >>> manager = PoolManager(10, my_field='wheat') ++ >>> manager.connection_from_url('http://example.com') ++ >>> manager.connection_pool_kw['my_field'] = 'barley' ++ >>> manager.connection_from_url('http://example.com') ++ >>> len(manager.pools) ++ 1 ++ ++To make the pool manager create new pools when the value of ++``my_field`` changes, you can define a custom pool key and alter ++the ``key_fn_by_scheme`` instance variable on :class:`.PoolManager`. ++ ++.. doctest :: ++ ++ >>> import functools ++ >>> from collections import namedtuple ++ >>> from urllib3.poolmanager import PoolManager, HTTPPoolKey ++ >>> from urllib3.poolmanager import default_key_normalizer as normalizer ++ >>> CustomKey = namedtuple('CustomKey', HTTPPoolKey._fields + ('my_field',)) ++ >>> manager = PoolManager(10, my_field='wheat') ++ >>> manager.key_fn_by_scheme['http'] = functools.partial(normalizer, CustomKey) ++ >>> manager.connection_from_url('http://example.com') ++ >>> manager.connection_pool_kw['my_field'] = 'barley' ++ >>> manager.connection_from_url('http://example.com') ++ >>> len(manager.pools) ++ 2 ++ + The API of a :class:`.PoolManager` object is similar to that of a + :doc:`ConnectionPool `, so they can be passed around interchangeably. + +@@ -59,6 +97,12 @@ API + + .. autoclass:: PoolManager + :inherited-members: ++ .. autoclass:: BasePoolKey ++ :inherited-members: ++ .. autoclass:: HTTPPoolKey ++ :inherited-members: ++ .. autoclass:: HTTPSPoolKey ++ :inherited-members: + + ProxyManager + ============ +diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py +index 6195d51..fb134fb 100644 +--- a/test/test_poolmanager.py ++++ b/test/test_poolmanager.py +@@ -1,11 +1,21 @@ ++import functools + import unittest ++from collections import namedtuple + +-from urllib3.poolmanager import PoolManager ++from urllib3.poolmanager import ( ++ _default_key_normalizer, ++ HTTPPoolKey, ++ HTTPSPoolKey, ++ key_fn_by_scheme, ++ PoolManager, ++ SSL_KEYWORDS, ++) + from urllib3 import connection_from_url + from urllib3.exceptions import ( + ClosedPoolError, + LocationValueError, + ) ++from urllib3.util import retry, timeout + + + class TestPoolManager(unittest.TestCase): +@@ -87,6 +97,224 @@ class TestPoolManager(unittest.TestCase): + + self.assertEqual(len(p.pools), 0) + ++ def test_http_pool_key_fields(self): ++ """Assert the HTTPPoolKey fields are honored when selecting a pool.""" ++ connection_pool_kw = { ++ 'timeout': timeout.Timeout(3.14), ++ 'retries': retry.Retry(total=6, connect=2), ++ 'block': True, ++ 'strict': True, ++ 'source_address': '127.0.0.1', ++ } ++ p = PoolManager() ++ conn_pools = [ ++ p.connection_from_url('http://example.com/'), ++ p.connection_from_url('http://example.com:8000/'), ++ p.connection_from_url('http://other.example.com/'), ++ ] ++ ++ for key, value in connection_pool_kw.items(): ++ p.connection_pool_kw[key] = value ++ conn_pools.append(p.connection_from_url('http://example.com/')) ++ ++ self.assertTrue( ++ all( ++ x is not y ++ for i, x in enumerate(conn_pools) ++ for j, y in enumerate(conn_pools) ++ if i != j ++ ) ++ ) ++ self.assertTrue( ++ all( ++ isinstance(key, HTTPPoolKey) ++ for key in p.pools.keys()) ++ ) ++ ++ def test_http_pool_key_extra_kwargs(self): ++ """Assert non-HTTPPoolKey fields are ignored when selecting a pool.""" ++ p = PoolManager() ++ conn_pool = p.connection_from_url('http://example.com/') ++ p.connection_pool_kw['some_kwarg'] = 'that should be ignored' ++ other_conn_pool = p.connection_from_url('http://example.com/') ++ ++ self.assertTrue(conn_pool is other_conn_pool) ++ ++ def test_http_pool_key_https_kwargs(self): ++ """Assert HTTPSPoolKey fields are ignored when selecting a HTTP pool.""" ++ p = PoolManager() ++ conn_pool = p.connection_from_url('http://example.com/') ++ for key in SSL_KEYWORDS: ++ p.connection_pool_kw[key] = 'this should be ignored' ++ other_conn_pool = p.connection_from_url('http://example.com/') ++ ++ self.assertTrue(conn_pool is other_conn_pool) ++ ++ def test_https_pool_key_fields(self): ++ """Assert the HTTPSPoolKey fields are honored when selecting a pool.""" ++ connection_pool_kw = { ++ 'timeout': timeout.Timeout(3.14), ++ 'retries': retry.Retry(total=6, connect=2), ++ 'block': True, ++ 'strict': True, ++ 'source_address': '127.0.0.1', ++ 'key_file': '/root/totally_legit.key', ++ 'cert_file': '/root/totally_legit.crt', ++ 'cert_reqs': 'CERT_REQUIRED', ++ 'ca_certs': '/root/path_to_pem', ++ 'ssl_version': 'SSLv23_METHOD', ++ } ++ p = PoolManager() ++ conn_pools = [ ++ p.connection_from_url('https://example.com/'), ++ p.connection_from_url('https://example.com:4333/'), ++ p.connection_from_url('https://other.example.com/'), ++ ] ++ # Asking for a connection pool with the same key should give us an ++ # existing pool. ++ dup_pools = [] ++ ++ for key, value in connection_pool_kw.items(): ++ p.connection_pool_kw[key] = value ++ conn_pools.append(p.connection_from_url('https://example.com/')) ++ dup_pools.append(p.connection_from_url('https://example.com/')) ++ ++ self.assertTrue( ++ all( ++ x is not y ++ for i, x in enumerate(conn_pools) ++ for j, y in enumerate(conn_pools) ++ if i != j ++ ) ++ ) ++ self.assertTrue(all(pool in conn_pools for pool in dup_pools)) ++ self.assertTrue( ++ all( ++ isinstance(key, HTTPSPoolKey) ++ for key in p.pools.keys()) ++ ) ++ ++ def test_https_pool_key_extra_kwargs(self): ++ """Assert non-HTTPSPoolKey fields are ignored when selecting a pool.""" ++ p = PoolManager() ++ conn_pool = p.connection_from_url('https://example.com/') ++ p.connection_pool_kw['some_kwarg'] = 'that should be ignored' ++ other_conn_pool = p.connection_from_url('https://example.com/') ++ ++ self.assertTrue(conn_pool is other_conn_pool) ++ ++ def test_default_pool_key_funcs_copy(self): ++ """Assert each PoolManager gets a copy of ``pool_keys_by_scheme``.""" ++ p = PoolManager() ++ self.assertEqual(p.key_fn_by_scheme, p.key_fn_by_scheme) ++ self.assertFalse(p.key_fn_by_scheme is key_fn_by_scheme) ++ ++ def test_pools_keyed_with_from_host(self): ++ """Assert pools are still keyed correctly with connection_from_host.""" ++ ssl_kw = { ++ 'key_file': '/root/totally_legit.key', ++ 'cert_file': '/root/totally_legit.crt', ++ 'cert_reqs': 'CERT_REQUIRED', ++ 'ca_certs': '/root/path_to_pem', ++ 'ssl_version': 'SSLv23_METHOD', ++ } ++ p = PoolManager(5, **ssl_kw) ++ conns = [] ++ conns.append( ++ p.connection_from_host('example.com', 443, scheme='https') ++ ) ++ ++ for k in ssl_kw: ++ p.connection_pool_kw[k] = 'newval' ++ conns.append( ++ p.connection_from_host('example.com', 443, scheme='https') ++ ) ++ ++ self.assertTrue( ++ all( ++ x is not y ++ for i, x in enumerate(conns) ++ for j, y in enumerate(conns) ++ if i != j ++ ) ++ ) ++ ++ def test_https_connection_from_url_case_insensitive(self): ++ """Assert scheme case is ignored when pooling HTTPS connections.""" ++ p = PoolManager() ++ pool = p.connection_from_url('https://example.com/') ++ other_pool = p.connection_from_url('HTTPS://EXAMPLE.COM/') ++ ++ self.assertEqual(1, len(p.pools)) ++ self.assertTrue(pool is other_pool) ++ self.assertTrue(all(isinstance(key, HTTPSPoolKey) for key in p.pools.keys())) ++ ++ def test_https_connection_from_host_case_insensitive(self): ++ """Assert scheme case is ignored when getting the https key class.""" ++ p = PoolManager() ++ pool = p.connection_from_host('example.com', scheme='https') ++ other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTPS') ++ ++ self.assertEqual(1, len(p.pools)) ++ self.assertTrue(pool is other_pool) ++ self.assertTrue(all(isinstance(key, HTTPSPoolKey) for key in p.pools.keys())) ++ ++ def test_https_connection_from_context_case_insensitive(self): ++ """Assert scheme case is ignored when getting the https key class.""" ++ p = PoolManager() ++ context = {'scheme': 'https', 'host': 'example.com', 'port': '443'} ++ other_context = {'scheme': 'HTTPS', 'host': 'EXAMPLE.COM', 'port': '443'} ++ pool = p.connection_from_context(context) ++ other_pool = p.connection_from_context(other_context) ++ ++ self.assertEqual(1, len(p.pools)) ++ self.assertTrue(pool is other_pool) ++ self.assertTrue(all(isinstance(key, HTTPSPoolKey) for key in p.pools.keys())) ++ ++ def test_http_connection_from_url_case_insensitive(self): ++ """Assert scheme case is ignored when pooling HTTP connections.""" ++ p = PoolManager() ++ pool = p.connection_from_url('http://example.com/') ++ other_pool = p.connection_from_url('HTTP://EXAMPLE.COM/') ++ ++ self.assertEqual(1, len(p.pools)) ++ self.assertTrue(pool is other_pool) ++ self.assertTrue(all(isinstance(key, HTTPPoolKey) for key in p.pools.keys())) ++ ++ def test_http_connection_from_host_case_insensitive(self): ++ """Assert scheme case is ignored when getting the https key class.""" ++ p = PoolManager() ++ pool = p.connection_from_host('example.com', scheme='http') ++ other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTP') ++ ++ self.assertEqual(1, len(p.pools)) ++ self.assertTrue(pool is other_pool) ++ self.assertTrue(all(isinstance(key, HTTPPoolKey) for key in p.pools.keys())) ++ ++ def test_http_connection_from_context_case_insensitive(self): ++ """Assert scheme case is ignored when getting the https key class.""" ++ p = PoolManager() ++ context = {'scheme': 'http', 'host': 'example.com', 'port': '8080'} ++ other_context = {'scheme': 'HTTP', 'host': 'EXAMPLE.COM', 'port': '8080'} ++ pool = p.connection_from_context(context) ++ other_pool = p.connection_from_context(other_context) ++ ++ self.assertEqual(1, len(p.pools)) ++ self.assertTrue(pool is other_pool) ++ self.assertTrue(all(isinstance(key, HTTPPoolKey) for key in p.pools.keys())) ++ ++ def test_custom_pool_key(self): ++ """Assert it is possible to define addition pool key fields.""" ++ custom_key = namedtuple('CustomKey', HTTPPoolKey._fields + ('my_field',)) ++ p = PoolManager(10, my_field='barley') ++ ++ p.key_fn_by_scheme['http'] = functools.partial(_default_key_normalizer, custom_key) ++ p.connection_from_url('http://example.com') ++ p.connection_pool_kw['my_field'] = 'wheat' ++ p.connection_from_url('http://example.com') ++ ++ self.assertEqual(2, len(p.pools)) ++ + + if __name__ == '__main__': + unittest.main() +diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py +index b8d1e74..4fdae8d 100644 +--- a/urllib3/poolmanager.py ++++ b/urllib3/poolmanager.py +@@ -1,3 +1,5 @@ ++import collections ++import functools + import logging + + try: # Python 3 +@@ -17,16 +19,69 @@ from .util.retry import Retry + __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] + + ++log = logging.getLogger(__name__) ++ ++SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', ++ 'ssl_version', 'ca_cert_dir') ++ ++# The base fields to use when determining what pool to get a connection from; ++# these do not rely on the ``connection_pool_kw`` and can be determined by the ++# URL and potentially the ``urllib3.connection.port_by_scheme`` dictionary. ++# ++# All custom key schemes should include the fields in this key at a minimum. ++BasePoolKey = collections.namedtuple('BasePoolKey', ('scheme', 'host', 'port')) ++ ++# The fields to use when determining what pool to get a HTTP and HTTPS ++# connection from. All additional fields must be present in the PoolManager's ++# ``connection_pool_kw`` instance variable. ++HTTPPoolKey = collections.namedtuple( ++ 'HTTPPoolKey', BasePoolKey._fields + ('timeout', 'retries', 'strict', ++ 'block', 'source_address') ++) ++HTTPSPoolKey = collections.namedtuple( ++ 'HTTPSPoolKey', HTTPPoolKey._fields + SSL_KEYWORDS ++) ++ ++ ++def _default_key_normalizer(key_class, request_context): ++ """ ++ Create a pool key of type ``key_class`` for a request. ++ ++ According to RFC 3986, both the scheme and host are case-insensitive. ++ Therefore, this function normalizes both before constructing the pool ++ key for an HTTPS request. If you wish to change this behaviour, provide ++ alternate callables to ``key_fn_by_scheme``. ++ ++ :param key_class: ++ The class to use when constructing the key. This should be a namedtuple ++ with the ``scheme`` and ``host`` keys at a minimum. ++ ++ :param request_context: ++ A dictionary-like object that contain the context for a request. ++ It should contain a key for each field in the :class:`HTTPPoolKey` ++ """ ++ context = {} ++ for key in key_class._fields: ++ context[key] = request_context.get(key) ++ context['scheme'] = context['scheme'].lower() ++ context['host'] = context['host'].lower() ++ return key_class(**context) ++ ++ ++# A dictionary that maps a scheme to a callable that creates a pool key. ++# This can be used to alter the way pool keys are constructed, if desired. ++# Each PoolManager makes a copy of this dictionary so they can be configured ++# globally here, or individually on the instance. ++key_fn_by_scheme = { ++ 'http': functools.partial(_default_key_normalizer, HTTPPoolKey), ++ 'https': functools.partial(_default_key_normalizer, HTTPSPoolKey), ++} ++ + pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, + } + +-log = logging.getLogger(__name__) +- +-SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', +- 'ssl_version') +- + + class PoolManager(RequestMethods): + """ +@@ -64,6 +119,11 @@ class PoolManager(RequestMethods): + self.pools = RecentlyUsedContainer(num_pools, + dispose_func=lambda p: p.close()) + ++ # Locally set the pool classes and keys so other PoolManagers can ++ # override them. ++ self.pool_classes_by_scheme = pool_classes_by_scheme ++ self.key_fn_by_scheme = key_fn_by_scheme.copy() ++ + def __enter__(self): + return self + +@@ -109,10 +169,36 @@ class PoolManager(RequestMethods): + if not host: + raise LocationValueError("No host specified.") + +- scheme = scheme or 'http' +- port = port or port_by_scheme.get(scheme, 80) +- pool_key = (scheme, host, port) ++ request_context = self.connection_pool_kw.copy() ++ request_context['scheme'] = scheme or 'http' ++ if not port: ++ port = port_by_scheme.get(request_context['scheme'].lower(), 80) ++ request_context['port'] = port ++ request_context['host'] = host ++ ++ return self.connection_from_context(request_context) ++ ++ def connection_from_context(self, request_context): ++ """ ++ Get a :class:`ConnectionPool` based on the request context. ++ ++ ``request_context`` must at least contain the ``scheme`` key and its ++ value must be a key in ``key_fn_by_scheme`` instance variable. ++ """ ++ scheme = request_context['scheme'].lower() ++ pool_key_constructor = self.key_fn_by_scheme[scheme] ++ pool_key = pool_key_constructor(request_context) ++ ++ return self.connection_from_pool_key(pool_key) + ++ def connection_from_pool_key(self, pool_key): ++ """ ++ Get a :class:`ConnectionPool` based on the provided pool key. ++ ++ ``pool_key`` should be a namedtuple that only contains immutable ++ objects. At a minimum it must have the ``scheme``, ``host``, and ++ ``port`` fields. ++ """ + with self.pools.lock: + # If the scheme, host, or port doesn't match existing open + # connections, open a new ConnectionPool. +@@ -121,7 +207,7 @@ class PoolManager(RequestMethods): + return pool + + # Make a fresh ConnectionPool of the desired type +- pool = self._new_pool(scheme, host, port) ++ pool = self._new_pool(pool_key.scheme, pool_key.host, pool_key.port) + self.pools[pool_key] = pool + + return pool +-- +2.5.5 + diff --git a/SOURCES/python-urllib3-default-ssl-cert-validate.patch b/SOURCES/python-urllib3-default-ssl-cert-validate.patch new file mode 100644 index 0000000..f5551ed --- /dev/null +++ b/SOURCES/python-urllib3-default-ssl-cert-validate.patch @@ -0,0 +1,14 @@ +diff -up urllib3-1.10.2/urllib3/connectionpool.py.ms urllib3-1.10.2/urllib3/connectionpool.py +--- urllib3-1.10.2/urllib3/connectionpool.py.ms 2015-04-09 14:47:43.871891490 +0200 ++++ urllib3-1.10.2/urllib3/connectionpool.py 2015-04-09 14:48:40.709302298 +0200 +@@ -675,8 +675,8 @@ class HTTPSConnectionPool(HTTPConnection + strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, + block=False, headers=None, retries=None, + _proxy=None, _proxy_headers=None, +- key_file=None, cert_file=None, cert_reqs=None, +- ca_certs=None, ssl_version=None, ++ key_file=None, cert_file=None, cert_reqs='CERT_REQUIRED', ++ ca_certs='/etc/ssl/certs/ca-bundle.crt', ssl_version=None, + assert_hostname=None, assert_fingerprint=None, + **conn_kw): + diff --git a/SPECS/python-urllib3.spec b/SPECS/python-urllib3.spec new file mode 100644 index 0000000..3104ec2 --- /dev/null +++ b/SPECS/python-urllib3.spec @@ -0,0 +1,218 @@ +# Tests are EPEL only +%bcond_with tests + +# No Python 3 by default +%bcond_with python3 + +%global srcname urllib3 + +Name: python-%{srcname} +Version: 1.10.2 +Release: 7%{?dist} +Summary: Python HTTP library with thread-safe connection pooling and file post + +License: MIT +URL: http://urllib3.readthedocs.org/ +Source0: https://pypi.python.org/packages/source/u/%{srcname}/%{srcname}-%{version}.tar.gz + +# Patch to change default behaviour to check SSL certs for validity +# https://bugzilla.redhat.com/show_bug.cgi?id=855320 +Patch0: python-urllib3-default-ssl-cert-validate.patch + +# Patch for the PoolManager instance to consider additional SSL +# configuration when providing a pooled connection for a request. +# https://bugzilla.redhat.com/show_bug.cgi?id=1329395 +# Upstream issue: https://github.com/shazow/urllib3/pull/830 +Patch1: key-connection-pools-off-custom-keys.patch + +# Support IP address SAN fields. +# https://bugzilla.redhat.com/show_bug.cgi?id=1434114 +# Upstream: https://github.com/shazow/urllib3/pull/922 +Patch2: Add-support-for-IP-address-SAN-fields.patch + +# Patch for CVE-2018-20060 +# Cross-host redirect does not remove Authorization header allow +# for credential exposure +# Backported without tests! +# https://bugzilla.redhat.com/show_bug.cgi?id=1649153 +# Upstream: https://github.com/urllib3/urllib3/pull/1346 +Patch3: CVE-2018-20060.patch + +# Patch for CVE-2019-9740 +# CRLF injection due to not encoding the '\r\n' sequence leading to possible +# attack on internal service. +# https://github.com/urllib3/urllib3/pull/1591/commits/c147f359520cab339ec96b3ef96e471c0da261f6 +# https://github.com/urllib3/urllib3/pull/1593/commits/e951dfc83a642b0b5239559cb1c8cc287481f1ae +# https://bugzilla.redhat.com/show_bug.cgi?id=1700824 +Patch4: CVE-2019-9740.patch + +BuildArch: noarch + +Requires: ca-certificates + +# Previously bundled things: +Requires: python-six +Requires: python-backports-ssl_match_hostname +Requires: python-ipaddress + +%if 0%{?rhel} <= 6 +BuildRequires: python-ordereddict +Requires: python-ordereddict +%endif + +BuildRequires: python2-devel +BuildRequires: python-setuptools +BuildRequires: python-six +BuildRequires: python-backports-ssl_match_hostname +BuildRequires: python-ipaddress +# For unittests +%if %{with tests} +BuildRequires: python-nose +BuildRequires: python-tornado +BuildRequires: python-mock +%endif + +%{?python_provide:%python_provide %{name}} + +%description +Python HTTP module with connection pooling and file POST abilities. + + +%if %{with python3} +%package -n python3-%{srcname} +Summary: Python 3 HTTP library with thread-safe connection pooling and file post +%{?python_provide:%python_provide python3-%{srcname}} + +Requires: ca-certificates +Requires: python3-six +BuildRequires: python3-devel + +# For unittests +%if %{with tests} +BuildRequires: python3-nose +BuildRequires: python3-six +BuildRequires: python3-tornado +BuildRequires: python3-mock +%endif # with tests + +%description -n python3-%{srcname} +Python3 HTTP module with connection pooling and file POST abilities. +%endif # with python3 + + +%prep +%setup -q -n %{srcname}-%{version} + +# Drop the dummyserver tests in koji. They fail there in real builds, but not +# in scratch builds (weird). +rm -rf test/with_dummyserver/ + +%patch0 -p1 +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 + + +%build +%py2_build + +%if %{with python3} +%py3_build +%endif + +%install +%py2_install + +rm -rf %{buildroot}/%{python2_sitelib}/urllib3/packages/six.py* +rm -rf %{buildroot}/%{python2_sitelib}/urllib3/packages/ssl_match_hostname/ + +mkdir -p %{buildroot}/%{python2_sitelib}/urllib3/packages/ +# ovirt composes remove *.py files, leaving only *.pyc files there; this means we have to symlink +# six.py* to make sure urllib3.packages.six will be importable +for i in ../../six.py{,o,c}; do + ln -s $i %{buildroot}/%{python2_sitelib}/urllib3/packages/ +done +ln -s ../../backports/ssl_match_hostname %{buildroot}/%{python2_sitelib}/urllib3/packages/ssl_match_hostname + +# dummyserver is part of the unittest framework +rm -rf %{buildroot}%{python2_sitelib}/dummyserver + +%if %{with python3} +%py3_install + +# dummyserver is part of the unittest framework +rm -rf %{buildroot}%{python3_sitelib}/dummyserver +%endif # with python3 + +%if %{with tests} +%check +nosetests-%{python2_version} + +%if %{with python3} +nosetests-%{python3_version} +%endif # with python3 +%endif # with tests + +%files +%doc CHANGES.rst README.rst CONTRIBUTORS.txt +%license LICENSE.txt +%{python2_sitelib}/urllib3* + +%if %{with python3} +%files -n python3-%{srcname} +%doc CHANGES.rst README.rst CONTRIBUTORS.txt +%license LICENSE.txt +%{python3_sitelib}/urllib3* +%endif # with python3 + +%changelog +* Fri May 03 2019 Miro Hrončok - 1.10.2-7 +- Provide python2-urllib3 +- Add patch for CVE-2019-11236 +Resolves: rhbz#1703360 + +* Mon Mar 04 2019 Lumír Balhar - 1.10.2-6 +- Source URL switched to HTTPS protocol +- Add patch for CVE-2018-20060 +Resolves: rhbz#1658471 + +* Wed Oct 11 2017 Iryna Shcherbina - 1.10.2-5 +- Add patch to support IP address SAN fields. +Resolves: rhbz#1434114 + +* Thu Sep 14 2017 Charalampos Stratakis - 1.10.2-4 +- Update patch to find ca_certs in the correct location. +Resolves: rhbz#1450213 + +* Mon Jan 23 2017 Iryna Shcherbina - 1.10.2-3 +- Fix PoolManager instance to take into account new SSL configuration +Resolves: rhbz#1329395 + +* Mon Jul 27 2015 bkabrda - 1.10.2-2 +- Fix the way we unbundle six to make ovirt work even when they remove .py files +Resolves: rhbz#1247093 + +* Mon Apr 13 2015 Matej Stuchlik - 1.10.2-1 +- Update to 1.10.2 +Resolves: rhbz#1226901 + +* Fri Mar 1 2013 Toshio Kuratomi - 1.5-5 +- Unbundling finished! + +* Fri Mar 01 2013 Ralph Bean - 1.5-4 +- Upstream patch to fix Accept header when behind a proxy. +- Reorganize patch numbers to more clearly distinguish them. + +* Wed Feb 27 2013 Ralph Bean - 1.5-3 +- Renamed patches to python-urllib3-* +- Fixed ssl check patch to use the correct cert path for Fedora. +- Included dependency on ca-certificates +- Cosmetic indentation changes to the .spec file. + +* Tue Feb 5 2013 Toshio Kuratomi - 1.5-2 +- python3-tornado BR and run all unittests on python3 + +* Mon Feb 04 2013 Toshio Kuratomi 1.5-1 +- Initial fedora build. +