|
|
75e569 |
From 99274b9e7524be7e8f041e327be1dd7ba3fdc8a4 Mon Sep 17 00:00:00 2001
|
|
|
75e569 |
From: Jeremy Cline <jcline@redhat.com>
|
|
|
75e569 |
Date: Thu, 21 Apr 2016 10:56:28 -0700
|
|
|
75e569 |
Subject: [PATCH] Key connection pools off custom keys
|
|
|
75e569 |
|
|
|
75e569 |
---
|
|
|
75e569 |
CONTRIBUTORS.txt | 3 +
|
|
|
75e569 |
docs/managers.rst | 44 +++++++++
|
|
|
75e569 |
test/test_poolmanager.py | 230 ++++++++++++++++++++++++++++++++++++++++++++++-
|
|
|
75e569 |
urllib3/poolmanager.py | 104 +++++++++++++++++++--
|
|
|
75e569 |
4 files changed, 371 insertions(+), 10 deletions(-)
|
|
|
75e569 |
|
|
|
75e569 |
diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
|
|
|
75e569 |
index 5141a50..f32faaa 100644
|
|
|
75e569 |
--- a/CONTRIBUTORS.txt
|
|
|
75e569 |
+++ b/CONTRIBUTORS.txt
|
|
|
75e569 |
@@ -142,5 +142,8 @@ In chronological order:
|
|
|
75e569 |
* Alex Gaynor <alex.gaynor@gmail.com>
|
|
|
75e569 |
* Updates to the default SSL configuration
|
|
|
75e569 |
|
|
|
75e569 |
+* Jeremy Cline <jeremy@jcline.org>
|
|
|
75e569 |
+ * Added connection pool keys by scheme
|
|
|
75e569 |
+
|
|
|
75e569 |
* [Your name or handle] <[email or website]>
|
|
|
75e569 |
* [Brief summary of your changes]
|
|
|
75e569 |
diff --git a/docs/managers.rst b/docs/managers.rst
|
|
|
75e569 |
index 6c841b7..ef91ca4 100644
|
|
|
75e569 |
--- a/docs/managers.rst
|
|
|
75e569 |
+++ b/docs/managers.rst
|
|
|
75e569 |
@@ -28,6 +28,44 @@ so you don't have to.
|
|
|
75e569 |
>>> conn.num_requests
|
|
|
75e569 |
3
|
|
|
75e569 |
|
|
|
75e569 |
+A :class:`.PoolManager` will create a new :doc:`ConnectionPool <pools>`
|
|
|
75e569 |
+when no :doc:`ConnectionPools <pools>` exist with a matching pool key.
|
|
|
75e569 |
+The pool key is derived using the requested URL and the current values
|
|
|
75e569 |
+of the ``connection_pool_kw`` instance variable on :class:`.PoolManager`.
|
|
|
75e569 |
+
|
|
|
75e569 |
+The keys in ``connection_pool_kw`` used when deriving the key are
|
|
|
75e569 |
+configurable. For example, by default the ``my_field`` key is not
|
|
|
75e569 |
+considered.
|
|
|
75e569 |
+
|
|
|
75e569 |
+.. doctest ::
|
|
|
75e569 |
+
|
|
|
75e569 |
+ >>> from urllib3.poolmanager import PoolManager
|
|
|
75e569 |
+ >>> manager = PoolManager(10, my_field='wheat')
|
|
|
75e569 |
+ >>> manager.connection_from_url('http://example.com')
|
|
|
75e569 |
+ >>> manager.connection_pool_kw['my_field'] = 'barley'
|
|
|
75e569 |
+ >>> manager.connection_from_url('http://example.com')
|
|
|
75e569 |
+ >>> len(manager.pools)
|
|
|
75e569 |
+ 1
|
|
|
75e569 |
+
|
|
|
75e569 |
+To make the pool manager create new pools when the value of
|
|
|
75e569 |
+``my_field`` changes, you can define a custom pool key and alter
|
|
|
75e569 |
+the ``key_fn_by_scheme`` instance variable on :class:`.PoolManager`.
|
|
|
75e569 |
+
|
|
|
75e569 |
+.. doctest ::
|
|
|
75e569 |
+
|
|
|
75e569 |
+ >>> import functools
|
|
|
75e569 |
+ >>> from collections import namedtuple
|
|
|
75e569 |
+ >>> from urllib3.poolmanager import PoolManager, HTTPPoolKey
|
|
|
75e569 |
+ >>> from urllib3.poolmanager import default_key_normalizer as normalizer
|
|
|
75e569 |
+ >>> CustomKey = namedtuple('CustomKey', HTTPPoolKey._fields + ('my_field',))
|
|
|
75e569 |
+ >>> manager = PoolManager(10, my_field='wheat')
|
|
|
75e569 |
+ >>> manager.key_fn_by_scheme['http'] = functools.partial(normalizer, CustomKey)
|
|
|
75e569 |
+ >>> manager.connection_from_url('http://example.com')
|
|
|
75e569 |
+ >>> manager.connection_pool_kw['my_field'] = 'barley'
|
|
|
75e569 |
+ >>> manager.connection_from_url('http://example.com')
|
|
|
75e569 |
+ >>> len(manager.pools)
|
|
|
75e569 |
+ 2
|
|
|
75e569 |
+
|
|
|
75e569 |
The API of a :class:`.PoolManager` object is similar to that of a
|
|
|
75e569 |
:doc:`ConnectionPool <pools>`, so they can be passed around interchangeably.
|
|
|
75e569 |
|
|
|
75e569 |
@@ -59,6 +97,12 @@ API
|
|
|
75e569 |
|
|
|
75e569 |
.. autoclass:: PoolManager
|
|
|
75e569 |
:inherited-members:
|
|
|
75e569 |
+ .. autoclass:: BasePoolKey
|
|
|
75e569 |
+ :inherited-members:
|
|
|
75e569 |
+ .. autoclass:: HTTPPoolKey
|
|
|
75e569 |
+ :inherited-members:
|
|
|
75e569 |
+ .. autoclass:: HTTPSPoolKey
|
|
|
75e569 |
+ :inherited-members:
|
|
|
75e569 |
|
|
|
75e569 |
ProxyManager
|
|
|
75e569 |
============
|
|
|
75e569 |
diff --git a/test/test_poolmanager.py b/test/test_poolmanager.py
|
|
|
75e569 |
index 6195d51..fb134fb 100644
|
|
|
75e569 |
--- a/test/test_poolmanager.py
|
|
|
75e569 |
+++ b/test/test_poolmanager.py
|
|
|
75e569 |
@@ -1,11 +1,21 @@
|
|
|
75e569 |
+import functools
|
|
|
75e569 |
import unittest
|
|
|
75e569 |
+from collections import namedtuple
|
|
|
75e569 |
|
|
|
75e569 |
-from urllib3.poolmanager import PoolManager
|
|
|
75e569 |
+from urllib3.poolmanager import (
|
|
|
75e569 |
+ _default_key_normalizer,
|
|
|
75e569 |
+ HTTPPoolKey,
|
|
|
75e569 |
+ HTTPSPoolKey,
|
|
|
75e569 |
+ key_fn_by_scheme,
|
|
|
75e569 |
+ PoolManager,
|
|
|
75e569 |
+ SSL_KEYWORDS,
|
|
|
75e569 |
+)
|
|
|
75e569 |
from urllib3 import connection_from_url
|
|
|
75e569 |
from urllib3.exceptions import (
|
|
|
75e569 |
ClosedPoolError,
|
|
|
75e569 |
LocationValueError,
|
|
|
75e569 |
)
|
|
|
75e569 |
+from urllib3.util import retry, timeout
|
|
|
75e569 |
|
|
|
75e569 |
|
|
|
75e569 |
class TestPoolManager(unittest.TestCase):
|
|
|
75e569 |
@@ -87,6 +97,224 @@ class TestPoolManager(unittest.TestCase):
|
|
|
75e569 |
|
|
|
75e569 |
self.assertEqual(len(p.pools), 0)
|
|
|
75e569 |
|
|
|
75e569 |
+ def test_http_pool_key_fields(self):
|
|
|
75e569 |
+ """Assert the HTTPPoolKey fields are honored when selecting a pool."""
|
|
|
75e569 |
+ connection_pool_kw = {
|
|
|
75e569 |
+ 'timeout': timeout.Timeout(3.14),
|
|
|
75e569 |
+ 'retries': retry.Retry(total=6, connect=2),
|
|
|
75e569 |
+ 'block': True,
|
|
|
75e569 |
+ 'strict': True,
|
|
|
75e569 |
+ 'source_address': '127.0.0.1',
|
|
|
75e569 |
+ }
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ conn_pools = [
|
|
|
75e569 |
+ p.connection_from_url('http://example.com/'),
|
|
|
75e569 |
+ p.connection_from_url('http://example.com:8000/'),
|
|
|
75e569 |
+ p.connection_from_url('http://other.example.com/'),
|
|
|
75e569 |
+ ]
|
|
|
75e569 |
+
|
|
|
75e569 |
+ for key, value in connection_pool_kw.items():
|
|
|
75e569 |
+ p.connection_pool_kw[key] = value
|
|
|
75e569 |
+ conn_pools.append(p.connection_from_url('http://example.com/'))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertTrue(
|
|
|
75e569 |
+ all(
|
|
|
75e569 |
+ x is not y
|
|
|
75e569 |
+ for i, x in enumerate(conn_pools)
|
|
|
75e569 |
+ for j, y in enumerate(conn_pools)
|
|
|
75e569 |
+ if i != j
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+ self.assertTrue(
|
|
|
75e569 |
+ all(
|
|
|
75e569 |
+ isinstance(key, HTTPPoolKey)
|
|
|
75e569 |
+ for key in p.pools.keys())
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_http_pool_key_extra_kwargs(self):
|
|
|
75e569 |
+ """Assert non-HTTPPoolKey fields are ignored when selecting a pool."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ conn_pool = p.connection_from_url('http://example.com/')
|
|
|
75e569 |
+ p.connection_pool_kw['some_kwarg'] = 'that should be ignored'
|
|
|
75e569 |
+ other_conn_pool = p.connection_from_url('http://example.com/')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertTrue(conn_pool is other_conn_pool)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_http_pool_key_https_kwargs(self):
|
|
|
75e569 |
+ """Assert HTTPSPoolKey fields are ignored when selecting a HTTP pool."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ conn_pool = p.connection_from_url('http://example.com/')
|
|
|
75e569 |
+ for key in SSL_KEYWORDS:
|
|
|
75e569 |
+ p.connection_pool_kw[key] = 'this should be ignored'
|
|
|
75e569 |
+ other_conn_pool = p.connection_from_url('http://example.com/')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertTrue(conn_pool is other_conn_pool)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_https_pool_key_fields(self):
|
|
|
75e569 |
+ """Assert the HTTPSPoolKey fields are honored when selecting a pool."""
|
|
|
75e569 |
+ connection_pool_kw = {
|
|
|
75e569 |
+ 'timeout': timeout.Timeout(3.14),
|
|
|
75e569 |
+ 'retries': retry.Retry(total=6, connect=2),
|
|
|
75e569 |
+ 'block': True,
|
|
|
75e569 |
+ 'strict': True,
|
|
|
75e569 |
+ 'source_address': '127.0.0.1',
|
|
|
75e569 |
+ 'key_file': '/root/totally_legit.key',
|
|
|
75e569 |
+ 'cert_file': '/root/totally_legit.crt',
|
|
|
75e569 |
+ 'cert_reqs': 'CERT_REQUIRED',
|
|
|
75e569 |
+ 'ca_certs': '/root/path_to_pem',
|
|
|
75e569 |
+ 'ssl_version': 'SSLv23_METHOD',
|
|
|
75e569 |
+ }
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ conn_pools = [
|
|
|
75e569 |
+ p.connection_from_url('https://example.com/'),
|
|
|
75e569 |
+ p.connection_from_url('https://example.com:4333/'),
|
|
|
75e569 |
+ p.connection_from_url('https://other.example.com/'),
|
|
|
75e569 |
+ ]
|
|
|
75e569 |
+ # Asking for a connection pool with the same key should give us an
|
|
|
75e569 |
+ # existing pool.
|
|
|
75e569 |
+ dup_pools = []
|
|
|
75e569 |
+
|
|
|
75e569 |
+ for key, value in connection_pool_kw.items():
|
|
|
75e569 |
+ p.connection_pool_kw[key] = value
|
|
|
75e569 |
+ conn_pools.append(p.connection_from_url('https://example.com/'))
|
|
|
75e569 |
+ dup_pools.append(p.connection_from_url('https://example.com/'))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertTrue(
|
|
|
75e569 |
+ all(
|
|
|
75e569 |
+ x is not y
|
|
|
75e569 |
+ for i, x in enumerate(conn_pools)
|
|
|
75e569 |
+ for j, y in enumerate(conn_pools)
|
|
|
75e569 |
+ if i != j
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+ self.assertTrue(all(pool in conn_pools for pool in dup_pools))
|
|
|
75e569 |
+ self.assertTrue(
|
|
|
75e569 |
+ all(
|
|
|
75e569 |
+ isinstance(key, HTTPSPoolKey)
|
|
|
75e569 |
+ for key in p.pools.keys())
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_https_pool_key_extra_kwargs(self):
|
|
|
75e569 |
+ """Assert non-HTTPSPoolKey fields are ignored when selecting a pool."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ conn_pool = p.connection_from_url('https://example.com/')
|
|
|
75e569 |
+ p.connection_pool_kw['some_kwarg'] = 'that should be ignored'
|
|
|
75e569 |
+ other_conn_pool = p.connection_from_url('https://example.com/')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertTrue(conn_pool is other_conn_pool)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_default_pool_key_funcs_copy(self):
|
|
|
75e569 |
+ """Assert each PoolManager gets a copy of ``pool_keys_by_scheme``."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ self.assertEqual(p.key_fn_by_scheme, p.key_fn_by_scheme)
|
|
|
75e569 |
+ self.assertFalse(p.key_fn_by_scheme is key_fn_by_scheme)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_pools_keyed_with_from_host(self):
|
|
|
75e569 |
+ """Assert pools are still keyed correctly with connection_from_host."""
|
|
|
75e569 |
+ ssl_kw = {
|
|
|
75e569 |
+ 'key_file': '/root/totally_legit.key',
|
|
|
75e569 |
+ 'cert_file': '/root/totally_legit.crt',
|
|
|
75e569 |
+ 'cert_reqs': 'CERT_REQUIRED',
|
|
|
75e569 |
+ 'ca_certs': '/root/path_to_pem',
|
|
|
75e569 |
+ 'ssl_version': 'SSLv23_METHOD',
|
|
|
75e569 |
+ }
|
|
|
75e569 |
+ p = PoolManager(5, **ssl_kw)
|
|
|
75e569 |
+ conns = []
|
|
|
75e569 |
+ conns.append(
|
|
|
75e569 |
+ p.connection_from_host('example.com', 443, scheme='https')
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+
|
|
|
75e569 |
+ for k in ssl_kw:
|
|
|
75e569 |
+ p.connection_pool_kw[k] = 'newval'
|
|
|
75e569 |
+ conns.append(
|
|
|
75e569 |
+ p.connection_from_host('example.com', 443, scheme='https')
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertTrue(
|
|
|
75e569 |
+ all(
|
|
|
75e569 |
+ x is not y
|
|
|
75e569 |
+ for i, x in enumerate(conns)
|
|
|
75e569 |
+ for j, y in enumerate(conns)
|
|
|
75e569 |
+ if i != j
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+ )
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_https_connection_from_url_case_insensitive(self):
|
|
|
75e569 |
+ """Assert scheme case is ignored when pooling HTTPS connections."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ pool = p.connection_from_url('https://example.com/')
|
|
|
75e569 |
+ other_pool = p.connection_from_url('HTTPS://EXAMPLE.COM/')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(1, len(p.pools))
|
|
|
75e569 |
+ self.assertTrue(pool is other_pool)
|
|
|
75e569 |
+ self.assertTrue(all(isinstance(key, HTTPSPoolKey) for key in p.pools.keys()))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_https_connection_from_host_case_insensitive(self):
|
|
|
75e569 |
+ """Assert scheme case is ignored when getting the https key class."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ pool = p.connection_from_host('example.com', scheme='https')
|
|
|
75e569 |
+ other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTPS')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(1, len(p.pools))
|
|
|
75e569 |
+ self.assertTrue(pool is other_pool)
|
|
|
75e569 |
+ self.assertTrue(all(isinstance(key, HTTPSPoolKey) for key in p.pools.keys()))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_https_connection_from_context_case_insensitive(self):
|
|
|
75e569 |
+ """Assert scheme case is ignored when getting the https key class."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ context = {'scheme': 'https', 'host': 'example.com', 'port': '443'}
|
|
|
75e569 |
+ other_context = {'scheme': 'HTTPS', 'host': 'EXAMPLE.COM', 'port': '443'}
|
|
|
75e569 |
+ pool = p.connection_from_context(context)
|
|
|
75e569 |
+ other_pool = p.connection_from_context(other_context)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(1, len(p.pools))
|
|
|
75e569 |
+ self.assertTrue(pool is other_pool)
|
|
|
75e569 |
+ self.assertTrue(all(isinstance(key, HTTPSPoolKey) for key in p.pools.keys()))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_http_connection_from_url_case_insensitive(self):
|
|
|
75e569 |
+ """Assert scheme case is ignored when pooling HTTP connections."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ pool = p.connection_from_url('http://example.com/')
|
|
|
75e569 |
+ other_pool = p.connection_from_url('HTTP://EXAMPLE.COM/')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(1, len(p.pools))
|
|
|
75e569 |
+ self.assertTrue(pool is other_pool)
|
|
|
75e569 |
+ self.assertTrue(all(isinstance(key, HTTPPoolKey) for key in p.pools.keys()))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_http_connection_from_host_case_insensitive(self):
|
|
|
75e569 |
+ """Assert scheme case is ignored when getting the https key class."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ pool = p.connection_from_host('example.com', scheme='http')
|
|
|
75e569 |
+ other_pool = p.connection_from_host('EXAMPLE.COM', scheme='HTTP')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(1, len(p.pools))
|
|
|
75e569 |
+ self.assertTrue(pool is other_pool)
|
|
|
75e569 |
+ self.assertTrue(all(isinstance(key, HTTPPoolKey) for key in p.pools.keys()))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_http_connection_from_context_case_insensitive(self):
|
|
|
75e569 |
+ """Assert scheme case is ignored when getting the https key class."""
|
|
|
75e569 |
+ p = PoolManager()
|
|
|
75e569 |
+ context = {'scheme': 'http', 'host': 'example.com', 'port': '8080'}
|
|
|
75e569 |
+ other_context = {'scheme': 'HTTP', 'host': 'EXAMPLE.COM', 'port': '8080'}
|
|
|
75e569 |
+ pool = p.connection_from_context(context)
|
|
|
75e569 |
+ other_pool = p.connection_from_context(other_context)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(1, len(p.pools))
|
|
|
75e569 |
+ self.assertTrue(pool is other_pool)
|
|
|
75e569 |
+ self.assertTrue(all(isinstance(key, HTTPPoolKey) for key in p.pools.keys()))
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def test_custom_pool_key(self):
|
|
|
75e569 |
+ """Assert it is possible to define addition pool key fields."""
|
|
|
75e569 |
+ custom_key = namedtuple('CustomKey', HTTPPoolKey._fields + ('my_field',))
|
|
|
75e569 |
+ p = PoolManager(10, my_field='barley')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ p.key_fn_by_scheme['http'] = functools.partial(_default_key_normalizer, custom_key)
|
|
|
75e569 |
+ p.connection_from_url('http://example.com')
|
|
|
75e569 |
+ p.connection_pool_kw['my_field'] = 'wheat'
|
|
|
75e569 |
+ p.connection_from_url('http://example.com')
|
|
|
75e569 |
+
|
|
|
75e569 |
+ self.assertEqual(2, len(p.pools))
|
|
|
75e569 |
+
|
|
|
75e569 |
|
|
|
75e569 |
if __name__ == '__main__':
|
|
|
75e569 |
unittest.main()
|
|
|
75e569 |
diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py
|
|
|
75e569 |
index b8d1e74..4fdae8d 100644
|
|
|
75e569 |
--- a/urllib3/poolmanager.py
|
|
|
75e569 |
+++ b/urllib3/poolmanager.py
|
|
|
75e569 |
@@ -1,3 +1,5 @@
|
|
|
75e569 |
+import collections
|
|
|
75e569 |
+import functools
|
|
|
75e569 |
import logging
|
|
|
75e569 |
|
|
|
75e569 |
try: # Python 3
|
|
|
75e569 |
@@ -17,16 +19,69 @@ from .util.retry import Retry
|
|
|
75e569 |
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
|
|
|
75e569 |
|
|
|
75e569 |
|
|
|
75e569 |
+log = logging.getLogger(__name__)
|
|
|
75e569 |
+
|
|
|
75e569 |
+SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
|
|
|
75e569 |
+ 'ssl_version', 'ca_cert_dir')
|
|
|
75e569 |
+
|
|
|
75e569 |
+# The base fields to use when determining what pool to get a connection from;
|
|
|
75e569 |
+# these do not rely on the ``connection_pool_kw`` and can be determined by the
|
|
|
75e569 |
+# URL and potentially the ``urllib3.connection.port_by_scheme`` dictionary.
|
|
|
75e569 |
+#
|
|
|
75e569 |
+# All custom key schemes should include the fields in this key at a minimum.
|
|
|
75e569 |
+BasePoolKey = collections.namedtuple('BasePoolKey', ('scheme', 'host', 'port'))
|
|
|
75e569 |
+
|
|
|
75e569 |
+# The fields to use when determining what pool to get a HTTP and HTTPS
|
|
|
75e569 |
+# connection from. All additional fields must be present in the PoolManager's
|
|
|
75e569 |
+# ``connection_pool_kw`` instance variable.
|
|
|
75e569 |
+HTTPPoolKey = collections.namedtuple(
|
|
|
75e569 |
+ 'HTTPPoolKey', BasePoolKey._fields + ('timeout', 'retries', 'strict',
|
|
|
75e569 |
+ 'block', 'source_address')
|
|
|
75e569 |
+)
|
|
|
75e569 |
+HTTPSPoolKey = collections.namedtuple(
|
|
|
75e569 |
+ 'HTTPSPoolKey', HTTPPoolKey._fields + SSL_KEYWORDS
|
|
|
75e569 |
+)
|
|
|
75e569 |
+
|
|
|
75e569 |
+
|
|
|
75e569 |
+def _default_key_normalizer(key_class, request_context):
|
|
|
75e569 |
+ """
|
|
|
75e569 |
+ Create a pool key of type ``key_class`` for a request.
|
|
|
75e569 |
+
|
|
|
75e569 |
+ According to RFC 3986, both the scheme and host are case-insensitive.
|
|
|
75e569 |
+ Therefore, this function normalizes both before constructing the pool
|
|
|
75e569 |
+ key for an HTTPS request. If you wish to change this behaviour, provide
|
|
|
75e569 |
+ alternate callables to ``key_fn_by_scheme``.
|
|
|
75e569 |
+
|
|
|
75e569 |
+ :param key_class:
|
|
|
75e569 |
+ The class to use when constructing the key. This should be a namedtuple
|
|
|
75e569 |
+ with the ``scheme`` and ``host`` keys at a minimum.
|
|
|
75e569 |
+
|
|
|
75e569 |
+ :param request_context:
|
|
|
75e569 |
+ A dictionary-like object that contain the context for a request.
|
|
|
75e569 |
+ It should contain a key for each field in the :class:`HTTPPoolKey`
|
|
|
75e569 |
+ """
|
|
|
75e569 |
+ context = {}
|
|
|
75e569 |
+ for key in key_class._fields:
|
|
|
75e569 |
+ context[key] = request_context.get(key)
|
|
|
75e569 |
+ context['scheme'] = context['scheme'].lower()
|
|
|
75e569 |
+ context['host'] = context['host'].lower()
|
|
|
75e569 |
+ return key_class(**context)
|
|
|
75e569 |
+
|
|
|
75e569 |
+
|
|
|
75e569 |
+# A dictionary that maps a scheme to a callable that creates a pool key.
|
|
|
75e569 |
+# This can be used to alter the way pool keys are constructed, if desired.
|
|
|
75e569 |
+# Each PoolManager makes a copy of this dictionary so they can be configured
|
|
|
75e569 |
+# globally here, or individually on the instance.
|
|
|
75e569 |
+key_fn_by_scheme = {
|
|
|
75e569 |
+ 'http': functools.partial(_default_key_normalizer, HTTPPoolKey),
|
|
|
75e569 |
+ 'https': functools.partial(_default_key_normalizer, HTTPSPoolKey),
|
|
|
75e569 |
+}
|
|
|
75e569 |
+
|
|
|
75e569 |
pool_classes_by_scheme = {
|
|
|
75e569 |
'http': HTTPConnectionPool,
|
|
|
75e569 |
'https': HTTPSConnectionPool,
|
|
|
75e569 |
}
|
|
|
75e569 |
|
|
|
75e569 |
-log = logging.getLogger(__name__)
|
|
|
75e569 |
-
|
|
|
75e569 |
-SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs',
|
|
|
75e569 |
- 'ssl_version')
|
|
|
75e569 |
-
|
|
|
75e569 |
|
|
|
75e569 |
class PoolManager(RequestMethods):
|
|
|
75e569 |
"""
|
|
|
75e569 |
@@ -64,6 +119,11 @@ class PoolManager(RequestMethods):
|
|
|
75e569 |
self.pools = RecentlyUsedContainer(num_pools,
|
|
|
75e569 |
dispose_func=lambda p: p.close())
|
|
|
75e569 |
|
|
|
75e569 |
+ # Locally set the pool classes and keys so other PoolManagers can
|
|
|
75e569 |
+ # override them.
|
|
|
75e569 |
+ self.pool_classes_by_scheme = pool_classes_by_scheme
|
|
|
75e569 |
+ self.key_fn_by_scheme = key_fn_by_scheme.copy()
|
|
|
75e569 |
+
|
|
|
75e569 |
def __enter__(self):
|
|
|
75e569 |
return self
|
|
|
75e569 |
|
|
|
75e569 |
@@ -109,10 +169,36 @@ class PoolManager(RequestMethods):
|
|
|
75e569 |
if not host:
|
|
|
75e569 |
raise LocationValueError("No host specified.")
|
|
|
75e569 |
|
|
|
75e569 |
- scheme = scheme or 'http'
|
|
|
75e569 |
- port = port or port_by_scheme.get(scheme, 80)
|
|
|
75e569 |
- pool_key = (scheme, host, port)
|
|
|
75e569 |
+ request_context = self.connection_pool_kw.copy()
|
|
|
75e569 |
+ request_context['scheme'] = scheme or 'http'
|
|
|
75e569 |
+ if not port:
|
|
|
75e569 |
+ port = port_by_scheme.get(request_context['scheme'].lower(), 80)
|
|
|
75e569 |
+ request_context['port'] = port
|
|
|
75e569 |
+ request_context['host'] = host
|
|
|
75e569 |
+
|
|
|
75e569 |
+ return self.connection_from_context(request_context)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ def connection_from_context(self, request_context):
|
|
|
75e569 |
+ """
|
|
|
75e569 |
+ Get a :class:`ConnectionPool` based on the request context.
|
|
|
75e569 |
+
|
|
|
75e569 |
+ ``request_context`` must at least contain the ``scheme`` key and its
|
|
|
75e569 |
+ value must be a key in ``key_fn_by_scheme`` instance variable.
|
|
|
75e569 |
+ """
|
|
|
75e569 |
+ scheme = request_context['scheme'].lower()
|
|
|
75e569 |
+ pool_key_constructor = self.key_fn_by_scheme[scheme]
|
|
|
75e569 |
+ pool_key = pool_key_constructor(request_context)
|
|
|
75e569 |
+
|
|
|
75e569 |
+ return self.connection_from_pool_key(pool_key)
|
|
|
75e569 |
|
|
|
75e569 |
+ def connection_from_pool_key(self, pool_key):
|
|
|
75e569 |
+ """
|
|
|
75e569 |
+ Get a :class:`ConnectionPool` based on the provided pool key.
|
|
|
75e569 |
+
|
|
|
75e569 |
+ ``pool_key`` should be a namedtuple that only contains immutable
|
|
|
75e569 |
+ objects. At a minimum it must have the ``scheme``, ``host``, and
|
|
|
75e569 |
+ ``port`` fields.
|
|
|
75e569 |
+ """
|
|
|
75e569 |
with self.pools.lock:
|
|
|
75e569 |
# If the scheme, host, or port doesn't match existing open
|
|
|
75e569 |
# connections, open a new ConnectionPool.
|
|
|
75e569 |
@@ -121,7 +207,7 @@ class PoolManager(RequestMethods):
|
|
|
75e569 |
return pool
|
|
|
75e569 |
|
|
|
75e569 |
# Make a fresh ConnectionPool of the desired type
|
|
|
75e569 |
- pool = self._new_pool(scheme, host, port)
|
|
|
75e569 |
+ pool = self._new_pool(pool_key.scheme, pool_key.host, pool_key.port)
|
|
|
75e569 |
self.pools[pool_key] = pool
|
|
|
75e569 |
|
|
|
75e569 |
return pool
|
|
|
75e569 |
--
|
|
|
75e569 |
2.5.5
|
|
|
75e569 |
|