|
 |
babe2e |
commit 30b1a7ba876bc798199aba99c013983cba79d72a
|
|
|
babe2e |
Author: Michal Domonkos <mdomonko@redhat.com>
|
|
|
babe2e |
Date: Wed Nov 23 12:54:05 2016 +0100
|
|
|
babe2e |
|
|
|
babe2e |
Add curl_obj option. BZ 1204825
|
|
|
babe2e |
|
|
|
babe2e |
diff --git a/urlgrabber/grabber.py b/urlgrabber/grabber.py
|
|
|
babe2e |
index 074a82f..25d8d17 100644
|
|
|
babe2e |
--- a/urlgrabber/grabber.py
|
|
|
babe2e |
+++ b/urlgrabber/grabber.py
|
|
|
babe2e |
@@ -69,6 +69,24 @@ GENERAL ARGUMENTS (kwargs)
|
|
|
babe2e |
is None progress_obj is used in compatibility mode: finished files
|
|
|
babe2e |
are shown but there's no in-progress display.
|
|
|
babe2e |
|
|
|
babe2e |
+ curl_obj = None
|
|
|
babe2e |
+
|
|
|
babe2e |
+ a pycurl.Curl instance to be used instead of the default module-level
|
|
|
babe2e |
+ instance.
|
|
|
babe2e |
+
|
|
|
babe2e |
+ Note that you don't have to configure the passed instance in any way;
|
|
|
babe2e |
+ urlgrabber will do all the necessary work.
|
|
|
babe2e |
+
|
|
|
babe2e |
+ This option exists primarily to allow using urlgrabber from multiple
|
|
|
babe2e |
+ threads in your application, in which case you would want to instantiate a
|
|
|
babe2e |
+ fresh Curl object for each thread, to avoid race conditions. See the curl
|
|
|
babe2e |
+ documentation on thread safety for more information:
|
|
|
babe2e |
+ https://curl.haxx.se/libcurl/c/threadsafe.html
|
|
|
babe2e |
+
|
|
|
babe2e |
+ Note that connection reuse (keepalive=1) is limited to the Curl instance it
|
|
|
babe2e |
+ was enabled on so if you're using multiple instances in your application,
|
|
|
babe2e |
+ connections won't be shared among them.
|
|
|
babe2e |
+
|
|
|
babe2e |
text = None
|
|
|
babe2e |
|
|
|
babe2e |
specifies alternative text to be passed to the progress meter
|
|
|
babe2e |
@@ -971,6 +989,7 @@ class URLGrabberOptions:
|
|
|
babe2e |
"""
|
|
|
babe2e |
self.progress_obj = None
|
|
|
babe2e |
self.multi_progress_obj = None
|
|
|
babe2e |
+ self.curl_obj = None
|
|
|
babe2e |
self.throttle = 1.0
|
|
|
babe2e |
self.bandwidth = 0
|
|
|
babe2e |
self.retry = None
|
|
|
babe2e |
@@ -1622,7 +1641,10 @@ class PyCurlFileObject(object):
|
|
|
babe2e |
raise err
|
|
|
babe2e |
|
|
|
babe2e |
def _do_open(self):
|
|
|
babe2e |
- self.curl_obj = _curl_cache
|
|
|
babe2e |
+ if hasattr(self.opts, 'curl_obj') and self.opts.curl_obj is not None:
|
|
|
babe2e |
+ self.curl_obj = self.opts.curl_obj
|
|
|
babe2e |
+ else:
|
|
|
babe2e |
+ self.curl_obj = _curl_cache
|
|
|
babe2e |
self.curl_obj.reset() # reset all old settings away, just in case
|
|
|
babe2e |
# setup any ranges
|
|
|
babe2e |
self._set_opts()
|