diff -up urlgrabber-3.10/test/test_mirror.py.orig urlgrabber-3.10/test/test_mirror.py

--- urlgrabber-3.10/test/test_mirror.py.orig	2013-08-26 09:09:07.000000000 +0200

+++ urlgrabber-3.10/test/test_mirror.py	2016-06-29 18:26:06.790393129 +0200

@@ -268,33 +268,55 @@ class ActionTests(TestCase):

         self.assertEquals(self.g.calls, expected_calls)

         self.assertEquals(urlgrabber.mirror.DEBUG.logs, expected_logs)

+import thread, socket

+LOCALPORT = 'localhost', 2000

 class HttpReplyCode(TestCase):

     def setUp(self):

+        # start the server

+        self.exit = False

         def server():

-            import socket

             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

             s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

-            s.bind(('localhost', 2000)); s.listen(1)

+            s.bind(LOCALPORT); s.listen(1)

             while 1:

                 c, a = s.accept()

+                if self.exit: c.close(); break

                 while not c.recv(4096).endswith('\r\n\r\n'): pass

                 c.sendall('HTTP/1.1 %d %s\r\n' % self.reply)

+                if self.content is not None:

+                    c.sendall('Content-Length: %d\r\n\r\n' % len(self.content))

+                    c.sendall(self.content)

                 c.close()

-        import thread

-        self.reply = 503, "Busy"

+            s.close()

+            self.exit = False

         thread.start_new_thread(server, ())

+        # create grabber and mirror group objects

         def failure(obj):

             self.code = getattr(obj.exception, 'code', None)

             return {}

         self.g  = URLGrabber()

-        self.mg = MirrorGroup(self.g, ['http://localhost:2000/'], failure_callback = failure)

+        self.mg = MirrorGroup(self.g, ['http://%s:%d' % LOCALPORT],

+                              failure_callback = failure)

+

+    def tearDown(self):

+        # shut down the server

+        self.exit = True

+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

+        s.connect(LOCALPORT); s.close() # wake it up

+        while self.exit: pass # poor man's join

     def test_grab(self):

+        'tests the propagation of HTTP reply code'

+        self.reply = 503, "Busy"

+        self.content = None

+

+        # single

         self.assertRaises(URLGrabError, self.mg.urlgrab, 'foo')

         self.assertEquals(self.code, 503); del self.code

+        # multi

         err = []

         self.mg.urlgrab('foo', async = True, failfunc = err.append)

         urlgrabber.grabber.parallel_wait()

diff -up urlgrabber-3.10/test/test_mirror.py.orig urlgrabber-3.10/test/test_mirror.py

--- urlgrabber-3.10/test/test_mirror.py.orig	2016-06-29 18:26:06.790393129 +0200

+++ urlgrabber-3.10/test/test_mirror.py	2016-06-29 18:26:58.886148544 +0200

@@ -268,13 +268,14 @@ class ActionTests(TestCase):

         self.assertEquals(self.g.calls, expected_calls)

         self.assertEquals(urlgrabber.mirror.DEBUG.logs, expected_logs)

-import thread, socket

+import threading, socket

 LOCALPORT = 'localhost', 2000

 class HttpReplyCode(TestCase):

     def setUp(self):

         # start the server

         self.exit = False

+        self.process = lambda data: None

         def server():

             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

             s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

@@ -282,7 +283,10 @@ class HttpReplyCode(TestCase):

             while 1:

                 c, a = s.accept()

                 if self.exit: c.close(); break

-                while not c.recv(4096).endswith('\r\n\r\n'): pass

+                data = ''

+                while not data.endswith('\r\n\r\n'):

+                    data = c.recv(4096)

+                self.process(data)

                 c.sendall('HTTP/1.1 %d %s\r\n' % self.reply)

                 if self.content is not None:

                     c.sendall('Content-Length: %d\r\n\r\n' % len(self.content))

@@ -290,7 +294,8 @@ class HttpReplyCode(TestCase):

                 c.close()

             s.close()

             self.exit = False

-        thread.start_new_thread(server, ())

+        self.thread = threading.Thread(target=server)

+        self.thread.start()

         # create grabber and mirror group objects

         def failure(obj):

@@ -305,7 +310,7 @@ class HttpReplyCode(TestCase):

         self.exit = True

         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

         s.connect(LOCALPORT); s.close() # wake it up

-        while self.exit: pass # poor man's join

+        self.thread.join()

     def test_grab(self):

         'tests the propagation of HTTP reply code'

@@ -323,6 +328,45 @@ class HttpReplyCode(TestCase):

         self.assertEquals([e.exception.errno for e in err], [256])

         self.assertEquals(self.code, 503); del self.code

+    def test_retry_no_cache(self):

+        'test bypassing proxy cache on failure'

+        def process(data):

+            if 'Pragma:no-cache' in data:

+                self.content = 'version2'

+            else:

+                self.content = 'version1'

+

+        def checkfunc_read(obj):

+            if obj.data == 'version1':

+                raise URLGrabError(-1, 'Outdated version of foo')

+

+        def checkfunc_grab(obj):

+            with open('foo') as f:

+                if f.read() == 'version1':

+                    raise URLGrabError(-1, 'Outdated version of foo')

+

+        self.process = process

+        self.reply = 200, "OK"

+

+        opts = self.g.opts

+        opts.retry = 3

+        opts.retry_no_cache = True

+

+        # single

+        opts.checkfunc = checkfunc_read

+        try:

+            self.mg.urlread('foo')

+        except URLGrabError as e:

+            self.fail(str(e))

+

+        # multi

+        opts.checkfunc = checkfunc_grab

+        self.mg.urlgrab('foo', async=True)

+        try:

+            urlgrabber.grabber.parallel_wait()

+        except URLGrabError as e:

+            self.fail(str(e))

+

 def suite():

     tl = TestLoader()

     return tl.loadTestsFromModule(sys.modules[__name__])

diff -up urlgrabber-3.10/urlgrabber/grabber.py.orig urlgrabber-3.10/urlgrabber/grabber.py

--- urlgrabber-3.10/urlgrabber/grabber.py.orig	2016-06-29 18:25:53.964453346 +0200

+++ urlgrabber-3.10/urlgrabber/grabber.py	2016-06-29 18:26:58.886148544 +0200

@@ -171,6 +171,12 @@ GENERAL ARGUMENTS (kwargs)

     The libproxy code is only used if the proxies dictionary

     does not provide any proxies.

+  no_cache = False

+

+    When True, server-side cache will be disabled for http and https

+    requests.  This is equivalent to setting

+      http_headers = (('Pragma', 'no-cache'),)

+

   prefix = None

     a url prefix that will be prepended to all requested urls.  For

@@ -383,10 +389,11 @@ RETRY RELATED ARGUMENTS

     identical to checkfunc, except for the attributes defined in the

     CallbackObject instance.  The attributes for failure_callback are:

-      exception = the raised exception

-      url       = the url we're trying to fetch

-      tries     = the number of tries so far (including this one)

-      retry     = the value of the retry option

+      exception      = the raised exception

+      url            = the url we're trying to fetch

+      tries          = the number of tries so far (including this one)

+      retry          = the value of the retry option

+      retry_no_cache = the value of the retry_no_cache option

     The callback is present primarily to inform the calling program of

     the failure, but if it raises an exception (including the one it's

@@ -431,6 +438,19 @@ RETRY RELATED ARGUMENTS

     passed the same arguments, so you could use the same function for

     both.

+  retry_no_cache = False

+

+    When True, automatically enable no_cache for future retries if

+    checkfunc performs an unsuccessful check.

+

+    This option is useful if your application expects a set of files

+    from the same server to form an atomic unit and you write your

+    checkfunc to ensure each file being downloaded belongs to such a

+    unit.  If transparent proxy caching is in effect, the files can

+    become out-of-sync, disrupting the atomicity.  Enabling this option

+    will prevent that, while ensuring that you still enjoy the benefits

+    of caching when possible.

+

 BANDWIDTH THROTTLING

   urlgrabber supports throttling via two values: throttle and

@@ -1001,6 +1021,8 @@ class URLGrabberOptions:

         self.half_life = 30*24*60*60 # 30 days

         self.default_speed = 1e6 # 1 MBit

         self.ftp_disable_epsv = False

+        self.no_cache = False

+        self.retry_no_cache = False

     def __repr__(self):

         return self.format()

@@ -1077,7 +1099,8 @@ class URLGrabber(object):

             if callback:

                 if DEBUG: DEBUG.info('calling callback: %s', callback)

                 obj = CallbackObject(exception=exception, url=args[0],

-                                     tries=tries, retry=opts.retry)

+                                     tries=tries, retry=opts.retry,

+                                     retry_no_cache=opts.retry_no_cache)

                 _run_callback(callback, obj)

             if (opts.retry is None) or (tries == opts.retry):

@@ -1089,6 +1112,8 @@ class URLGrabber(object):

                 if DEBUG: DEBUG.info('retrycode (%i) not in list %s, re-raising',

                                      retrycode, opts.retrycodes)

                 raise

+            if retrycode is not None and retrycode < 0 and opts.retry_no_cache:

+                opts.no_cache = True

     def urlopen(self, url, opts=None, **kwargs):

         """open the url and return a file object

@@ -1429,11 +1454,15 @@ class PyCurlFileObject(object):

                 self.curl_obj.setopt(pycurl.SSLKEYPASSWD, opts.ssl_key_pass)

         #headers:

-        if opts.http_headers and self.scheme in ('http', 'https'):

+        if self.scheme in ('http', 'https'):

             headers = []

-            for (tag, content) in opts.http_headers:

-                headers.append('%s:%s' % (tag, content))

-            self.curl_obj.setopt(pycurl.HTTPHEADER, headers)

+            if opts.http_headers is not None:

+                for (tag, content) in opts.http_headers:

+                    headers.append('%s:%s' % (tag, content))

+            if opts.no_cache:

+                headers.append('Pragma:no-cache')

+            if headers:

+                self.curl_obj.setopt(pycurl.HTTPHEADER, headers)

         # ranges:

         if opts.range or opts.reget:

@@ -2055,7 +2084,8 @@ class _ExternalDownloader:

         'ssl_key_pass',

         'ssl_verify_peer', 'ssl_verify_host',

         'size', 'max_header_size', 'ip_resolve',

-        'ftp_disable_epsv'

+        'ftp_disable_epsv',

+        'no_cache',

     )

     def start(self, opts):

@@ -2236,6 +2266,8 @@ def parallel_wait(meter=None):

                 except URLGrabError, ug_err:

                     retry = 0 # no retries

             if opts.tries < retry and ug_err.errno in opts.retrycodes:

+                if ug_err.errno < 0 and opts.retry_no_cache:

+                    opts.no_cache = True

                 start(opts, opts.tries + 1) # simple retry

                 continue