From 0d499852b3264353e4fd8afb6c0bd700839d3fb7 Mon Sep 17 00:00:00 2001
From: Kotresh HR <khiremat@redhat.com>
Date: Mon, 20 Mar 2017 05:21:59 -0400
Subject: [PATCH 378/393] geo-rep: Retry on EBUSY

Do not crash on EBUSY error. Add EBUSY
retry errno list. Crash only if the error
persists even after max retries.

> BUG: 1434018
> Signed-off-by: Kotresh HR <khiremat@redhat.com>
> Reviewed-on: https://review.gluster.org/16924
> Smoke: Gluster Build System <jenkins@build.gluster.org>
> NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
> Reviewed-by: Aravinda VK <avishwan@redhat.com>
> CentOS-regression: Gluster Build System <jenkins@build.gluster.org>

Change-Id: Ia067ccc6547731f28f2a315d400705e616cbf662
BUG: 1427870
Signed-off-by: Kotresh HR <khiremat@redhat.com>
Reviewed-on: https://code.engineering.redhat.com/gerrit/103382
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>
---
 geo-replication/syncdaemon/resource.py   | 17 +++++++++--------
 geo-replication/syncdaemon/syncdutils.py |  4 ++--
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/geo-replication/syncdaemon/resource.py b/geo-replication/syncdaemon/resource.py
index e81ddfe..6397cf3 100644
--- a/geo-replication/syncdaemon/resource.py
+++ b/geo-replication/syncdaemon/resource.py
@@ -618,11 +618,12 @@ class Server(object):
             if not matching_disk_gfid(gfid, entry):
                 return
 
-            er = errno_wrap(os.unlink, [entry], [ENOENT, ESTALE, EISDIR])
+            er = errno_wrap(os.unlink, [entry], [ENOENT, ESTALE, EISDIR],
+                            [EBUSY])
             if isinstance(er, int):
                 if er == EISDIR:
                     er = errno_wrap(os.rmdir, [entry], [ENOENT, ESTALE,
-                                                        ENOTEMPTY])
+                                                        ENOTEMPTY], [EBUSY])
                     if er == ENOTEMPTY:
                         return er
 
@@ -677,7 +678,7 @@ class Server(object):
                 if not matching_disk_gfid(gfid, entry):
                     return
                 er = errno_wrap(os.remove, [fullname], [ENOENT, ESTALE,
-                                                        EISDIR])
+                                                        EISDIR], [EBUSY])
 
                 if er == EISDIR:
                     recursive_rmdir(gfid, entry, fullname)
@@ -685,7 +686,7 @@ class Server(object):
             if not matching_disk_gfid(gfid, entry):
                 return
 
-            errno_wrap(os.rmdir, [path], [ENOENT, ESTALE])
+            errno_wrap(os.rmdir, [path], [ENOENT, ESTALE], [EBUSY])
 
         def rename_with_disk_gfid_confirmation(gfid, entry, en):
             if not matching_disk_gfid(gfid, entry):
@@ -698,7 +699,7 @@ class Server(object):
 
             cmd_ret = errno_wrap(os.rename,
                                  [entry, en],
-                                 [ENOENT, EEXIST], [ESTALE])
+                                 [ENOENT, EEXIST], [ESTALE, EBUSY])
             collect_failure(e, cmd_ret)
 
 
@@ -788,12 +789,12 @@ class Server(object):
                             # we have a hard link, we can now unlink source
                             try:
                                 errno_wrap(os.unlink, [entry],
-                                           [ENOENT, ESTALE])
+                                           [ENOENT, ESTALE], [EBUSY])
                             except OSError as e:
                                 if e.errno == EISDIR:
                                     try:
                                         errno_wrap(os.rmdir, [entry],
-                                                   [ENOENT, ESTALE])
+                                                   [ENOENT, ESTALE], [EBUSY])
                                     except OSError as e:
                                         if e.errno == ENOTEMPTY:
                                             logging.error(
@@ -811,7 +812,7 @@ class Server(object):
                 cmd_ret = errno_wrap(Xattr.lsetxattr,
                                      [pg, 'glusterfs.gfid.newfile', blob],
                                      [EEXIST, ENOENT],
-                                     [ESTALE, EINVAL])
+                                     [ESTALE, EINVAL, EBUSY])
                 failed = collect_failure(e, cmd_ret)
 
                 # If directory creation is failed, return immediately before
diff --git a/geo-replication/syncdaemon/syncdutils.py b/geo-replication/syncdaemon/syncdutils.py
index b8e1e6a..e85857f 100644
--- a/geo-replication/syncdaemon/syncdutils.py
+++ b/geo-replication/syncdaemon/syncdutils.py
@@ -504,12 +504,12 @@ def errno_wrap(call, arg=[], errnos=[], retry_errnos=[]):
                 # probably a screwed state, cannot do much...
                 logging.warn('reached maximum retries (%s)...%s' %
                              (repr(arg), ex))
-                return ex.errno
+                raise
             time.sleep(0.250)  # retry the call
 
 
 def lstat(e):
-    return errno_wrap(os.lstat, [e], [ENOENT], [ESTALE])
+    return errno_wrap(os.lstat, [e], [ENOENT], [ESTALE, EBUSY])
 
 class NoPurgeTimeAvailable(Exception):
     pass
-- 
1.8.3.1