Blame SOURCES/0006-Issue-51102-RFE-ds-replcheck-make-online-timeout-con.patch

5873fa
From c0cb15445c1434b3d317b1c06ab1a0ba8dbc6f04 Mon Sep 17 00:00:00 2001
5873fa
From: Mark Reynolds <mreynolds@redhat.com>
5873fa
Date: Tue, 19 May 2020 15:11:53 -0400
5873fa
Subject: [PATCH 06/12] Issue 51102 - RFE - ds-replcheck - make online timeout
5873fa
 configurable
5873fa
5873fa
Bug Description:  When doing an online check with replicas that are very
5873fa
                  far apart the connection can time out as the hardcoded
5873fa
                  timeout is 5 seconds.
5873fa
5873fa
Fix Description:  Change the default timeout to never timeout, and add an
5873fa
                  CLI option to specify a specific timeout.
5873fa
5873fa
                  Also caught all the possible LDAP exceptions so we can
5873fa
                  cleanly "fail".  Fixed some python syntax issues, and
5873fa
                  improved the entry inconsistency report
5873fa
5873fa
relates: https://pagure.io/389-ds-base/issue/51102
5873fa
5873fa
Reviewed by: firstyear & spichugi(Thanks!)
5873fa
---
5873fa
 ldap/admin/src/scripts/ds-replcheck | 90 ++++++++++++++++++-----------
5873fa
 1 file changed, 57 insertions(+), 33 deletions(-)
5873fa
5873fa
diff --git a/ldap/admin/src/scripts/ds-replcheck b/ldap/admin/src/scripts/ds-replcheck
5873fa
index 30bcfd65d..5bb7dfce3 100755
5873fa
--- a/ldap/admin/src/scripts/ds-replcheck
5873fa
+++ b/ldap/admin/src/scripts/ds-replcheck
5873fa
@@ -1,7 +1,7 @@
5873fa
 #!/usr/bin/python3
5873fa
 
5873fa
 # --- BEGIN COPYRIGHT BLOCK ---
5873fa
-# Copyright (C) 2018 Red Hat, Inc.
5873fa
+# Copyright (C) 2020 Red Hat, Inc.
5873fa
 # All rights reserved.
5873fa
 #
5873fa
 # License: GPL (version 3 or any later version).
5873fa
@@ -21,10 +21,9 @@ import getpass
5873fa
 import signal
5873fa
 from ldif import LDIFRecordList
5873fa
 from ldap.ldapobject import SimpleLDAPObject
5873fa
-from ldap.cidict import cidict
5873fa
 from ldap.controls import SimplePagedResultsControl
5873fa
 from lib389._entry import Entry
5873fa
-from lib389.utils import ensure_str, ensure_list_str, ensure_int
5873fa
+from lib389.utils import ensure_list_str, ensure_int
5873fa
 
5873fa
 VERSION = "2.0"
5873fa
 RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))'
5873fa
@@ -185,11 +184,11 @@ def report_conflict(entry, attr, opts):
5873fa
     report = True
5873fa
 
5873fa
     if 'nscpentrywsi' in entry.data:
5873fa
-        found = False
5873fa
         for val in entry.data['nscpentrywsi']:
5873fa
             if val.lower().startswith(attr + ';'):
5873fa
                 if (opts['starttime'] - extract_time(val)) <= opts['lag']:
5873fa
                     report = False
5873fa
+                    break
5873fa
 
5873fa
     return report
5873fa
 
5873fa
@@ -321,6 +320,9 @@ def ldif_search(LDIF, dn):
5873fa
     count = 0
5873fa
     ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname']
5873fa
     val = ""
5873fa
+    attr = ""
5873fa
+    state_attr = ""
5873fa
+    part_dn = ""
5873fa
     result['entry'] = None
5873fa
     result['conflict'] = None
5873fa
     result['tombstone'] = False
5873fa
@@ -570,6 +572,7 @@ def cmp_entry(mentry, rentry, opts):
5873fa
                         if val.lower().startswith(mattr + ';'):
5873fa
                             if not found:
5873fa
                                 diff['diff'].append("      Master:")
5873fa
+                            diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))
5873fa
                             diff['diff'].append("        - State Info: %s" % (val))
5873fa
                             diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))
5873fa
                             found = True
5873fa
@@ -588,6 +591,7 @@ def cmp_entry(mentry, rentry, opts):
5873fa
                         if val.lower().startswith(mattr + ';'):
5873fa
                             if not found:
5873fa
                                 diff['diff'].append("      Replica:")
5873fa
+                            diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))
5873fa
                             diff['diff'].append("        - State Info: %s" % (val))
5873fa
                             diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))
5873fa
                             found = True
5873fa
@@ -654,7 +658,6 @@ def do_offline_report(opts, output_file=None):
5873fa
     rconflicts = []
5873fa
     rtombstones = 0
5873fa
     mtombstones = 0
5873fa
-    idx = 0
5873fa
 
5873fa
     # Open LDIF files
5873fa
     try:
5873fa
@@ -926,7 +929,7 @@ def validate_suffix(ldapnode, suffix, hostname):
5873fa
     :return - True if suffix exists, otherwise False
5873fa
     """
5873fa
     try:
5873fa
-        master_basesuffix = ldapnode.search_s(suffix, ldap.SCOPE_BASE )
5873fa
+        ldapnode.search_s(suffix, ldap.SCOPE_BASE)
5873fa
     except ldap.NO_SUCH_OBJECT:
5873fa
         print("Error: Failed to validate suffix in {}. {} does not exist.".format(hostname, suffix))
5873fa
         return False
5873fa
@@ -968,12 +971,12 @@ def connect_to_replicas(opts):
5873fa
     replica = SimpleLDAPObject(ruri)
5873fa
 
5873fa
     # Set timeouts
5873fa
-    master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
5873fa
-    master.set_option(ldap.OPT_TIMEOUT,5.0)
5873fa
-    replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
5873fa
-    replica.set_option(ldap.OPT_TIMEOUT,5.0)
5873fa
+    master.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
5873fa
+    master.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
5873fa
+    replica.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
5873fa
+    replica.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
5873fa
 
5873fa
-    # Setup Secure Conenction
5873fa
+    # Setup Secure Connection
5873fa
     if opts['certdir'] is not None:
5873fa
         # Setup Master
5873fa
         if opts['mprotocol'] != LDAPI:
5873fa
@@ -1003,7 +1006,7 @@ def connect_to_replicas(opts):
5873fa
     try:
5873fa
         master.simple_bind_s(opts['binddn'], opts['bindpw'])
5873fa
     except ldap.SERVER_DOWN as e:
5873fa
-        print("Cannot connect to %r" % muri)
5873fa
+        print(f"Cannot connect to {muri} ({str(e)})")
5873fa
         sys.exit(1)
5873fa
     except ldap.LDAPError as e:
5873fa
         print("Error: Failed to authenticate to Master: ({}).  "
5873fa
@@ -1014,7 +1017,7 @@ def connect_to_replicas(opts):
5873fa
     try:
5873fa
         replica.simple_bind_s(opts['binddn'], opts['bindpw'])
5873fa
     except ldap.SERVER_DOWN as e:
5873fa
-        print("Cannot connect to %r" % ruri)
5873fa
+        print(f"Cannot connect to {ruri} ({str(e)})")
5873fa
         sys.exit(1)
5873fa
     except ldap.LDAPError as e:
5873fa
         print("Error: Failed to authenticate to Replica: ({}).  "
5873fa
@@ -1218,7 +1221,6 @@ def do_online_report(opts, output_file=None):
5873fa
     """
5873fa
     m_done = False
5873fa
     r_done = False
5873fa
-    done = False
5873fa
     report = {}
5873fa
     report['diff'] = []
5873fa
     report['m_missing'] = []
5873fa
@@ -1257,15 +1259,22 @@ def do_online_report(opts, output_file=None):
5873fa
 
5873fa
     # Read the results and start comparing
5873fa
     while not m_done or not r_done:
5873fa
-        if not m_done:
5873fa
-            m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
5873fa
-        elif not r_done:
5873fa
-            m_rdata = []
5873fa
-
5873fa
-        if not r_done:
5873fa
-            r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
5873fa
-        elif not m_done:
5873fa
-            r_rdata = []
5873fa
+        try:
5873fa
+            if not m_done:
5873fa
+                m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
5873fa
+            elif not r_done:
5873fa
+                m_rdata = []
5873fa
+        except ldap.LDAPError as e:
5873fa
+            print("Error: Problem getting the results from the master: %s", str(e))
5873fa
+            sys.exit(1)
5873fa
+        try:
5873fa
+            if not r_done:
5873fa
+                r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
5873fa
+            elif not m_done:
5873fa
+                r_rdata = []
5873fa
+        except ldap.LDAPError as e:
5873fa
+            print("Error: Problem getting the results from the replica: %s", str(e))
5873fa
+            sys.exit(1)
5873fa
 
5873fa
         # Convert entries
5873fa
         mresult = convert_entries(m_rdata)
5873fa
@@ -1291,11 +1300,15 @@ def do_online_report(opts, output_file=None):
5873fa
                 ]
5873fa
             if m_pctrls:
5873fa
                 if m_pctrls[0].cookie:
5873fa
-                    # Copy cookie from response control to request control
5873fa
-                    req_pr_ctrl.cookie = m_pctrls[0].cookie
5873fa
-                    master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
5873fa
-                        "(|(objectclass=*)(objectclass=ldapsubentry))",
5873fa
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
5873fa
+                    try:
5873fa
+                        # Copy cookie from response control to request control
5873fa
+                        req_pr_ctrl.cookie = m_pctrls[0].cookie
5873fa
+                        master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
5873fa
+                            "(|(objectclass=*)(objectclass=ldapsubentry))",
5873fa
+                            ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
5873fa
+                    except ldap.LDAPError as e:
5873fa
+                        print("Error: Problem searching the master: %s", str(e))
5873fa
+                        sys.exit(1)
5873fa
                 else:
5873fa
                     m_done = True  # No more pages available
5873fa
             else:
5873fa
@@ -1311,11 +1324,15 @@ def do_online_report(opts, output_file=None):
5873fa
 
5873fa
             if r_pctrls:
5873fa
                 if r_pctrls[0].cookie:
5873fa
-                    # Copy cookie from response control to request control
5873fa
-                    req_pr_ctrl.cookie = r_pctrls[0].cookie
5873fa
-                    replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
5873fa
-                        "(|(objectclass=*)(objectclass=ldapsubentry))",
5873fa
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
5873fa
+                    try:
5873fa
+                        # Copy cookie from response control to request control
5873fa
+                        req_pr_ctrl.cookie = r_pctrls[0].cookie
5873fa
+                        replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
5873fa
+                            "(|(objectclass=*)(objectclass=ldapsubentry))",
5873fa
+                            ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
5873fa
+                    except ldap.LDAPError as e:
5873fa
+                        print("Error: Problem searching the replica: %s", str(e))
5873fa
+                        sys.exit(1)
5873fa
                 else:
5873fa
                     r_done = True  # No more pages available
5873fa
             else:
5873fa
@@ -1426,6 +1443,9 @@ def init_online_params(args):
5873fa
         # prompt for password
5873fa
         opts['bindpw'] = getpass.getpass('Enter password: ')
5873fa
 
5873fa
+    # lastly handle the timeout
5873fa
+    opts['timeout'] = int(args.timeout)
5873fa
+
5873fa
     return opts
5873fa
 
5873fa
 
5873fa
@@ -1553,6 +1573,8 @@ def main():
5873fa
     state_parser.add_argument('-y', '--pass-file', help='A text file containing the clear text password for the bind dn', dest='pass_file', default=None)
5873fa
     state_parser.add_argument('-Z', '--cert-dir', help='The certificate database directory for secure connections',
5873fa
                               dest='certdir', default=None)
5873fa
+    state_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',
5873fa
+                              type=int, dest='timeout', default=-1)
5873fa
 
5873fa
     # Online mode
5873fa
     online_parser = subparsers.add_parser('online', help="Compare two online replicas for differences")
5873fa
@@ -1577,6 +1599,8 @@ def main():
5873fa
     online_parser.add_argument('-p', '--page-size', help='The paged-search result grouping size (default 500 entries)',
5873fa
                                dest='pagesize', default=500)
5873fa
     online_parser.add_argument('-o', '--out-file', help='The output file', dest='file', default=None)
5873fa
+    online_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',
5873fa
+                               type=int, dest='timeout', default=-1)
5873fa
 
5873fa
     # Offline LDIF mode
5873fa
     offline_parser = subparsers.add_parser('offline', help="Compare two replication LDIF files for differences (LDIF file generated by 'db2ldif -r')")
5873fa
-- 
5873fa
2.26.2
5873fa