Blob Blame Raw
From c0cb15445c1434b3d317b1c06ab1a0ba8dbc6f04 Mon Sep 17 00:00:00 2001
From: Mark Reynolds <mreynolds@redhat.com>
Date: Tue, 19 May 2020 15:11:53 -0400
Subject: [PATCH 06/12] Issue 51102 - RFE - ds-replcheck - make online timeout
 configurable

Bug Description:  When doing an online check with replicas that are very
                  far apart the connection can time out as the hardcoded
                  timeout is 5 seconds.

Fix Description:  Change the default timeout to never timeout, and add an
                  CLI option to specify a specific timeout.

                  Also caught all the possible LDAP exceptions so we can
                  cleanly "fail".  Fixed some python syntax issues, and
                  improved the entry inconsistency report

relates: https://pagure.io/389-ds-base/issue/51102

Reviewed by: firstyear & spichugi(Thanks!)
---
 ldap/admin/src/scripts/ds-replcheck | 90 ++++++++++++++++++-----------
 1 file changed, 57 insertions(+), 33 deletions(-)

diff --git a/ldap/admin/src/scripts/ds-replcheck b/ldap/admin/src/scripts/ds-replcheck
index 30bcfd65d..5bb7dfce3 100755
--- a/ldap/admin/src/scripts/ds-replcheck
+++ b/ldap/admin/src/scripts/ds-replcheck
@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 # --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2018 Red Hat, Inc.
+# Copyright (C) 2020 Red Hat, Inc.
 # All rights reserved.
 #
 # License: GPL (version 3 or any later version).
@@ -21,10 +21,9 @@ import getpass
 import signal
 from ldif import LDIFRecordList
 from ldap.ldapobject import SimpleLDAPObject
-from ldap.cidict import cidict
 from ldap.controls import SimplePagedResultsControl
 from lib389._entry import Entry
-from lib389.utils import ensure_str, ensure_list_str, ensure_int
+from lib389.utils import ensure_list_str, ensure_int
 
 VERSION = "2.0"
 RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))'
@@ -185,11 +184,11 @@ def report_conflict(entry, attr, opts):
     report = True
 
     if 'nscpentrywsi' in entry.data:
-        found = False
         for val in entry.data['nscpentrywsi']:
             if val.lower().startswith(attr + ';'):
                 if (opts['starttime'] - extract_time(val)) <= opts['lag']:
                     report = False
+                    break
 
     return report
 
@@ -321,6 +320,9 @@ def ldif_search(LDIF, dn):
     count = 0
     ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname']
     val = ""
+    attr = ""
+    state_attr = ""
+    part_dn = ""
     result['entry'] = None
     result['conflict'] = None
     result['tombstone'] = False
@@ -570,6 +572,7 @@ def cmp_entry(mentry, rentry, opts):
                         if val.lower().startswith(mattr + ';'):
                             if not found:
                                 diff['diff'].append("      Master:")
+                            diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))
                             diff['diff'].append("        - State Info: %s" % (val))
                             diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))
                             found = True
@@ -588,6 +591,7 @@ def cmp_entry(mentry, rentry, opts):
                         if val.lower().startswith(mattr + ';'):
                             if not found:
                                 diff['diff'].append("      Replica:")
+                            diff['diff'].append("        - Value:      %s" % (val.split(':')[1].lstrip()))
                             diff['diff'].append("        - State Info: %s" % (val))
                             diff['diff'].append("        - Date:       %s\n" % (time.ctime(extract_time(val))))
                             found = True
@@ -654,7 +658,6 @@ def do_offline_report(opts, output_file=None):
     rconflicts = []
     rtombstones = 0
     mtombstones = 0
-    idx = 0
 
     # Open LDIF files
     try:
@@ -926,7 +929,7 @@ def validate_suffix(ldapnode, suffix, hostname):
     :return - True if suffix exists, otherwise False
     """
     try:
-        master_basesuffix = ldapnode.search_s(suffix, ldap.SCOPE_BASE )
+        ldapnode.search_s(suffix, ldap.SCOPE_BASE)
     except ldap.NO_SUCH_OBJECT:
         print("Error: Failed to validate suffix in {}. {} does not exist.".format(hostname, suffix))
         return False
@@ -968,12 +971,12 @@ def connect_to_replicas(opts):
     replica = SimpleLDAPObject(ruri)
 
     # Set timeouts
-    master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
-    master.set_option(ldap.OPT_TIMEOUT,5.0)
-    replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0)
-    replica.set_option(ldap.OPT_TIMEOUT,5.0)
+    master.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
+    master.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
+    replica.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout'])
+    replica.set_option(ldap.OPT_TIMEOUT, opts['timeout'])
 
-    # Setup Secure Conenction
+    # Setup Secure Connection
     if opts['certdir'] is not None:
         # Setup Master
         if opts['mprotocol'] != LDAPI:
@@ -1003,7 +1006,7 @@ def connect_to_replicas(opts):
     try:
         master.simple_bind_s(opts['binddn'], opts['bindpw'])
     except ldap.SERVER_DOWN as e:
-        print("Cannot connect to %r" % muri)
+        print(f"Cannot connect to {muri} ({str(e)})")
         sys.exit(1)
     except ldap.LDAPError as e:
         print("Error: Failed to authenticate to Master: ({}).  "
@@ -1014,7 +1017,7 @@ def connect_to_replicas(opts):
     try:
         replica.simple_bind_s(opts['binddn'], opts['bindpw'])
     except ldap.SERVER_DOWN as e:
-        print("Cannot connect to %r" % ruri)
+        print(f"Cannot connect to {ruri} ({str(e)})")
         sys.exit(1)
     except ldap.LDAPError as e:
         print("Error: Failed to authenticate to Replica: ({}).  "
@@ -1218,7 +1221,6 @@ def do_online_report(opts, output_file=None):
     """
     m_done = False
     r_done = False
-    done = False
     report = {}
     report['diff'] = []
     report['m_missing'] = []
@@ -1257,15 +1259,22 @@ def do_online_report(opts, output_file=None):
 
     # Read the results and start comparing
     while not m_done or not r_done:
-        if not m_done:
-            m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
-        elif not r_done:
-            m_rdata = []
-
-        if not r_done:
-            r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
-        elif not m_done:
-            r_rdata = []
+        try:
+            if not m_done:
+                m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid)
+            elif not r_done:
+                m_rdata = []
+        except ldap.LDAPError as e:
+            print("Error: Problem getting the results from the master: %s", str(e))
+            sys.exit(1)
+        try:
+            if not r_done:
+                r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid)
+            elif not m_done:
+                r_rdata = []
+        except ldap.LDAPError as e:
+            print("Error: Problem getting the results from the replica: %s", str(e))
+            sys.exit(1)
 
         # Convert entries
         mresult = convert_entries(m_rdata)
@@ -1291,11 +1300,15 @@ def do_online_report(opts, output_file=None):
                 ]
             if m_pctrls:
                 if m_pctrls[0].cookie:
-                    # Copy cookie from response control to request control
-                    req_pr_ctrl.cookie = m_pctrls[0].cookie
-                    master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
-                        "(|(objectclass=*)(objectclass=ldapsubentry))",
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    try:
+                        # Copy cookie from response control to request control
+                        req_pr_ctrl.cookie = m_pctrls[0].cookie
+                        master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
+                            "(|(objectclass=*)(objectclass=ldapsubentry))",
+                            ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    except ldap.LDAPError as e:
+                        print("Error: Problem searching the master: %s", str(e))
+                        sys.exit(1)
                 else:
                     m_done = True  # No more pages available
             else:
@@ -1311,11 +1324,15 @@ def do_online_report(opts, output_file=None):
 
             if r_pctrls:
                 if r_pctrls[0].cookie:
-                    # Copy cookie from response control to request control
-                    req_pr_ctrl.cookie = r_pctrls[0].cookie
-                    replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
-                        "(|(objectclass=*)(objectclass=ldapsubentry))",
-                        ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    try:
+                        # Copy cookie from response control to request control
+                        req_pr_ctrl.cookie = r_pctrls[0].cookie
+                        replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE,
+                            "(|(objectclass=*)(objectclass=ldapsubentry))",
+                            ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls)
+                    except ldap.LDAPError as e:
+                        print("Error: Problem searching the replica: %s", str(e))
+                        sys.exit(1)
                 else:
                     r_done = True  # No more pages available
             else:
@@ -1426,6 +1443,9 @@ def init_online_params(args):
         # prompt for password
         opts['bindpw'] = getpass.getpass('Enter password: ')
 
+    # lastly handle the timeout
+    opts['timeout'] = int(args.timeout)
+
     return opts
 
 
@@ -1553,6 +1573,8 @@ def main():
     state_parser.add_argument('-y', '--pass-file', help='A text file containing the clear text password for the bind dn', dest='pass_file', default=None)
     state_parser.add_argument('-Z', '--cert-dir', help='The certificate database directory for secure connections',
                               dest='certdir', default=None)
+    state_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',
+                              type=int, dest='timeout', default=-1)
 
     # Online mode
     online_parser = subparsers.add_parser('online', help="Compare two online replicas for differences")
@@ -1577,6 +1599,8 @@ def main():
     online_parser.add_argument('-p', '--page-size', help='The paged-search result grouping size (default 500 entries)',
                                dest='pagesize', default=500)
     online_parser.add_argument('-o', '--out-file', help='The output file', dest='file', default=None)
+    online_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections.  Default is no timeout.',
+                               type=int, dest='timeout', default=-1)
 
     # Offline LDIF mode
     offline_parser = subparsers.add_parser('offline', help="Compare two replication LDIF files for differences (LDIF file generated by 'db2ldif -r')")
-- 
2.26.2