From c0cb15445c1434b3d317b1c06ab1a0ba8dbc6f04 Mon Sep 17 00:00:00 2001 From: Mark Reynolds Date: Tue, 19 May 2020 15:11:53 -0400 Subject: [PATCH 06/12] Issue 51102 - RFE - ds-replcheck - make online timeout configurable Bug Description: When doing an online check with replicas that are very far apart the connection can time out as the hardcoded timeout is 5 seconds. Fix Description: Change the default timeout to never timeout, and add an CLI option to specify a specific timeout. Also caught all the possible LDAP exceptions so we can cleanly "fail". Fixed some python syntax issues, and improved the entry inconsistency report relates: https://pagure.io/389-ds-base/issue/51102 Reviewed by: firstyear & spichugi(Thanks!) --- ldap/admin/src/scripts/ds-replcheck | 90 ++++++++++++++++++----------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/ldap/admin/src/scripts/ds-replcheck b/ldap/admin/src/scripts/ds-replcheck index 30bcfd65d..5bb7dfce3 100755 --- a/ldap/admin/src/scripts/ds-replcheck +++ b/ldap/admin/src/scripts/ds-replcheck @@ -1,7 +1,7 @@ #!/usr/bin/python3 # --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2018 Red Hat, Inc. +# Copyright (C) 2020 Red Hat, Inc. # All rights reserved. # # License: GPL (version 3 or any later version). @@ -21,10 +21,9 @@ import getpass import signal from ldif import LDIFRecordList from ldap.ldapobject import SimpleLDAPObject -from ldap.cidict import cidict from ldap.controls import SimplePagedResultsControl from lib389._entry import Entry -from lib389.utils import ensure_str, ensure_list_str, ensure_int +from lib389.utils import ensure_list_str, ensure_int VERSION = "2.0" RUV_FILTER = '(&(nsuniqueid=ffffffff-ffffffff-ffffffff-ffffffff)(objectclass=nstombstone))' @@ -185,11 +184,11 @@ def report_conflict(entry, attr, opts): report = True if 'nscpentrywsi' in entry.data: - found = False for val in entry.data['nscpentrywsi']: if val.lower().startswith(attr + ';'): if (opts['starttime'] - extract_time(val)) <= opts['lag']: report = False + break return report @@ -321,6 +320,9 @@ def ldif_search(LDIF, dn): count = 0 ignore_list = ['conflictcsn', 'modifytimestamp', 'modifiersname'] val = "" + attr = "" + state_attr = "" + part_dn = "" result['entry'] = None result['conflict'] = None result['tombstone'] = False @@ -570,6 +572,7 @@ def cmp_entry(mentry, rentry, opts): if val.lower().startswith(mattr + ';'): if not found: diff['diff'].append(" Master:") + diff['diff'].append(" - Value: %s" % (val.split(':')[1].lstrip())) diff['diff'].append(" - State Info: %s" % (val)) diff['diff'].append(" - Date: %s\n" % (time.ctime(extract_time(val)))) found = True @@ -588,6 +591,7 @@ def cmp_entry(mentry, rentry, opts): if val.lower().startswith(mattr + ';'): if not found: diff['diff'].append(" Replica:") + diff['diff'].append(" - Value: %s" % (val.split(':')[1].lstrip())) diff['diff'].append(" - State Info: %s" % (val)) diff['diff'].append(" - Date: %s\n" % (time.ctime(extract_time(val)))) found = True @@ -654,7 +658,6 @@ def do_offline_report(opts, output_file=None): rconflicts = [] rtombstones = 0 mtombstones = 0 - idx = 0 # Open LDIF files try: @@ -926,7 +929,7 @@ def validate_suffix(ldapnode, suffix, hostname): :return - True if suffix exists, otherwise False """ try: - master_basesuffix = ldapnode.search_s(suffix, ldap.SCOPE_BASE ) + ldapnode.search_s(suffix, ldap.SCOPE_BASE) except ldap.NO_SUCH_OBJECT: print("Error: Failed to validate suffix in {}. {} does not exist.".format(hostname, suffix)) return False @@ -968,12 +971,12 @@ def connect_to_replicas(opts): replica = SimpleLDAPObject(ruri) # Set timeouts - master.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0) - master.set_option(ldap.OPT_TIMEOUT,5.0) - replica.set_option(ldap.OPT_NETWORK_TIMEOUT,5.0) - replica.set_option(ldap.OPT_TIMEOUT,5.0) + master.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout']) + master.set_option(ldap.OPT_TIMEOUT, opts['timeout']) + replica.set_option(ldap.OPT_NETWORK_TIMEOUT, opts['timeout']) + replica.set_option(ldap.OPT_TIMEOUT, opts['timeout']) - # Setup Secure Conenction + # Setup Secure Connection if opts['certdir'] is not None: # Setup Master if opts['mprotocol'] != LDAPI: @@ -1003,7 +1006,7 @@ def connect_to_replicas(opts): try: master.simple_bind_s(opts['binddn'], opts['bindpw']) except ldap.SERVER_DOWN as e: - print("Cannot connect to %r" % muri) + print(f"Cannot connect to {muri} ({str(e)})") sys.exit(1) except ldap.LDAPError as e: print("Error: Failed to authenticate to Master: ({}). " @@ -1014,7 +1017,7 @@ def connect_to_replicas(opts): try: replica.simple_bind_s(opts['binddn'], opts['bindpw']) except ldap.SERVER_DOWN as e: - print("Cannot connect to %r" % ruri) + print(f"Cannot connect to {ruri} ({str(e)})") sys.exit(1) except ldap.LDAPError as e: print("Error: Failed to authenticate to Replica: ({}). " @@ -1218,7 +1221,6 @@ def do_online_report(opts, output_file=None): """ m_done = False r_done = False - done = False report = {} report['diff'] = [] report['m_missing'] = [] @@ -1257,15 +1259,22 @@ def do_online_report(opts, output_file=None): # Read the results and start comparing while not m_done or not r_done: - if not m_done: - m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid) - elif not r_done: - m_rdata = [] - - if not r_done: - r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid) - elif not m_done: - r_rdata = [] + try: + if not m_done: + m_rtype, m_rdata, m_rmsgid, m_rctrls = master.result3(master_msgid) + elif not r_done: + m_rdata = [] + except ldap.LDAPError as e: + print("Error: Problem getting the results from the master: %s", str(e)) + sys.exit(1) + try: + if not r_done: + r_rtype, r_rdata, r_rmsgid, r_rctrls = replica.result3(replica_msgid) + elif not m_done: + r_rdata = [] + except ldap.LDAPError as e: + print("Error: Problem getting the results from the replica: %s", str(e)) + sys.exit(1) # Convert entries mresult = convert_entries(m_rdata) @@ -1291,11 +1300,15 @@ def do_online_report(opts, output_file=None): ] if m_pctrls: if m_pctrls[0].cookie: - # Copy cookie from response control to request control - req_pr_ctrl.cookie = m_pctrls[0].cookie - master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE, - "(|(objectclass=*)(objectclass=ldapsubentry))", - ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls) + try: + # Copy cookie from response control to request control + req_pr_ctrl.cookie = m_pctrls[0].cookie + master_msgid = master.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE, + "(|(objectclass=*)(objectclass=ldapsubentry))", + ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls) + except ldap.LDAPError as e: + print("Error: Problem searching the master: %s", str(e)) + sys.exit(1) else: m_done = True # No more pages available else: @@ -1311,11 +1324,15 @@ def do_online_report(opts, output_file=None): if r_pctrls: if r_pctrls[0].cookie: - # Copy cookie from response control to request control - req_pr_ctrl.cookie = r_pctrls[0].cookie - replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE, - "(|(objectclass=*)(objectclass=ldapsubentry))", - ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls) + try: + # Copy cookie from response control to request control + req_pr_ctrl.cookie = r_pctrls[0].cookie + replica_msgid = replica.search_ext(opts['suffix'], ldap.SCOPE_SUBTREE, + "(|(objectclass=*)(objectclass=ldapsubentry))", + ['*', 'createtimestamp', 'nscpentrywsi', 'conflictcsn', 'nsds5replconflict'], serverctrls=controls) + except ldap.LDAPError as e: + print("Error: Problem searching the replica: %s", str(e)) + sys.exit(1) else: r_done = True # No more pages available else: @@ -1426,6 +1443,9 @@ def init_online_params(args): # prompt for password opts['bindpw'] = getpass.getpass('Enter password: ') + # lastly handle the timeout + opts['timeout'] = int(args.timeout) + return opts @@ -1553,6 +1573,8 @@ def main(): state_parser.add_argument('-y', '--pass-file', help='A text file containing the clear text password for the bind dn', dest='pass_file', default=None) state_parser.add_argument('-Z', '--cert-dir', help='The certificate database directory for secure connections', dest='certdir', default=None) + state_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections. Default is no timeout.', + type=int, dest='timeout', default=-1) # Online mode online_parser = subparsers.add_parser('online', help="Compare two online replicas for differences") @@ -1577,6 +1599,8 @@ def main(): online_parser.add_argument('-p', '--page-size', help='The paged-search result grouping size (default 500 entries)', dest='pagesize', default=500) online_parser.add_argument('-o', '--out-file', help='The output file', dest='file', default=None) + online_parser.add_argument('-t', '--timeout', help='The timeout for the LDAP connections. Default is no timeout.', + type=int, dest='timeout', default=-1) # Offline LDIF mode offline_parser = subparsers.add_parser('offline', help="Compare two replication LDIF files for differences (LDIF file generated by 'db2ldif -r')") -- 2.26.2