diff --git a/SOURCES/0001-Issue-4383-Do-not-normalize-escaped-spaces-in-a-DN.patch b/SOURCES/0001-Issue-4383-Do-not-normalize-escaped-spaces-in-a-DN.patch new file mode 100644 index 0000000..1b08b52 --- /dev/null +++ b/SOURCES/0001-Issue-4383-Do-not-normalize-escaped-spaces-in-a-DN.patch @@ -0,0 +1,159 @@ +From 81dcaf1c37c2de24c46672df8d4f968c2fb40a6e Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Wed, 11 Nov 2020 08:59:18 -0500 +Subject: [PATCH 1/3] Issue 4383 - Do not normalize escaped spaces in a DN + +Bug Description: Adding an entry with an escaped leading space leads to many + problems. Mainly id2entry can get corrupted during an + import of such an entry, and the entryrdn index is not + updated correctly + +Fix Description: In slapi_dn_normalize_ext() leave an escaped space intact. + +Relates: https://github.com/389ds/389-ds-base/issues/4383 + +Reviewed by: firstyear, progier, and tbordaz (Thanks!!!) +--- + .../tests/suites/syntax/acceptance_test.py | 75 ++++++++++++++++++- + ldap/servers/slapd/dn.c | 8 +- + 2 files changed, 77 insertions(+), 6 deletions(-) + +diff --git a/dirsrvtests/tests/suites/syntax/acceptance_test.py b/dirsrvtests/tests/suites/syntax/acceptance_test.py +index 543718689..7939a99a7 100644 +--- a/dirsrvtests/tests/suites/syntax/acceptance_test.py ++++ b/dirsrvtests/tests/suites/syntax/acceptance_test.py +@@ -1,5 +1,5 @@ + # --- BEGIN COPYRIGHT BLOCK --- +-# Copyright (C) 2019 Red Hat, Inc. ++# Copyright (C) 2020 Red Hat, Inc. + # All rights reserved. + # + # License: GPL (version 3 or any later version). +@@ -7,13 +7,12 @@ + # --- END COPYRIGHT BLOCK --- + + import ldap +-import logging + import pytest + import os + from lib389.schema import Schema + from lib389.config import Config + from lib389.idm.user import UserAccounts +-from lib389.idm.group import Groups ++from lib389.idm.group import Group, Groups + from lib389._constants import DEFAULT_SUFFIX + from lib389.topologies import log, topology_st as topo + +@@ -127,7 +126,7 @@ def test_invalid_dn_syntax_crash(topo): + 4. Success + """ + +- # Create group ++ # Create group + groups = Groups(topo.standalone, DEFAULT_SUFFIX) + group = groups.create(properties={'cn': ' test'}) + +@@ -145,6 +144,74 @@ def test_invalid_dn_syntax_crash(topo): + groups.list() + + ++@pytest.mark.parametrize("props, rawdn", [ ++ ({'cn': ' leadingSpace'}, "cn=\\20leadingSpace,ou=Groups,dc=example,dc=com"), ++ ({'cn': 'trailingSpace '}, "cn=trailingSpace\\20,ou=Groups,dc=example,dc=com")]) ++def test_dn_syntax_spaces_delete(topo, props, rawdn): ++ """Test that an entry with a space as the first character in the DN can be ++ deleted without error. We also want to make sure the indexes are properly ++ updated by repeatedly adding and deleting the entry, and that the entry cache ++ is properly maintained. ++ ++ :id: b993f37c-c2b0-4312-992c-a9048ff98965 ++ :parametrized: yes ++ :setup: Standalone Instance ++ :steps: ++ 1. Create a group with a DN that has a space as the first/last ++ character. ++ 2. Delete group ++ 3. Add group ++ 4. Modify group ++ 5. Restart server and modify entry ++ 6. Delete group ++ 7. Add group back ++ 8. Delete group using specific DN ++ :expectedresults: ++ 1. Success ++ 2. Success ++ 3. Success ++ 4. Success ++ 5. Success ++ 6. Success ++ 7. Success ++ 8. Success ++ """ ++ ++ # Create group ++ groups = Groups(topo.standalone, DEFAULT_SUFFIX) ++ group = groups.create(properties=props.copy()) ++ ++ # Delete group (verifies DN/RDN parsing works and cache is correct) ++ group.delete() ++ ++ # Add group again (verifies entryrdn index was properly updated) ++ groups = Groups(topo.standalone, DEFAULT_SUFFIX) ++ group = groups.create(properties=props.copy()) ++ ++ # Modify the group (verifies dn/rdn parsing is correct) ++ group.replace('description', 'escaped space group') ++ ++ # Restart the server. This will pull the entry from the database and ++ # convert it into a cache entry, which is different than how a client ++ # first adds an entry and is put into the cache before being written to ++ # disk. ++ topo.standalone.restart() ++ ++ # Make sure we can modify the entry (verifies cache entry was created ++ # correctly) ++ group.replace('description', 'escaped space group after restart') ++ ++ # Make sure it can still be deleted (verifies cache again). ++ group.delete() ++ ++ # Add it back so we can delete it using a specific DN (sanity test to verify ++ # another DN/RDN parsing variation). ++ groups = Groups(topo.standalone, DEFAULT_SUFFIX) ++ group = groups.create(properties=props.copy()) ++ group = Group(topo.standalone, dn=rawdn) ++ group.delete() ++ ++ + if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode +diff --git a/ldap/servers/slapd/dn.c b/ldap/servers/slapd/dn.c +index 2af3f38fc..3980b897f 100644 +--- a/ldap/servers/slapd/dn.c ++++ b/ldap/servers/slapd/dn.c +@@ -894,8 +894,7 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len) + s++; + } + } +- } else if (s + 2 < ends && +- isxdigit(*(s + 1)) && isxdigit(*(s + 2))) { ++ } else if (s + 2 < ends && isxdigit(*(s + 1)) && isxdigit(*(s + 2))) { + /* esc hexpair ==> real character */ + int n = slapi_hexchar2int(*(s + 1)); + int n2 = slapi_hexchar2int(*(s + 2)); +@@ -903,6 +902,11 @@ slapi_dn_normalize_ext(char *src, size_t src_len, char **dest, size_t *dest_len) + if (n == 0) { /* don't change \00 */ + *d++ = *++s; + *d++ = *++s; ++ } else if (n == 32) { /* leave \20 (space) intact */ ++ *d++ = *s; ++ *d++ = *++s; ++ *d++ = *++s; ++ s++; + } else { + *d++ = n; + s += 3; +-- +2.26.2 + diff --git a/SOURCES/0002-ticket-2058-Add-keep-alive-entry-after-on-line-initi.patch b/SOURCES/0002-ticket-2058-Add-keep-alive-entry-after-on-line-initi.patch new file mode 100644 index 0000000..e82fdf8 --- /dev/null +++ b/SOURCES/0002-ticket-2058-Add-keep-alive-entry-after-on-line-initi.patch @@ -0,0 +1,232 @@ +From 29c9e1c3c760f0941b022d45d14c248e9ceb9738 Mon Sep 17 00:00:00 2001 +From: progier389 <72748589+progier389@users.noreply.github.com> +Date: Tue, 3 Nov 2020 12:18:50 +0100 +Subject: [PATCH 2/3] ticket 2058: Add keep alive entry after on-line + initialization - second version (#4399) + +Bug description: +Keep alive entry is not created on target master after on line initialization, +and its RUVelement stays empty until a direct update is issued on that master + +Fix description: +The patch allows a consumer (configured as a master) to create (if it did not +exist before) the consumer's keep alive entry. It creates it at the end of a +replication session at a time we are sure the changelog exists and will not +be reset. It allows a consumer to have RUVelement with csn in the RUV at the +first incoming replication session. + +That is basically lkrispen's proposal with an associated pytest testcase + +Second version changes: + - moved the testcase to suites/replication/regression_test.py + - set up the topology from a 2 master topology then + reinitialized the replicas from an ldif without replication metadata + rather than using the cli. + - search for keepalive entries using search_s instead of getEntry + - add a comment about keep alive entries purpose + +last commit: + - wait that ruv are in sync before checking keep alive entries + +Reviewed by: droideck, Firstyear + +Platforms tested: F32 + +relates: #2058 +--- + .../suites/replication/regression_test.py | 130 ++++++++++++++++++ + .../plugins/replication/repl5_replica.c | 14 ++ + ldap/servers/plugins/replication/repl_extop.c | 4 + + 3 files changed, 148 insertions(+) + +diff --git a/dirsrvtests/tests/suites/replication/regression_test.py b/dirsrvtests/tests/suites/replication/regression_test.py +index 844d762b9..14b9d6a44 100644 +--- a/dirsrvtests/tests/suites/replication/regression_test.py ++++ b/dirsrvtests/tests/suites/replication/regression_test.py +@@ -98,6 +98,30 @@ def _move_ruv(ldif_file): + for dn, entry in ldif_list: + ldif_writer.unparse(dn, entry) + ++def _remove_replication_data(ldif_file): ++ """ Remove the replication data from ldif file: ++ db2lif without -r includes some of the replica data like ++ - nsUniqueId ++ - keepalive entries ++ This function filters the ldif fil to remove these data ++ """ ++ ++ with open(ldif_file) as f: ++ parser = ldif.LDIFRecordList(f) ++ parser.parse() ++ ++ ldif_list = parser.all_records ++ # Iterate on a copy of the ldif entry list ++ for dn, entry in ldif_list[:]: ++ if dn.startswith('cn=repl keep alive'): ++ ldif_list.remove((dn,entry)) ++ else: ++ entry.pop('nsUniqueId') ++ with open(ldif_file, 'w') as f: ++ ldif_writer = ldif.LDIFWriter(f) ++ for dn, entry in ldif_list: ++ ldif_writer.unparse(dn, entry) ++ + + @pytest.fixture(scope="module") + def topo_with_sigkill(request): +@@ -897,6 +921,112 @@ def test_moving_entry_make_online_init_fail(topology_m2): + assert len(m1entries) == len(m2entries) + + ++def get_keepalive_entries(instance,replica): ++ # Returns the keep alive entries that exists with the suffix of the server instance ++ try: ++ entries = instance.search_s(replica.get_suffix(), ldap.SCOPE_ONELEVEL, ++ "(&(objectclass=ldapsubentry)(cn=repl keep alive*))", ++ ['cn', 'nsUniqueId', 'modifierTimestamp']) ++ except ldap.LDAPError as e: ++ log.fatal('Failed to retrieve keepalive entry (%s) on instance %s: error %s' % (dn, instance, str(e))) ++ assert False ++ # No error, so lets log the keepalive entries ++ if log.isEnabledFor(logging.DEBUG): ++ for ret in entries: ++ log.debug("Found keepalive entry:\n"+str(ret)); ++ return entries ++ ++def verify_keepalive_entries(topo, expected): ++ #Check that keep alive entries exists (or not exists) for every masters on every masters ++ #Note: The testing method is quite basic: counting that there is one keepalive entry per master. ++ # that is ok for simple test cases like test_online_init_should_create_keepalive_entries but ++ # not for the general case as keep alive associated with no more existing master may exists ++ # (for example after: db2ldif / demote a master / ldif2db / init other masters) ++ # ==> if the function is somehow pushed in lib389, a check better than simply counting the entries ++ # should be done. ++ for masterId in topo.ms: ++ master=topo.ms[masterId] ++ for replica in Replicas(master).list(): ++ if (replica.get_role() != ReplicaRole.MASTER): ++ continue ++ replica_info = f'master: {masterId} RID: {replica.get_rid()} suffix: {replica.get_suffix()}' ++ log.debug(f'Checking keepAliveEntries on {replica_info}') ++ keepaliveEntries = get_keepalive_entries(master, replica); ++ expectedCount = len(topo.ms) if expected else 0 ++ foundCount = len(keepaliveEntries) ++ if (foundCount == expectedCount): ++ log.debug(f'Found {foundCount} keepalive entries as expected on {replica_info}.') ++ else: ++ log.error(f'{foundCount} Keepalive entries are found ' ++ f'while {expectedCount} were expected on {replica_info}.') ++ assert False ++ ++ ++def test_online_init_should_create_keepalive_entries(topo_m2): ++ """Check that keep alive entries are created when initializinf a master from another one ++ ++ :id: d5940e71-d18a-4b71-aaf7-b9185361fffe ++ :setup: Two masters replication setup ++ :steps: ++ 1. Generate ldif without replication data ++ 2 Init both masters from that ldif ++ 3 Check that keep alive entries does not exists ++ 4 Perform on line init of master2 from master1 ++ 5 Check that keep alive entries exists ++ :expectedresults: ++ 1. No error while generating ldif ++ 2. No error while importing the ldif file ++ 3. No keepalive entrie should exists on any masters ++ 4. No error while initializing master2 ++ 5. All keepalive entries should exist on every masters ++ ++ """ ++ ++ repl = ReplicationManager(DEFAULT_SUFFIX) ++ m1 = topo_m2.ms["master1"] ++ m2 = topo_m2.ms["master2"] ++ # Step 1: Generate ldif without replication data ++ m1.stop() ++ m2.stop() ++ ldif_file = '%s/norepl.ldif' % m1.get_ldif_dir() ++ m1.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX], ++ excludeSuffixes=None, repl_data=False, ++ outputfile=ldif_file, encrypt=False) ++ # Remove replication metadata that are still in the ldif ++ _remove_replication_data(ldif_file) ++ ++ # Step 2: Init both masters from that ldif ++ m1.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file) ++ m2.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file) ++ m1.start() ++ m2.start() ++ ++ """ Replica state is now as if CLI setup has been done using: ++ dsconf master1 replication enable --suffix "${SUFFIX}" --role master ++ dsconf master2 replication enable --suffix "${SUFFIX}" --role master ++ dsconf master1 replication create-manager --name "${REPLICATION_MANAGER_NAME}" --passwd "${REPLICATION_MANAGER_PASSWORD}" ++ dsconf master2 replication create-manager --name "${REPLICATION_MANAGER_NAME}" --passwd "${REPLICATION_MANAGER_PASSWORD}" ++ dsconf master1 repl-agmt create --suffix "${SUFFIX}" ++ dsconf master2 repl-agmt create --suffix "${SUFFIX}" ++ """ ++ ++ # Step 3: No keepalive entrie should exists on any masters ++ verify_keepalive_entries(topo_m2, False) ++ ++ # Step 4: Perform on line init of master2 from master1 ++ agmt = Agreements(m1).list()[0] ++ agmt.begin_reinit() ++ (done, error) = agmt.wait_reinit() ++ assert done is True ++ assert error is False ++ ++ # Step 5: All keepalive entries should exists on every masters ++ # Verify the keep alive entry once replication is in sync ++ # (that is the step that fails when bug is not fixed) ++ repl.wait_for_ruv(m2,m1) ++ verify_keepalive_entries(topo_m2, True); ++ ++ + if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode +diff --git a/ldap/servers/plugins/replication/repl5_replica.c b/ldap/servers/plugins/replication/repl5_replica.c +index f01782330..f0ea0f8ef 100644 +--- a/ldap/servers/plugins/replication/repl5_replica.c ++++ b/ldap/servers/plugins/replication/repl5_replica.c +@@ -373,6 +373,20 @@ replica_destroy(void **arg) + slapi_ch_free((void **)arg); + } + ++/****************************************************************************** ++ ******************** REPLICATION KEEP ALIVE ENTRIES ************************** ++ ****************************************************************************** ++ * They are subentries of the replicated suffix and there is one per master. * ++ * These entries exist only to trigger a change that get replicated over the * ++ * topology. * ++ * Their main purpose is to generate records in the changelog and they are * ++ * updated from time to time by fractional replication to insure that at * ++ * least a change must be replicated by FR after a great number of not * ++ * replicated changes are found in the changelog. The interest is that the * ++ * fractional RUV get then updated so less changes need to be walked in the * ++ * changelog when searching for the first change to send * ++ ******************************************************************************/ ++ + #define KEEP_ALIVE_ATTR "keepalivetimestamp" + #define KEEP_ALIVE_ENTRY "repl keep alive" + #define KEEP_ALIVE_DN_FORMAT "cn=%s %d,%s" +diff --git a/ldap/servers/plugins/replication/repl_extop.c b/ldap/servers/plugins/replication/repl_extop.c +index 14c8e0bcc..af486f730 100644 +--- a/ldap/servers/plugins/replication/repl_extop.c ++++ b/ldap/servers/plugins/replication/repl_extop.c +@@ -1173,6 +1173,10 @@ multimaster_extop_EndNSDS50ReplicationRequest(Slapi_PBlock *pb) + */ + if (cl5GetState() == CL5_STATE_OPEN) { + replica_log_ruv_elements(r); ++ /* now that the changelog is open and started, we can alos cretae the ++ * keep alive entry without risk that db and cl will not match ++ */ ++ replica_subentry_check(replica_get_root(r), replica_get_rid(r)); + } + + /* ONREPL code that dealt with new RUV, etc was moved into the code +-- +2.26.2 + diff --git a/SOURCES/0003-do-not-add-referrals-for-masters-with-different-data.patch b/SOURCES/0003-do-not-add-referrals-for-masters-with-different-data.patch new file mode 100644 index 0000000..411958e --- /dev/null +++ b/SOURCES/0003-do-not-add-referrals-for-masters-with-different-data.patch @@ -0,0 +1,513 @@ +From e202c62c3b4c92163d2de9f3da9a9f3efc81e4b8 Mon Sep 17 00:00:00 2001 +From: progier389 <72748589+progier389@users.noreply.github.com> +Date: Thu, 12 Nov 2020 18:50:04 +0100 +Subject: [PATCH 3/3] do not add referrals for masters with different data + generation #2054 (#4427) + +Bug description: +The problem is that some operation mandatory in the usual cases are +also performed when replication cannot take place because the +database set are differents (i.e: RUV generation ids are different) + +One of the issue is that the csn generator state is updated when +starting a replication session (it is a problem when trying to +reset the time skew, as freshly reinstalled replicas get infected +by the old ones) + +A second issue is that the RUV got updated when ending a replication session +(which may add replica that does not share the same data set, +then update operations on consumer retun referrals towards wrong masters + +Fix description: +The fix checks the RUVs generation id before updating the csn generator +and before updating the RUV. + +Reviewed by: mreynolds + firstyear + vashirov + +Platforms tested: F32 +--- + .../suites/replication/regression_test.py | 290 ++++++++++++++++++ + ldap/servers/plugins/replication/repl5.h | 1 + + .../plugins/replication/repl5_inc_protocol.c | 20 +- + .../plugins/replication/repl5_replica.c | 39 ++- + src/lib389/lib389/dseldif.py | 37 +++ + 5 files changed, 368 insertions(+), 19 deletions(-) + +diff --git a/dirsrvtests/tests/suites/replication/regression_test.py b/dirsrvtests/tests/suites/replication/regression_test.py +index 14b9d6a44..a72af6b30 100644 +--- a/dirsrvtests/tests/suites/replication/regression_test.py ++++ b/dirsrvtests/tests/suites/replication/regression_test.py +@@ -13,6 +13,7 @@ from lib389.idm.user import TEST_USER_PROPERTIES, UserAccounts + from lib389.pwpolicy import PwPolicyManager + from lib389.utils import * + from lib389.topologies import topology_m2 as topo_m2, TopologyMain, topology_m3 as topo_m3, create_topology, _remove_ssca_db, topology_i2 as topo_i2 ++from lib389.topologies import topology_m2c2 as topo_m2c2 + from lib389._constants import * + from lib389.idm.organizationalunit import OrganizationalUnits + from lib389.idm.user import UserAccount +@@ -22,6 +23,7 @@ from lib389.idm.directorymanager import DirectoryManager + from lib389.replica import Replicas, ReplicationManager, Changelog5, BootstrapReplicationManager + from lib389.agreement import Agreements + from lib389 import pid_from_file ++from lib389.dseldif import * + + + pytestmark = pytest.mark.tier1 +@@ -1027,6 +1029,294 @@ def test_online_init_should_create_keepalive_entries(topo_m2): + verify_keepalive_entries(topo_m2, True); + + ++def get_agreement(agmts, consumer): ++ # Get agreement towards consumer among the agremment list ++ for agmt in agmts.list(): ++ if (agmt.get_attr_val_utf8('nsDS5ReplicaPort') == str(consumer.port) and ++ agmt.get_attr_val_utf8('nsDS5ReplicaHost') == consumer.host): ++ return agmt ++ return None; ++ ++ ++def test_ruv_url_not_added_if_different_uuid(topo_m2c2): ++ """Check that RUV url is not updated if RUV generation uuid are different ++ ++ :id: 7cc30a4e-0ffd-4758-8f00-e500279af344 ++ :setup: Two masters + two consumers replication setup ++ :steps: ++ 1. Generate ldif without replication data ++ 2. Init both masters from that ldif ++ (to clear the ruvs and generates different generation uuid) ++ 3. Perform on line init from master1 to consumer1 ++ and from master2 to consumer2 ++ 4. Perform update on both masters ++ 5. Check that c1 RUV does not contains URL towards m2 ++ 6. Check that c2 RUV does contains URL towards m2 ++ 7. Perform on line init from master1 to master2 ++ 8. Perform update on master2 ++ 9. Check that c1 RUV does contains URL towards m2 ++ :expectedresults: ++ 1. No error while generating ldif ++ 2. No error while importing the ldif file ++ 3. No error and Initialization done. ++ 4. No error ++ 5. master2 replicaid should not be in the consumer1 RUV ++ 6. master2 replicaid should be in the consumer2 RUV ++ 7. No error and Initialization done. ++ 8. No error ++ 9. master2 replicaid should be in the consumer1 RUV ++ ++ """ ++ ++ # Variables initialization ++ repl = ReplicationManager(DEFAULT_SUFFIX) ++ ++ m1 = topo_m2c2.ms["master1"] ++ m2 = topo_m2c2.ms["master2"] ++ c1 = topo_m2c2.cs["consumer1"] ++ c2 = topo_m2c2.cs["consumer2"] ++ ++ replica_m1 = Replicas(m1).get(DEFAULT_SUFFIX) ++ replica_m2 = Replicas(m2).get(DEFAULT_SUFFIX) ++ replica_c1 = Replicas(c1).get(DEFAULT_SUFFIX) ++ replica_c2 = Replicas(c2).get(DEFAULT_SUFFIX) ++ ++ replicid_m2 = replica_m2.get_rid() ++ ++ agmts_m1 = Agreements(m1, replica_m1.dn) ++ agmts_m2 = Agreements(m2, replica_m2.dn) ++ ++ m1_m2 = get_agreement(agmts_m1, m2) ++ m1_c1 = get_agreement(agmts_m1, c1) ++ m1_c2 = get_agreement(agmts_m1, c2) ++ m2_m1 = get_agreement(agmts_m2, m1) ++ m2_c1 = get_agreement(agmts_m2, c1) ++ m2_c2 = get_agreement(agmts_m2, c2) ++ ++ # Step 1: Generate ldif without replication data ++ m1.stop() ++ m2.stop() ++ ldif_file = '%s/norepl.ldif' % m1.get_ldif_dir() ++ m1.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX], ++ excludeSuffixes=None, repl_data=False, ++ outputfile=ldif_file, encrypt=False) ++ # Remove replication metadata that are still in the ldif ++ # _remove_replication_data(ldif_file) ++ ++ # Step 2: Init both masters from that ldif ++ m1.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file) ++ m2.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file) ++ m1.start() ++ m2.start() ++ ++ # Step 3: Perform on line init from master1 to consumer1 ++ # and from master2 to consumer2 ++ m1_c1.begin_reinit() ++ m2_c2.begin_reinit() ++ (done, error) = m1_c1.wait_reinit() ++ assert done is True ++ assert error is False ++ (done, error) = m2_c2.wait_reinit() ++ assert done is True ++ assert error is False ++ ++ # Step 4: Perform update on both masters ++ repl.test_replication(m1, c1) ++ repl.test_replication(m2, c2) ++ ++ # Step 5: Check that c1 RUV does not contains URL towards m2 ++ ruv = replica_c1.get_ruv() ++ log.debug(f"c1 RUV: {ruv}") ++ url=ruv._rid_url.get(replica_m2.get_rid()) ++ if (url == None): ++ log.debug(f"No URL for RID {replica_m2.get_rid()} in RUV"); ++ else: ++ log.debug(f"URL for RID {replica_m2.get_rid()} in RUV is {url}"); ++ log.error(f"URL for RID {replica_m2.get_rid()} found in RUV") ++ #Note: this assertion fails if issue 2054 is not fixed. ++ assert False ++ ++ # Step 6: Check that c2 RUV does contains URL towards m2 ++ ruv = replica_c2.get_ruv() ++ log.debug(f"c1 RUV: {ruv} {ruv._rids} ") ++ url=ruv._rid_url.get(replica_m2.get_rid()) ++ if (url == None): ++ log.error(f"No URL for RID {replica_m2.get_rid()} in RUV"); ++ assert False ++ else: ++ log.debug(f"URL for RID {replica_m2.get_rid()} in RUV is {url}"); ++ ++ ++ # Step 7: Perform on line init from master1 to master2 ++ m1_m2.begin_reinit() ++ (done, error) = m1_m2.wait_reinit() ++ assert done is True ++ assert error is False ++ ++ # Step 8: Perform update on master2 ++ repl.test_replication(m2, c1) ++ ++ # Step 9: Check that c1 RUV does contains URL towards m2 ++ ruv = replica_c1.get_ruv() ++ log.debug(f"c1 RUV: {ruv} {ruv._rids} ") ++ url=ruv._rid_url.get(replica_m2.get_rid()) ++ if (url == None): ++ log.error(f"No URL for RID {replica_m2.get_rid()} in RUV"); ++ assert False ++ else: ++ log.debug(f"URL for RID {replica_m2.get_rid()} in RUV is {url}"); ++ ++ ++def test_csngen_state_not_updated_if_different_uuid(topo_m2c2): ++ """Check that csngen remote offset is not updated if RUV generation uuid are different ++ ++ :id: 77694b8e-22ae-11eb-89b2-482ae39447e5 ++ :setup: Two masters + two consumers replication setup ++ :steps: ++ 1. Disable m1<->m2 agreement to avoid propagate timeSkew ++ 2. Generate ldif without replication data ++ 3. Increase time skew on master2 ++ 4. Init both masters from that ldif ++ (to clear the ruvs and generates different generation uuid) ++ 5. Perform on line init from master1 to consumer1 and master2 to consumer2 ++ 6. Perform update on both masters ++ 7: Check that c1 has no time skew ++ 8: Check that c2 has time skew ++ 9. Init master2 from master1 ++ 10. Perform update on master2 ++ 11. Check that c1 has time skew ++ :expectedresults: ++ 1. No error ++ 2. No error while generating ldif ++ 3. No error ++ 4. No error while importing the ldif file ++ 5. No error and Initialization done. ++ 6. No error ++ 7. c1 time skew should be lesser than threshold ++ 8. c2 time skew should be higher than threshold ++ 9. No error and Initialization done. ++ 10. No error ++ 11. c1 time skew should be higher than threshold ++ ++ """ ++ ++ # Variables initialization ++ repl = ReplicationManager(DEFAULT_SUFFIX) ++ ++ m1 = topo_m2c2.ms["master1"] ++ m2 = topo_m2c2.ms["master2"] ++ c1 = topo_m2c2.cs["consumer1"] ++ c2 = topo_m2c2.cs["consumer2"] ++ ++ replica_m1 = Replicas(m1).get(DEFAULT_SUFFIX) ++ replica_m2 = Replicas(m2).get(DEFAULT_SUFFIX) ++ replica_c1 = Replicas(c1).get(DEFAULT_SUFFIX) ++ replica_c2 = Replicas(c2).get(DEFAULT_SUFFIX) ++ ++ replicid_m2 = replica_m2.get_rid() ++ ++ agmts_m1 = Agreements(m1, replica_m1.dn) ++ agmts_m2 = Agreements(m2, replica_m2.dn) ++ ++ m1_m2 = get_agreement(agmts_m1, m2) ++ m1_c1 = get_agreement(agmts_m1, c1) ++ m1_c2 = get_agreement(agmts_m1, c2) ++ m2_m1 = get_agreement(agmts_m2, m1) ++ m2_c1 = get_agreement(agmts_m2, c1) ++ m2_c2 = get_agreement(agmts_m2, c2) ++ ++ # Step 1: Disable m1<->m2 agreement to avoid propagate timeSkew ++ m1_m2.pause() ++ m2_m1.pause() ++ ++ # Step 2: Generate ldif without replication data ++ m1.stop() ++ m2.stop() ++ ldif_file = '%s/norepl.ldif' % m1.get_ldif_dir() ++ m1.db2ldif(bename=DEFAULT_BENAME, suffixes=[DEFAULT_SUFFIX], ++ excludeSuffixes=None, repl_data=False, ++ outputfile=ldif_file, encrypt=False) ++ # Remove replication metadata that are still in the ldif ++ # _remove_replication_data(ldif_file) ++ ++ # Step 3: Increase time skew on master2 ++ timeSkew=6*3600 ++ # We can modify master2 time skew ++ # But the time skew on the consumer may be smaller ++ # depending on when the cnsgen generation time is updated ++ # and when first csn get replicated. ++ # Since we use timeSkew has threshold value to detect ++ # whether there are time skew or not, ++ # lets add a significative margin (longer than the test duration) ++ # to avoid any risk of erroneous failure ++ timeSkewMargin = 300 ++ DSEldif(m2)._increaseTimeSkew(DEFAULT_SUFFIX, timeSkew+timeSkewMargin) ++ ++ # Step 4: Init both masters from that ldif ++ m1.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file) ++ m2.ldif2db(DEFAULT_BENAME, None, None, None, ldif_file) ++ m1.start() ++ m2.start() ++ ++ # Step 5: Perform on line init from master1 to consumer1 ++ # and from master2 to consumer2 ++ m1_c1.begin_reinit() ++ m2_c2.begin_reinit() ++ (done, error) = m1_c1.wait_reinit() ++ assert done is True ++ assert error is False ++ (done, error) = m2_c2.wait_reinit() ++ assert done is True ++ assert error is False ++ ++ # Step 6: Perform update on both masters ++ repl.test_replication(m1, c1) ++ repl.test_replication(m2, c2) ++ ++ # Step 7: Check that c1 has no time skew ++ # Stop server to insure that dse.ldif is uptodate ++ c1.stop() ++ c1_nsState = DSEldif(c1).readNsState(DEFAULT_SUFFIX)[0] ++ c1_timeSkew = int(c1_nsState['time_skew']) ++ log.debug(f"c1 time skew: {c1_timeSkew}") ++ if (c1_timeSkew >= timeSkew): ++ log.error(f"c1 csngen state has unexpectedly been synchronized with m2: time skew {c1_timeSkew}") ++ assert False ++ c1.start() ++ ++ # Step 8: Check that c2 has time skew ++ # Stop server to insure that dse.ldif is uptodate ++ c2.stop() ++ c2_nsState = DSEldif(c2).readNsState(DEFAULT_SUFFIX)[0] ++ c2_timeSkew = int(c2_nsState['time_skew']) ++ log.debug(f"c2 time skew: {c2_timeSkew}") ++ if (c2_timeSkew < timeSkew): ++ log.error(f"c2 csngen state has not been synchronized with m2: time skew {c2_timeSkew}") ++ assert False ++ c2.start() ++ ++ # Step 9: Perform on line init from master1 to master2 ++ m1_c1.pause() ++ m1_m2.resume() ++ m1_m2.begin_reinit() ++ (done, error) = m1_m2.wait_reinit() ++ assert done is True ++ assert error is False ++ ++ # Step 10: Perform update on master2 ++ repl.test_replication(m2, c1) ++ ++ # Step 11: Check that c1 has time skew ++ # Stop server to insure that dse.ldif is uptodate ++ c1.stop() ++ c1_nsState = DSEldif(c1).readNsState(DEFAULT_SUFFIX)[0] ++ c1_timeSkew = int(c1_nsState['time_skew']) ++ log.debug(f"c1 time skew: {c1_timeSkew}") ++ if (c1_timeSkew < timeSkew): ++ log.error(f"c1 csngen state has not been synchronized with m2: time skew {c1_timeSkew}") ++ assert False ++ ++ + if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode +diff --git a/ldap/servers/plugins/replication/repl5.h b/ldap/servers/plugins/replication/repl5.h +index b35f724c2..f1c596a3f 100644 +--- a/ldap/servers/plugins/replication/repl5.h ++++ b/ldap/servers/plugins/replication/repl5.h +@@ -708,6 +708,7 @@ void replica_dump(Replica *r); + void replica_set_enabled(Replica *r, PRBool enable); + Replica *replica_get_replica_from_dn(const Slapi_DN *dn); + Replica *replica_get_replica_from_root(const char *repl_root); ++int replica_check_generation(Replica *r, const RUV *remote_ruv); + int replica_update_ruv(Replica *replica, const CSN *csn, const char *replica_purl); + Replica *replica_get_replica_for_op(Slapi_PBlock *pb); + /* the functions below manipulate replica hash */ +diff --git a/ldap/servers/plugins/replication/repl5_inc_protocol.c b/ldap/servers/plugins/replication/repl5_inc_protocol.c +index 29b1fb073..af5e5897c 100644 +--- a/ldap/servers/plugins/replication/repl5_inc_protocol.c ++++ b/ldap/servers/plugins/replication/repl5_inc_protocol.c +@@ -2161,26 +2161,12 @@ examine_update_vector(Private_Repl_Protocol *prp, RUV *remote_ruv) + } else if (NULL == remote_ruv) { + return_value = EXAMINE_RUV_PRISTINE_REPLICA; + } else { +- char *local_gen = NULL; +- char *remote_gen = ruv_get_replica_generation(remote_ruv); +- Object *local_ruv_obj; +- RUV *local_ruv; +- + PR_ASSERT(NULL != prp->replica); +- local_ruv_obj = replica_get_ruv(prp->replica); +- if (NULL != local_ruv_obj) { +- local_ruv = (RUV *)object_get_data(local_ruv_obj); +- PR_ASSERT(local_ruv); +- local_gen = ruv_get_replica_generation(local_ruv); +- object_release(local_ruv_obj); +- } +- if (NULL == remote_gen || NULL == local_gen || strcmp(remote_gen, local_gen) != 0) { +- return_value = EXAMINE_RUV_GENERATION_MISMATCH; +- } else { ++ if (replica_check_generation(prp->replica, remote_ruv)) { + return_value = EXAMINE_RUV_OK; ++ } else { ++ return_value = EXAMINE_RUV_GENERATION_MISMATCH; + } +- slapi_ch_free((void **)&remote_gen); +- slapi_ch_free((void **)&local_gen); + } + return return_value; + } +diff --git a/ldap/servers/plugins/replication/repl5_replica.c b/ldap/servers/plugins/replication/repl5_replica.c +index f0ea0f8ef..7e56d6557 100644 +--- a/ldap/servers/plugins/replication/repl5_replica.c ++++ b/ldap/servers/plugins/replication/repl5_replica.c +@@ -812,6 +812,36 @@ replica_set_ruv(Replica *r, RUV *ruv) + replica_unlock(r->repl_lock); + } + ++/* ++ * Check if replica generation is the same than the remote ruv one ++ */ ++int ++replica_check_generation(Replica *r, const RUV *remote_ruv) ++{ ++ int return_value; ++ char *local_gen = NULL; ++ char *remote_gen = ruv_get_replica_generation(remote_ruv); ++ Object *local_ruv_obj; ++ RUV *local_ruv; ++ ++ PR_ASSERT(NULL != r); ++ local_ruv_obj = replica_get_ruv(r); ++ if (NULL != local_ruv_obj) { ++ local_ruv = (RUV *)object_get_data(local_ruv_obj); ++ PR_ASSERT(local_ruv); ++ local_gen = ruv_get_replica_generation(local_ruv); ++ object_release(local_ruv_obj); ++ } ++ if (NULL == remote_gen || NULL == local_gen || strcmp(remote_gen, local_gen) != 0) { ++ return_value = PR_FALSE; ++ } else { ++ return_value = PR_TRUE; ++ } ++ slapi_ch_free_string(&remote_gen); ++ slapi_ch_free_string(&local_gen); ++ return return_value; ++} ++ + /* + * Update one particular CSN in an RUV. This is meant to be called + * whenever (a) the server has processed a client operation and +@@ -1298,6 +1328,11 @@ replica_update_csngen_state_ext(Replica *r, const RUV *ruv, const CSN *extracsn) + + PR_ASSERT(r && ruv); + ++ if (!replica_check_generation(r, ruv)) /* ruv has wrong generation - we are done */ ++ { ++ return 0; ++ } ++ + rc = ruv_get_max_csn(ruv, &csn); + if (rc != RUV_SUCCESS) { + return -1; +@@ -3713,8 +3748,8 @@ replica_update_ruv_consumer(Replica *r, RUV *supplier_ruv) + replica_lock(r->repl_lock); + + local_ruv = (RUV *)object_get_data(r->repl_ruv); +- +- if (is_cleaned_rid(supplier_id) || local_ruv == NULL) { ++ if (is_cleaned_rid(supplier_id) || local_ruv == NULL || ++ !replica_check_generation(r, supplier_ruv)) { + replica_unlock(r->repl_lock); + return; + } +diff --git a/src/lib389/lib389/dseldif.py b/src/lib389/lib389/dseldif.py +index 10baba4d7..6850c9a8a 100644 +--- a/src/lib389/lib389/dseldif.py ++++ b/src/lib389/lib389/dseldif.py +@@ -317,6 +317,43 @@ class DSEldif(DSLint): + + return states + ++ def _increaseTimeSkew(self, suffix, timeSkew): ++ # Increase csngen state local_offset by timeSkew ++ # Warning: instance must be stopped before calling this function ++ assert (timeSkew >= 0) ++ nsState = self.readNsState(suffix)[0] ++ self._instance.log.debug(f'_increaseTimeSkew nsState is {nsState}') ++ oldNsState = self.get(nsState['dn'], 'nsState', True) ++ self._instance.log.debug(f'oldNsState is {oldNsState}') ++ ++ # Lets reencode the new nsState ++ from lib389.utils import print_nice_time ++ if pack('h', 1) == pack('=h',1): ++ end = '>' ++ else: ++ raise ValueError("Unknown endian, unable to proceed") ++ ++ thelen = len(oldNsState) ++ if thelen <= 20: ++ pad = 2 # padding for short H values ++ timefmt = 'I' # timevals are unsigned 32-bit int ++ else: ++ pad = 6 # padding for short H values ++ timefmt = 'Q' # timevals are unsigned 64-bit int ++ fmtstr = "%sH%dx3%sH%dx" % (end, pad, timefmt, pad) ++ newNsState = base64.b64encode(pack(fmtstr, int(nsState['rid']), ++ int(nsState['gen_time']), int(nsState['local_offset'])+timeSkew, ++ int(nsState['remote_offset']), int(nsState['seq_num']))) ++ newNsState = newNsState.decode('utf-8') ++ self._instance.log.debug(f'newNsState is {newNsState}') ++ # Lets replace the value. ++ (entry_dn_i, attr_data) = self._find_attr(nsState['dn'], 'nsState') ++ attr_i = next(iter(attr_data)) ++ self._contents[entry_dn_i + attr_i] = f"nsState:: {newNsState}" ++ self._update() ++ + + class FSChecks(DSLint): + """This is for the healthcheck feature, check commonly used system config files the +-- +2.26.2 + diff --git a/SOURCES/0004-Ticket-50933-Update-2307compat.ldif.patch b/SOURCES/0004-Ticket-50933-Update-2307compat.ldif.patch new file mode 100644 index 0000000..5622a1a --- /dev/null +++ b/SOURCES/0004-Ticket-50933-Update-2307compat.ldif.patch @@ -0,0 +1,179 @@ +From 826a1bb4ea88915ac492828d1cc4a901623f7866 Mon Sep 17 00:00:00 2001 +From: William Brown +Date: Thu, 14 May 2020 14:31:47 +1000 +Subject: [PATCH 1/2] Ticket 50933 - Update 2307compat.ldif + +Bug Description: This resolves a potential conflict between 60nis.ldif +in freeipa and others with 2307compat, by removing the conflicting +definitions from 2307bis that were included. + +Fix Description: By not including these in 2307compat, this means that +sites that rely on the values provided by 2307bis may ALSO need +60nis.ldif to be present. However, these nis values seem like they are +likely very rare in reality, and this also will avoid potential +issues with freeipa. It also is the least disruptive as we don't need +to change an already defined file, and we don't have values where the name +to oid relationship changes. + +Fixes: #50933 +https://pagure.io/389-ds-base/issue/50933 + +Author: William Brown + +Review by: tbordaz (Thanks!) +--- + ldap/schema/10rfc2307compat.ldif | 66 -------------------------------- + ldap/schema/60autofs.ldif | 39 ++++++++++++------- + 2 files changed, 26 insertions(+), 79 deletions(-) + +diff --git a/ldap/schema/10rfc2307compat.ldif b/ldap/schema/10rfc2307compat.ldif +index 8810231ac..78c588d08 100644 +--- a/ldap/schema/10rfc2307compat.ldif ++++ b/ldap/schema/10rfc2307compat.ldif +@@ -176,50 +176,6 @@ attributeTypes: ( + SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 + SINGLE-VALUE + ) +-attributeTypes: ( +- 1.3.6.1.1.1.1.28 NAME 'nisPublicKey' +- DESC 'NIS public key' +- EQUALITY octetStringMatch +- SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 +- SINGLE-VALUE +- ) +-attributeTypes: ( +- 1.3.6.1.1.1.1.29 NAME 'nisSecretKey' +- DESC 'NIS secret key' +- EQUALITY octetStringMatch +- SYNTAX 1.3.6.1.4.1.1466.115.121.1.40 +- SINGLE-VALUE +- ) +-attributeTypes: ( +- 1.3.6.1.1.1.1.30 NAME 'nisDomain' +- DESC 'NIS domain' +- EQUALITY caseIgnoreIA5Match +- SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 +- ) +-attributeTypes: ( +- 1.3.6.1.1.1.1.31 NAME 'automountMapName' +- DESC 'automount Map Name' +- EQUALITY caseExactIA5Match +- SUBSTR caseExactIA5SubstringsMatch +- SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 +- SINGLE-VALUE +- ) +-attributeTypes: ( +- 1.3.6.1.1.1.1.32 NAME 'automountKey' +- DESC 'Automount Key value' +- EQUALITY caseExactIA5Match +- SUBSTR caseExactIA5SubstringsMatch +- SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 +- SINGLE-VALUE +- ) +-attributeTypes: ( +- 1.3.6.1.1.1.1.33 NAME 'automountInformation' +- DESC 'Automount information' +- EQUALITY caseExactIA5Match +- SUBSTR caseExactIA5SubstringsMatch +- SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 +- SINGLE-VALUE +- ) + # end of attribute types - beginning of objectclasses + objectClasses: ( + 1.3.6.1.1.1.2.0 NAME 'posixAccount' SUP top AUXILIARY +@@ -324,28 +280,6 @@ objectClasses: ( + seeAlso $ serialNumber' + MAY ( bootFile $ bootParameter $ cn $ description $ l $ o $ ou $ owner $ seeAlso $ serialNumber ) + ) +-objectClasses: ( +- 1.3.6.1.1.1.2.14 NAME 'nisKeyObject' SUP top AUXILIARY +- DESC 'An object with a public and secret key' +- MUST ( cn $ nisPublicKey $ nisSecretKey ) +- MAY ( uidNumber $ description ) +- ) +-objectClasses: ( +- 1.3.6.1.1.1.2.15 NAME 'nisDomainObject' SUP top AUXILIARY +- DESC 'Associates a NIS domain with a naming context' +- MUST nisDomain +- ) +-objectClasses: ( +- 1.3.6.1.1.1.2.16 NAME 'automountMap' SUP top STRUCTURAL +- MUST ( automountMapName ) +- MAY description +- ) +-objectClasses: ( +- 1.3.6.1.1.1.2.17 NAME 'automount' SUP top STRUCTURAL +- DESC 'Automount information' +- MUST ( automountKey $ automountInformation ) +- MAY description +- ) + ## namedObject is needed for groups without members + objectClasses: ( + 1.3.6.1.4.1.5322.13.1.1 NAME 'namedObject' SUP top STRUCTURAL +diff --git a/ldap/schema/60autofs.ldif b/ldap/schema/60autofs.ldif +index 084e9ec30..de3922aa2 100644 +--- a/ldap/schema/60autofs.ldif ++++ b/ldap/schema/60autofs.ldif +@@ -6,7 +6,23 @@ dn: cn=schema + ################################################################################ + # + attributeTypes: ( +- 1.3.6.1.1.1.1.33 ++ 1.3.6.1.1.1.1.31 NAME 'automountMapName' ++ DESC 'automount Map Name' ++ EQUALITY caseExactIA5Match ++ SUBSTR caseExactIA5SubstringsMatch ++ SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ++ SINGLE-VALUE ++ ) ++attributeTypes: ( ++ 1.3.6.1.1.1.1.32 NAME 'automountKey' ++ DESC 'Automount Key value' ++ EQUALITY caseExactIA5Match ++ SUBSTR caseExactIA5SubstringsMatch ++ SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ++ SINGLE-VALUE ++ ) ++attributeTypes: ( ++ 1.3.6.1.1.1.1.33 + NAME 'automountInformation' + DESC 'Information used by the autofs automounter' + EQUALITY caseExactIA5Match +@@ -18,25 +34,22 @@ attributeTypes: ( + ################################################################################ + # + objectClasses: ( +- 1.3.6.1.1.1.2.17 +- NAME 'automount' +- DESC 'An entry in an automounter map' ++ 1.3.6.1.1.1.2.16 ++ NAME 'automountMap' ++ DESC 'An group of related automount objects' + SUP top + STRUCTURAL +- MUST ( cn $ automountInformation ) +- MAY ( description ) ++ MAY ( ou $ automountMapName $ description ) + X-ORIGIN 'draft-howard-rfc2307bis' + ) +-# +-################################################################################ +-# + objectClasses: ( +- 1.3.6.1.1.1.2.16 +- NAME 'automountMap' +- DESC 'An group of related automount objects' ++ 1.3.6.1.1.1.2.17 ++ NAME 'automount' ++ DESC 'An entry in an automounter map' + SUP top + STRUCTURAL +- MUST ( ou ) ++ MUST ( automountInformation ) ++ MAY ( cn $ description $ automountKey ) + X-ORIGIN 'draft-howard-rfc2307bis' + ) + # +-- +2.26.2 + diff --git a/SOURCES/0005-Issue-50933-Fix-OID-change-between-10rfc2307-and-10r.patch b/SOURCES/0005-Issue-50933-Fix-OID-change-between-10rfc2307-and-10r.patch new file mode 100644 index 0000000..82fdf9d --- /dev/null +++ b/SOURCES/0005-Issue-50933-Fix-OID-change-between-10rfc2307-and-10r.patch @@ -0,0 +1,36 @@ +From 3d9ced9e340678cc02b1a36c2139492c95ef15a6 Mon Sep 17 00:00:00 2001 +From: Mark Reynolds +Date: Wed, 12 Aug 2020 12:46:42 -0400 +Subject: [PATCH 2/2] Issue 50933 - Fix OID change between 10rfc2307 and + 10rfc2307compat + +Bug Description: 10rfc2307compat changed the OID for nisMap objectclass to + match the standard OID, but this breaks replication with + older versions of DS. + +Fix Description: Continue to use the old(invalid?) oid for nisMap so that + replication does not break in a mixed version environment. + +Fixes: https://pagure.io/389-ds-base/issue/50933 + +Reviewed by: firstyear & tbordaz(Thanks!!) +--- + ldap/schema/10rfc2307compat.ldif | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/ldap/schema/10rfc2307compat.ldif b/ldap/schema/10rfc2307compat.ldif +index 78c588d08..8ba72e1e3 100644 +--- a/ldap/schema/10rfc2307compat.ldif ++++ b/ldap/schema/10rfc2307compat.ldif +@@ -253,7 +253,7 @@ objectClasses: ( + MAY ( nisNetgroupTriple $ memberNisNetgroup $ description ) + ) + objectClasses: ( +- 1.3.6.1.1.1.2.9 NAME 'nisMap' SUP top STRUCTURAL ++ 1.3.6.1.1.1.2.13 NAME 'nisMap' SUP top STRUCTURAL + DESC 'A generic abstraction of a NIS map' + MUST nisMapName + MAY description +-- +2.26.2 + diff --git a/SOURCES/0006-Ticket-51131-improve-mutex-alloc-in-conntable.patch b/SOURCES/0006-Ticket-51131-improve-mutex-alloc-in-conntable.patch new file mode 100644 index 0000000..4269446 --- /dev/null +++ b/SOURCES/0006-Ticket-51131-improve-mutex-alloc-in-conntable.patch @@ -0,0 +1,147 @@ +From 1085823bf5586d55103cfba249fdf212e9afcb7c Mon Sep 17 00:00:00 2001 +From: William Brown +Date: Thu, 4 Jun 2020 11:51:53 +1000 +Subject: [PATCH] Ticket 51131 - improve mutex alloc in conntable + +Bug Description: We previously did delayed allocation +of mutexs, which @tbordaz noted can lead to high usage +of the pthread mutex init routines. This was done under +the conntable lock, as well as cleaning the connection + +Fix Description: rather than delayed allocation, we +initialise everything at start up instead, which means +that while startup may have a delay, at run time we have +a smaller and lighter connection allocation routine, +that is able to release the CT lock sooner. + +https://pagure.io/389-ds-base/issue/51131 + +Author: William Brown + +Review by: ??? +--- + ldap/servers/slapd/conntable.c | 86 +++++++++++++++++++--------------- + 1 file changed, 47 insertions(+), 39 deletions(-) + +diff --git a/ldap/servers/slapd/conntable.c b/ldap/servers/slapd/conntable.c +index b23dc3435..feb9c0d75 100644 +--- a/ldap/servers/slapd/conntable.c ++++ b/ldap/servers/slapd/conntable.c +@@ -138,10 +138,21 @@ connection_table_new(int table_size) + ct->conn_next_offset = 1; + ct->conn_free_offset = 1; + ++ pthread_mutexattr_t monitor_attr = {0}; ++ pthread_mutexattr_init(&monitor_attr); ++ pthread_mutexattr_settype(&monitor_attr, PTHREAD_MUTEX_RECURSIVE); ++ + /* We rely on the fact that we called calloc, which zeros the block, so we don't + * init any structure element unless a zero value is troublesome later + */ + for (i = 0; i < table_size; i++) { ++ /* ++ * Technically this is a no-op due to calloc, but we should always be ++ * careful with things like this .... ++ */ ++ ct->c[i].c_state = CONN_STATE_FREE; ++ /* Start the conn setup. */ ++ + LBER_SOCKET invalid_socket; + /* DBDB---move this out of here once everything works */ + ct->c[i].c_sb = ber_sockbuf_alloc(); +@@ -161,11 +172,20 @@ connection_table_new(int table_size) + ct->c[i].c_prev = NULL; + ct->c[i].c_ci = i; + ct->c[i].c_fdi = SLAPD_INVALID_SOCKET_INDEX; +- /* +- * Technically this is a no-op due to calloc, but we should always be +- * careful with things like this .... +- */ +- ct->c[i].c_state = CONN_STATE_FREE; ++ ++ if (pthread_mutex_init(&(ct->c[i].c_mutex), &monitor_attr) != 0) { ++ slapi_log_err(SLAPI_LOG_ERR, "connection_table_get_connection", "pthread_mutex_init failed\n"); ++ exit(1); ++ } ++ ++ ct->c[i].c_pdumutex = PR_NewLock(); ++ if (ct->c[i].c_pdumutex == NULL) { ++ slapi_log_err(SLAPI_LOG_ERR, "connection_table_get_connection", "PR_NewLock failed\n"); ++ exit(1); ++ } ++ ++ /* Ready to rock, mark as such. */ ++ ct->c[i].c_state = CONN_STATE_INIT; + /* Prepare the connection into the freelist. */ + ct->c_freelist[i] = &(ct->c[i]); + } +@@ -241,44 +261,32 @@ connection_table_get_connection(Connection_Table *ct, int sd) + /* Never use slot 0 */ + ct->conn_next_offset += 1; + } +- /* Now prep the slot for usage. */ +- PR_ASSERT(c->c_next == NULL); +- PR_ASSERT(c->c_prev == NULL); +- PR_ASSERT(c->c_extension == NULL); +- +- if (c->c_state == CONN_STATE_FREE) { +- +- c->c_state = CONN_STATE_INIT; +- +- pthread_mutexattr_t monitor_attr = {0}; +- pthread_mutexattr_init(&monitor_attr); +- pthread_mutexattr_settype(&monitor_attr, PTHREAD_MUTEX_RECURSIVE); +- if (pthread_mutex_init(&(c->c_mutex), &monitor_attr) != 0) { +- slapi_log_err(SLAPI_LOG_ERR, "connection_table_get_connection", "pthread_mutex_init failed\n"); +- exit(1); +- } +- +- c->c_pdumutex = PR_NewLock(); +- if (c->c_pdumutex == NULL) { +- c->c_pdumutex = NULL; +- slapi_log_err(SLAPI_LOG_ERR, "connection_table_get_connection", "PR_NewLock failed\n"); +- exit(1); +- } +- } +- /* Let's make sure there's no cruft left on there from the last time this connection was used. */ +- /* Note: no need to lock c->c_mutex because this function is only +- * called by one thread (the slapd_daemon thread), and if we got this +- * far then `c' is not being used by any operation threads, etc. +- */ +- connection_cleanup(c); +- c->c_ct = ct; /* pointer to connection table that owns this connection */ ++ PR_Unlock(ct->table_mutex); + } else { +- /* couldn't find a Connection */ ++ /* couldn't find a Connection, table must be full */ + slapi_log_err(SLAPI_LOG_CONNS, "connection_table_get_connection", "Max open connections reached\n"); ++ PR_Unlock(ct->table_mutex); ++ return NULL; + } + +- /* We could move this to before the c alloc as there is no point to remain here. */ +- PR_Unlock(ct->table_mutex); ++ /* Now prep the slot for usage. */ ++ PR_ASSERT(c != NULL); ++ PR_ASSERT(c->c_next == NULL); ++ PR_ASSERT(c->c_prev == NULL); ++ PR_ASSERT(c->c_extension == NULL); ++ PR_ASSERT(c->c_state == CONN_STATE_INIT); ++ /* Let's make sure there's no cruft left on there from the last time this connection was used. */ ++ ++ /* ++ * Note: no need to lock c->c_mutex because this function is only ++ * called by one thread (the slapd_daemon thread), and if we got this ++ * far then `c' is not being used by any operation threads, etc. The ++ * memory ordering will be provided by the work queue sending c to a ++ * thread. ++ */ ++ connection_cleanup(c); ++ /* pointer to connection table that owns this connection */ ++ c->c_ct = ct; + + return c; + } +-- +2.26.2 + diff --git a/SOURCES/0007-Issue-4297-2nd-fix-for-on-ADD-replication-URP-issue-.patch b/SOURCES/0007-Issue-4297-2nd-fix-for-on-ADD-replication-URP-issue-.patch new file mode 100644 index 0000000..41f9315 --- /dev/null +++ b/SOURCES/0007-Issue-4297-2nd-fix-for-on-ADD-replication-URP-issue-.patch @@ -0,0 +1,66 @@ +From a9f53e9958861e6a7a827bd852d72d51a6512396 Mon Sep 17 00:00:00 2001 +From: tbordaz +Date: Wed, 25 Nov 2020 18:07:34 +0100 +Subject: [PATCH] Issue 4297 - 2nd fix for on ADD replication URP issue + internal searches with filter containing unescaped chars (#4439) + +Bug description: + Previous fix is buggy because slapi_filter_escape_filter_value returns + a escaped filter component not an escaped assertion value. + +Fix description: + use the escaped filter component + +relates: https://github.com/389ds/389-ds-base/issues/4297 + +Reviewed by: William Brown + +Platforms tested: F31 +--- + ldap/servers/plugins/replication/urp.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/ldap/servers/plugins/replication/urp.c b/ldap/servers/plugins/replication/urp.c +index f41dbc72d..ed340c9d8 100644 +--- a/ldap/servers/plugins/replication/urp.c ++++ b/ldap/servers/plugins/replication/urp.c +@@ -1411,12 +1411,12 @@ urp_add_check_tombstone (Slapi_PBlock *pb, char *sessionid, Slapi_Entry *entry, + Slapi_Entry **entries = NULL; + Slapi_PBlock *newpb; + char *basedn = slapi_entry_get_ndn(entry); +- char *escaped_basedn; ++ char *escaped_filter; + const Slapi_DN *suffix = slapi_get_suffix_by_dn(slapi_entry_get_sdn (entry)); +- escaped_basedn = slapi_filter_escape_filter_value("nscpentrydn", basedn); ++ escaped_filter = slapi_filter_escape_filter_value("nscpentrydn", basedn); + +- char *filter = slapi_filter_sprintf("(&(objectclass=nstombstone)(nscpentrydn=%s))", escaped_basedn); +- slapi_ch_free((void **)&escaped_basedn); ++ char *filter = slapi_filter_sprintf("(&(objectclass=nstombstone)%s)", escaped_filter); ++ slapi_ch_free((void **)&escaped_filter); + newpb = slapi_pblock_new(); + slapi_search_internal_set_pb(newpb, + slapi_sdn_get_dn(suffix), /* Base DN */ +@@ -1605,15 +1605,15 @@ urp_find_tombstone_for_glue (Slapi_PBlock *pb, char *sessionid, const Slapi_Entr + Slapi_Entry **entries = NULL; + Slapi_PBlock *newpb; + const char *basedn = slapi_sdn_get_dn(parentdn); +- char *escaped_basedn; +- escaped_basedn = slapi_filter_escape_filter_value("nscpentrydn", (char *)basedn); ++ char *escaped_filter; ++ escaped_filter = slapi_filter_escape_filter_value("nscpentrydn", (char *)basedn); + + char *conflict_csnstr = (char*)slapi_entry_attr_get_ref((Slapi_Entry *)entry, "conflictcsn"); + CSN *conflict_csn = csn_new_by_string(conflict_csnstr); + CSN *tombstone_csn = NULL; + +- char *filter = slapi_filter_sprintf("(&(objectclass=nstombstone)(nscpentrydn=%s))", escaped_basedn); +- slapi_ch_free((void **)&escaped_basedn); ++ char *filter = slapi_filter_sprintf("(&(objectclass=nstombstone)%s)", escaped_filter); ++ slapi_ch_free((void **)&escaped_filter); + newpb = slapi_pblock_new(); + char *parent_dn = slapi_dn_parent (basedn); + slapi_search_internal_set_pb(newpb, +-- +2.26.2 + diff --git a/SPECS/389-ds-base.spec b/SPECS/389-ds-base.spec index 3937284..1e798bc 100644 --- a/SPECS/389-ds-base.spec +++ b/SPECS/389-ds-base.spec @@ -45,7 +45,7 @@ ExcludeArch: i686 Summary: 389 Directory Server (base) Name: 389-ds-base Version: 1.4.3.16 -Release: %{?relprefix}1%{?prerel}%{?dist} +Release: %{?relprefix}3%{?prerel}%{?dist} License: GPLv3+ URL: https://www.port389.org Group: System Environment/Daemons @@ -174,6 +174,13 @@ Source2: %{name}-devel.README %if %{bundle_jemalloc} Source3: https://github.com/jemalloc/%{jemalloc_name}/releases/download/%{jemalloc_ver}/%{jemalloc_name}-%{jemalloc_ver}.tar.bz2 %endif +Patch01: 0001-Issue-4383-Do-not-normalize-escaped-spaces-in-a-DN.patch +Patch02: 0002-ticket-2058-Add-keep-alive-entry-after-on-line-initi.patch +Patch03: 0003-do-not-add-referrals-for-masters-with-different-data.patch +Patch04: 0004-Ticket-50933-Update-2307compat.ldif.patch +Patch05: 0005-Issue-50933-Fix-OID-change-between-10rfc2307-and-10r.patch +Patch06: 0006-Ticket-51131-improve-mutex-alloc-in-conntable.patch +Patch07: 0007-Issue-4297-2nd-fix-for-on-ADD-replication-URP-issue-.patch %description 389 Directory Server is an LDAPv3 compliant server. The base package includes @@ -791,6 +798,18 @@ exit 0 %doc README.md %changelog +* Wed Nov 25 2020 Mark Reynolds - 1.4.3.16-3 +- Bump version to 1.4.3.16-3 +- Resolves: Bug 1859219 - rfc2307 and rfc2307bis compat schema +- Resolves: Bug 1843604 - reduce the cost of allocation/free when open/close a connection +- Resolves: Bug 1898850 - Entries conflict not resolved by replication + +* Thu Nov 19 2020 Mark Reynolds - 1.4.3.16-2 +- Bump version to 1.4.3.16-2 +- Resolves: Bug 1859227 - create keep alive entry after on line init +- Resolves: Bug 1888863 - group rdn with leading space char and add fails error 21 invalid syntax and delete fails error 32 +- Resolves: Bug 1859228 - do not add referrals for masters with different data generation + * Mon Oct 26 2020 Mark Reynolds - 1.4.3.16-1 - Bump version to 1.4.3.16-1 - Resolves: Bug 1887415 - Sync repl - if a serie of updates target the same entry then the cookie get wrong changenumber