From 3712b35b7a61154be53649341885f49dce2820f4 Mon Sep 17 00:00:00 2001

From: Rich Megginson <rmeggins@redhat.com>

Date: Wed, 26 Jun 2013 13:35:39 -0600

Subject: [PATCH 75/99] Ticket #47410 - changelog db deadlocks with DNA and

 replication

https://fedorahosted.org/389/ticket/47410

Reviewed by: mreynolds (Thanks!)

Branch: 389-ds-base-1.2.11

Fix Description: The deadlock is caused by having an outer and an inner

transaction in one thread, and a replication reader in another thread.  The

outer transaction acquires a write lock on certain changelog db (cldb) pages

as a result of a previous nested transaction (e.g. a DNA shared config

area update).  The changelog reader in the cursor positioning operation

acquires read locks on certain other pages.  When another inner write

transaction occurs, it may attempt to acquire a write lock on a page held

by a read lock in the reader thread.  This will eventually fail because

the reader will not release its lock on the page until the outer transaction

releases the write lock on the page.

The solution is to change the way the deadlock detection thread works, to

use a different deadlock rejection policy.  When using DB_LOCK_MINWRITE

instead of the default DB_LOCK_YOUNGEST, the reader thread lock request is

rejected.  This means the code that positions the changelog cursor has to be

able to handle a DB_LOCK_DEADLOCK return.

Changing the deadlock rejection policy globally to DB_LOCK_MINWRITE has the

potential to cause any search to get a DB_LOCK_DEADLOCK from a db or cursor

get(), so this will need to be tested a great deal to make sure we can handle

all such cases.

Platforms tested: RHEL6 x86_64

Flag Day: no

Doc impact: no

(cherry picked from commit b573d80d9c3acc6dba1bd60bdf7bf3fe4f4168df)

(cherry picked from commit 2e1d6331a294378650bef9af8a5f24bd338ae01e)

(cherry picked from commit 1987727066e6248cac978139779193d0f56ff9ce)

(cherry picked from commit 489de18b135c4019d93a3cf89a108b2c3c4cc3d9)

---

 ldap/servers/plugins/replication/cl5.h         |  2 ++

 ldap/servers/plugins/replication/cl5_api.c     |  1 -

 ldap/servers/plugins/replication/cl5_clcache.c | 20 +++++++++++++++++++-

 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/ldap/servers/plugins/replication/cl5.h b/ldap/servers/plugins/replication/cl5.h

index 4c92ecd..33f8140 100644

--- a/ldap/servers/plugins/replication/cl5.h

+++ b/ldap/servers/plugins/replication/cl5.h

@@ -73,4 +73,6 @@ void changelog5_config_done (changelog5Config *config);

 /* frees the content and the config structure */

 void changelog5_config_free (changelog5Config **config);

+#define MAX_TRIALS			50				/* number of retries on db operations */

+

 #endif

diff --git a/ldap/servers/plugins/replication/cl5_api.c b/ldap/servers/plugins/replication/cl5_api.c

index 175eb80..e24cead 100644

--- a/ldap/servers/plugins/replication/cl5_api.c

+++ b/ldap/servers/plugins/replication/cl5_api.c

@@ -67,7 +67,6 @@

 #define GUARDIAN_FILE		"guardian"		/* name of the guardian file */

 #define VERSION_FILE		"DBVERSION"		/* name of the version file  */

-#define MAX_TRIALS			50				/* number of retries on db operations */

 #define V_5					5				/* changelog entry version */

 #define CHUNK_SIZE			64*1024

 #define DBID_SIZE			64

diff --git a/ldap/servers/plugins/replication/cl5_clcache.c b/ldap/servers/plugins/replication/cl5_clcache.c

index 202cb64..5329b4b 100644

--- a/ldap/servers/plugins/replication/cl5_clcache.c

+++ b/ldap/servers/plugins/replication/cl5_clcache.c

@@ -380,6 +380,7 @@ clcache_load_buffer_bulk ( CLC_Buffer *buf, int flag )

 	DB_TXN *txn = NULL;

 	DBC *cursor = NULL;

 	int rc = 0;

+	int tries = 0;

 #if 0 /* txn control seems not improving anything so turn it off */

 	if ( *(_pool->pl_dbenv) ) {

@@ -401,6 +402,7 @@ clcache_load_buffer_bulk ( CLC_Buffer *buf, int flag )

 	}

 	PR_Lock ( buf->buf_busy_list->bl_lock );

+retry:

 	if ( 0 == ( rc = clcache_open_cursor ( txn, buf, &cursor )) ) {

 		if ( flag == DB_NEXT ) {

@@ -422,10 +424,26 @@ clcache_load_buffer_bulk ( CLC_Buffer *buf, int flag )

 	/*

 	 * Don't keep a cursor open across the whole replication session.

-	 * That had caused noticable DB resource contention.

+	 * That had caused noticeable DB resource contention.

 	 */

 	if ( cursor ) {

 		cursor->c_close ( cursor );

+		cursor = NULL;

+	}

+	if ((rc == DB_LOCK_DEADLOCK) && (tries < MAX_TRIALS)) {

+		PRIntervalTime interval;

+

+		tries++;

+		slapi_log_error ( SLAPI_LOG_TRACE, "clcache_load_buffer_bulk",

+		                  "deadlock number [%d] - retrying\n", tries );

+		/* back off */

+		interval = PR_MillisecondsToInterval(slapi_rand() % 100);

+		DS_Sleep(interval);

+		goto retry;

+	}

+	if ((rc == DB_LOCK_DEADLOCK) && (tries >= MAX_TRIALS)) {

+		slapi_log_error ( SLAPI_LOG_REPL, "clcache_load_buffer_bulk",

+		                  "could not load buffer from changelog after %d tries\n", tries );

 	}

 #if 0 /* txn control seems not improving anything so turn it off */

-- 

1.8.1.4