From 2cfe91947ec8d40e28dfe2e1fd5bb1b351ca75f1 Mon Sep 17 00:00:00 2001

From: Rich Megginson <rmeggins@redhat.com>

Date: Wed, 6 Nov 2013 14:22:31 -0700

Subject: [PATCH 147/225] Ticket #47585 Replication Failures related to skipped

 entries due to cleaned rids

https://fedorahosted.org/389/ticket/47585

Reviewed by: nhosoi (Thanks!)

Branch: 389-ds-base-1.2.11

Fix Description: If a change was found in the changelog buffer that is

skipped due to having an unknown replica ID (rid), the entire buffer was

marked as CLC_STATE_NEW_RID.  When the buffer is exhausted and the iterator

code goes to read in the new buffer, it would not read in the new buffer

because it only loads a new buffer if the current buffer state is

CLC_STATE_READY.  I don't know why the entire buffer would be marked as

CLC_STATE_NEW_RID and stop iteration.  It seems to me that just the update

should be skipped, but new buffers should be loaded in order to keep sending

non-skipped updates to the consumer.

It is possible for a CSN with an unknown RID to get into the changelog if

the server with that RID had been removed by cleanruv/cleanallruv.  In that

case, the CSN should be skipped.  It is assumed that the change was already

sent - cleanallruv is supposed to wait until all known changes have been

seen before removing the RID from the RUV - so it is safe to skip it.

Added additional debugging, so that we can better tell why changelog entries

were skipped.

Platforms tested: RHEL6 x86_64

Flag Day: no

Doc impact: no

(cherry picked from commit cf08f1274404e4796966011a98a6a0acbbfd6070)

(cherry picked from commit 30bb98fb693ea1aac9774bdc43b923eacd72570a)

(cherry picked from commit fc70e4ac6accaa14d140e333829e98897f6ff164)

(cherry picked from commit 7eefa121784774a2c1535404f0b95b78970b43c2)

(cherry picked from commit c783d159a3baa005f0a5f007d886961e2128bdcc)

---

 ldap/servers/plugins/replication/cl5_clcache.c | 48 ++++++++++++++++++++++----

 1 file changed, 42 insertions(+), 6 deletions(-)

diff --git a/ldap/servers/plugins/replication/cl5_clcache.c b/ldap/servers/plugins/replication/cl5_clcache.c

index 7a6a446..8218312 100644

--- a/ldap/servers/plugins/replication/cl5_clcache.c

+++ b/ldap/servers/plugins/replication/cl5_clcache.c

@@ -120,6 +120,11 @@ struct clc_buffer {

 	int		 	 buf_load_cnt;		/* number of loads for session */

 	int		 	 buf_record_cnt;	/* number of changes for session */

 	int		 	 buf_record_skipped;	/* number of changes skipped */

+	int		 	 buf_skipped_new_rid;	/* number of changes skipped due to new_rid */

+	int		 	 buf_skipped_csn_gt_cons_maxcsn;	/* number of changes skipped due to csn greater than consumer maxcsn */

+	int		 	 buf_skipped_up_to_date;	/* number of changes skipped due to consumer being up-to-date for the given rid */

+	int		 	 buf_skipped_csn_gt_ruv;	/* number of changes skipped due to preceedents are not covered by local RUV snapshot */

+	int		 	 buf_skipped_csn_covered;	/* number of changes skipped due to CSNs already covered by consumer RUV */

 	/*

 	 * fields that should be accessed via bl_lock or pl_lock

@@ -252,6 +257,11 @@ clcache_get_buffer ( CLC_Buffer **buf, DB *db, ReplicaId consumer_rid, const RUV

 		(*buf)->buf_record_skipped = 0;

 		(*buf)->buf_cursor = NULL;

 		(*buf)->buf_num_cscbs = 0;

+		(*buf)->buf_skipped_new_rid = 0;

+		(*buf)->buf_skipped_csn_gt_cons_maxcsn = 0;

+		(*buf)->buf_skipped_up_to_date = 0;

+		(*buf)->buf_skipped_csn_gt_ruv = 0;

+		(*buf)->buf_skipped_csn_covered = 0;

 	}

 	else {

 		*buf = clcache_new_buffer ( consumer_rid );

@@ -287,11 +297,16 @@ clcache_return_buffer ( CLC_Buffer **buf )

 	int i;

 	slapi_log_error ( SLAPI_LOG_REPL, (*buf)->buf_agmt_name,

-			"session end: state=%d load=%d sent=%d skipped=%d\n",

-			 (*buf)->buf_state,

-			 (*buf)->buf_load_cnt,

-			 (*buf)->buf_record_cnt - (*buf)->buf_record_skipped,

-			 (*buf)->buf_record_skipped );

+			  "session end: state=%d load=%d sent=%d skipped=%d skipped_new_rid=%d "

+			  "skipped_csn_gt_cons_maxcsn=%d skipped_up_to_date=%d "

+			  "skipped_csn_gt_ruv=%d skipped_csn_covered=%d\n",

+			  (*buf)->buf_state,

+			  (*buf)->buf_load_cnt,

+			  (*buf)->buf_record_cnt - (*buf)->buf_record_skipped,

+			  (*buf)->buf_record_skipped, (*buf)->buf_skipped_new_rid,

+			  (*buf)->buf_skipped_csn_gt_cons_maxcsn,

+			  (*buf)->buf_skipped_up_to_date, (*buf)->buf_skipped_csn_gt_ruv,

+			  (*buf)->buf_skipped_csn_covered);

 	for ( i = 0; i < (*buf)->buf_num_cscbs; i++ ) {

 		clcache_free_cscb ( &(*buf)->buf_cscbs[i] );

@@ -676,6 +691,8 @@ clcache_skip_change ( CLC_Buffer *buf )

 	ReplicaId rid;

 	int skip = 1;

 	int i;

+	char buf_cur_csn_str[CSN_STRSIZE];

+	char oth_csn_str[CSN_STRSIZE];

 	do {

@@ -697,6 +714,14 @@ clcache_skip_change ( CLC_Buffer *buf )

 				 *  The consumer must have been "restored" and needs this newer update.

 				 */

 				skip = 0;

+			} else if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {

+				csn_as_string(buf->buf_current_csn, 0, buf_cur_csn_str);

+				csn_as_string(cons_maxcsn, 0, oth_csn_str);

+				slapi_log_error(SLAPI_LOG_REPL, buf->buf_agmt_name,

+					"Skipping update because the changelog buffer current csn [%s] is "

+				        "less than or equal to the consumer max csn [%s]\n",

+				        buf_cur_csn_str, oth_csn_str);

+				buf->buf_skipped_csn_gt_cons_maxcsn++;

 			}

 			csn_free(&cons_maxcsn);

 			break;

@@ -714,7 +739,14 @@ clcache_skip_change ( CLC_Buffer *buf )

 		/* Skip CSN whose RID is unknown to the local RUV snapshot */

 		if ( i >= buf->buf_num_cscbs ) {

-			buf->buf_state = CLC_STATE_NEW_RID;

+			if (slapi_is_loglevel_set(SLAPI_LOG_REPL)) {

+				csn_as_string(buf->buf_current_csn, 0, buf_cur_csn_str);

+				slapi_log_error(SLAPI_LOG_REPL, buf->buf_agmt_name,

+					"Skipping update because the changelog buffer current csn [%s] rid "

+				        "[%d] is not in the list of changelog csn buffers (length %d)\n",

+				        buf_cur_csn_str, rid, buf->buf_num_cscbs);

+			}

+			buf->buf_skipped_new_rid++;

 			break;

 		}

@@ -722,17 +754,20 @@ clcache_skip_change ( CLC_Buffer *buf )

 		/* Skip if the consumer is already up-to-date for the RID */

 		if ( cscb->state == CLC_STATE_UP_TO_DATE ) {

+			buf->buf_skipped_up_to_date++;

 			break;

 		}

 		/* Skip CSN whose preceedents are not covered by local RUV snapshot */

 		if ( cscb->state == CLC_STATE_CSN_GT_RUV ) {

+			buf->buf_skipped_csn_gt_ruv++;

 			break;

 		}

 		/* Skip CSNs already covered by consumer RUV */

 		if ( cscb->consumer_maxcsn &&

 			 csn_compare ( buf->buf_current_csn, cscb->consumer_maxcsn ) <= 0 ) {

+			buf->buf_skipped_csn_covered++;

 				break;

 		}

@@ -762,6 +797,7 @@ clcache_skip_change ( CLC_Buffer *buf )

 		/* Skip CSNs not covered by local RUV snapshot */

 		cscb->state = CLC_STATE_CSN_GT_RUV;

+		buf->buf_skipped_csn_gt_ruv++;

 	} while (0);

-- 

1.8.1.4