570667 - MMR: simultaneous total updates on the masters cause

         deadlock and data loss

https://bugzilla.redhat.com/show_bug.cgi?id=570667

Description: In the MMR topology, if a master receives a total
update request to initialize the other master and being initialized
by the other master at the same time, the 2 replication threads hang
and the replicated backend instance could be wiped out.

To prevent the server running the total update supplier and the
consumer at the same time, REPLICA_TOTAL_EXCL_SEND and _RECV bits
have been introduced.  If the server is sending the total update
to other replicas, the server rejects the total update request
on the backend.  But the server can send multiple total updates
to other replicas at the same time.  If the total update from
other master is in progress on the server, the server rejects
another total update from yet another master as well as a request
to initialize other replicas.

3 files changed, 58 insertions, 0 deletions

diff --git a/ldap/servers/plugins/replication/repl5.h b/ldap/servers/plugins/replication/repl5.h
index 97ce5569..c6859ddb 100644
--- a/ldap/servers/plugins/replication/repl5.h
+++ b/ldap/servers/plugins/replication/repl5.h
@@ -521,6 +521,12 @@ void replica_write_ruv (Replica *r);
 #define REPLICA_INCREMENTAL_IN_PROGRESS 2 /* Set only between start and stop inc */
 #define REPLICA_TOTAL_IN_PROGRESS 4 /* Set only between start and stop total */
 #define REPLICA_AGREEMENTS_DISABLED 8 /* Replica is offline */
+#define REPLICA_TOTAL_EXCL_SEND 16 /* The server is either sending or receiving
+                                      the total update.  Introducing it if SEND
+                                      is active, RECV should back off. And
+                                      vice versa.  But SEND can coexist. */
+#define REPLICA_TOTAL_EXCL_RECV 32 /* ditto */
+
 PRBool replica_is_state_flag_set(Replica *r, PRInt32 flag);
 void replica_set_state_flag (Replica *r, PRUint32 flag, PRBool clear); 
 void replica_set_tombstone_reap_stop(Replica *r, PRBool val);
diff --git a/ldap/servers/plugins/replication/repl5_protocol.c b/ldap/servers/plugins/replication/repl5_protocol.c
index 927c450a..efb32716 100644
--- a/ldap/servers/plugins/replication/repl5_protocol.c
+++ b/ldap/servers/plugins/replication/repl5_protocol.c
@@ -317,6 +317,28 @@ prot_thread_main(void *arg)
 		dev_debug("prot_thread_main(STATE_PERFORMING_INCREMENTAL_UPDATE): end");
 		break;
 	      case STATE_PERFORMING_TOTAL_UPDATE:
+		{
+		Slapi_DN *dn = agmt_get_replarea(agmt);
+		Replica *replica = NULL;
+		Object *replica_obj = replica_get_replica_from_dn(dn);
+		if (replica_obj)
+		{
+		    replica = (Replica*) object_get_data (replica_obj);
+		    /* If total update against this replica is in progress,
+		     * we should not initiate the total update to other replicas. */
+		    if (replica_is_state_flag_set(replica, REPLICA_TOTAL_EXCL_RECV))
+		    {
+		        object_release(replica_obj);
+                slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name,
+                    "%s: total update on the replica is in progress.  Cannot initiate the total update.\n", agmt_get_long_name(rp->agmt));
+		        break;
+		    }
+		    else
+		    {
+		        replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_SEND, 0);
+		    }
+		}
+
 		PR_Lock(rp->lock);
     
 		/* stop incremental protocol if running */
@@ -332,7 +354,13 @@ prot_thread_main(void *arg)
 		   replica initialization is completed. */
 		agmt_replica_init_done (agmt);
     
+		if (replica_obj)
+		{
+		    replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_SEND, 1);
+		    object_release(replica_obj);
+		}
 		break;
+		}
 	      case STATE_FINISHED:
 		dev_debug("prot_thread_main(STATE_FINISHED): exiting prot_thread_main");
 		done = 1;
diff --git a/ldap/servers/plugins/replication/repl_extop.c b/ldap/servers/plugins/replication/repl_extop.c
index b65c6c8f..c47ea93d 100644
--- a/ldap/servers/plugins/replication/repl_extop.c
+++ b/ldap/servers/plugins/replication/repl_extop.c
@@ -678,6 +678,25 @@ multimaster_extop_StartNSDS50ReplicationRequest(Slapi_PBlock *pb)
 		goto send_response;
 	}
 
+	if (REPL_PROTOCOL_50_TOTALUPDATE == connext->repl_protocol_version)
+	{
+		/* If total update has been initiated against other replicas or
+		 * this replica is already being initialized, we should return
+		 * an error immediately. */
+		if (replica_is_state_flag_set(replica,
+							REPLICA_TOTAL_EXCL_SEND|REPLICA_TOTAL_EXCL_RECV))
+		{
+			slapi_log_error(SLAPI_LOG_FATAL, repl_plugin_name,
+				"%s: total update on is initiated on the replica.  Cannot execute the total update from other master.\n", repl_root);
+			response = NSDS50_REPL_REPLICA_BUSY;
+			goto send_response;
+		}
+		else
+		{
+			replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_RECV, 0);
+		}
+	}
+
     /* check that this replica is not a 4.0 consumer */
     if (replica_is_legacy_consumer (replica))
     {
@@ -861,6 +880,11 @@ multimaster_extop_StartNSDS50ReplicationRequest(Slapi_PBlock *pb)
 	slapi_pblock_get(pb, SLAPI_CONNECTION, &connext->connection);
 
 send_response:
+	if (connext && replica &&
+		(REPL_PROTOCOL_50_TOTALUPDATE == connext->repl_protocol_version))
+	{
+		replica_set_state_flag (replica, REPLICA_TOTAL_EXCL_RECV, 1);
+	}
     if (response != NSDS50_REPL_REPLICA_READY)
     {
 		int resp_log_level = SLAPI_LOG_FATAL;