initial attempt at freezing databases in priority order

(This used to be ctdb commit e8d692590da1070c87a4144031e3306d190ebed2)

9 files changed, 253 insertions, 141 deletions

diff --git a/ctdb/client/ctdb_client.c b/ctdb/client/ctdb_client.c
index c7f0162cac..0948d4118a 100644
--- a/ctdb/client/ctdb_client.c
+++ b/ctdb/client/ctdb_client.c
@@ -1953,7 +1953,14 @@ int ctdb_ctrl_freeze_priority(struct ctdb_context *ctdb, struct timeval timeout,
 /* Freeze all databases */
 int ctdb_ctrl_freeze(struct ctdb_context *ctdb, struct timeval timeout, uint32_t destnode)
 {
-	return ctdb_ctrl_freeze_priority(ctdb, timeout, destnode, 0);
+	int i;
+
+	for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb_ctrl_freeze_priority(ctdb, timeout, destnode, i) != 0) {
+			return -1;
+		}
+	}
+	return 0;
 }
 
 /*
@@ -2375,7 +2382,7 @@ int ctdb_ctrl_modflags(struct ctdb_context *ctdb, struct timeval timeout, uint32
 	nodes = list_of_connected_nodes(ctdb, nodemap, tmp_ctx, true);
 
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_MODIFY_FLAGS,
-					nodes,
+					nodes, 0,
 					timeout, false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -2880,6 +2887,7 @@ int ctdb_client_async_wait(struct ctdb_context *ctdb, struct client_async_data *
 int ctdb_client_async_control(struct ctdb_context *ctdb,
 				enum ctdb_controls opcode,
 				uint32_t *nodes,
+				uint64_t srvid,
 				struct timeval timeout,
 				bool dont_log_errors,
 				TDB_DATA data,
@@ -2905,7 +2913,7 @@ int ctdb_client_async_control(struct ctdb_context *ctdb,
 	for (j=0; j<num_nodes; j++) {
 		uint32_t pnn = nodes[j];
 
-		state = ctdb_control_send(ctdb, pnn, 0, opcode, 
+		state = ctdb_control_send(ctdb, pnn, srvid, opcode, 
 					  0, data, async_data, &timeout, NULL);
 		if (state == NULL) {
 			DEBUG(DEBUG_ERR,(__location__ " Failed to call async control %u\n", (unsigned)opcode));
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index c54a8d2185..cc5729a117 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -371,6 +371,7 @@ enum ctdb_freeze_mode {CTDB_FREEZE_NONE, CTDB_FREEZE_PENDING, CTDB_FREEZE_FROZEN
 /* This capability is set if NATGW is enabled */
 #define CTDB_CAP_NATGW			0x00000008
 
+#define NUM_DB_PRIORITIES 3
 /* main state of the ctdb daemon */
 struct ctdb_context {
 	struct event_context *ev;
@@ -381,8 +382,10 @@ struct ctdb_context {
 	TALLOC_CTX *tickle_update_context;
 	TALLOC_CTX *keepalive_ctx;
 	struct ctdb_tunable tunable;
-	enum ctdb_freeze_mode freeze_mode;
-	struct ctdb_freeze_handle *freeze_handle;
+	enum ctdb_freeze_mode freeze_mode[NUM_DB_PRIORITIES+1];
+	struct ctdb_freeze_handle *freeze_handles[NUM_DB_PRIORITIES+1];
+	bool freeze_transaction_started;
+	uint32_t freeze_transaction_id;
 	struct ctdb_address address;
 	const char *name;
 	const char *db_directory;
@@ -1192,7 +1195,7 @@ void ctdb_request_control_reply(struct ctdb_context *ctdb, struct ctdb_req_contr
 				TDB_DATA *outdata, int32_t status, const char *errormsg);
 
 int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *c, bool *async_reply);
-int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c);
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority);
 
 int ctdb_start_recoverd(struct ctdb_context *ctdb);
 void ctdb_stop_recoverd(struct ctdb_context *ctdb);
@@ -1331,7 +1334,7 @@ int ctdb_ctrl_get_all_tunables(struct ctdb_context *ctdb,
 			       uint32_t destnode,
 			       struct ctdb_tunable *tunables);
 
-void ctdb_start_freeze(struct ctdb_context *ctdb);
+int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority);
 
 bool parse_ip_mask(const char *s, const char *iface, ctdb_sock_addr *addr, unsigned *mask);
 bool parse_ip_port(const char *s, ctdb_sock_addr *addr);
@@ -1426,6 +1429,7 @@ int ctdb_client_async_wait(struct ctdb_context *ctdb, struct client_async_data *
 int ctdb_client_async_control(struct ctdb_context *ctdb,
 				enum ctdb_controls opcode,
 				uint32_t *nodes,
+			      	uint64_t srvid,
 				struct timeval timeout,
 				bool dont_log_errors,
 				TDB_DATA data,
diff --git a/ctdb/server/ctdb_control.c b/ctdb/server/ctdb_control.c
index dbb828d783..57641e742c 100644
--- a/ctdb/server/ctdb_control.c
+++ b/ctdb/server/ctdb_control.c
@@ -98,9 +98,15 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 	}
 
 	case CTDB_CONTROL_STATISTICS: {
+		int i;
 		CHECK_CONTROL_DATA_SIZE(0);
 		ctdb->statistics.memory_used = talloc_total_size(NULL);
-		ctdb->statistics.frozen = (ctdb->freeze_mode == CTDB_FREEZE_FROZEN);
+		ctdb->statistics.frozen = 0;
+		for (i=1; i<= NUM_DB_PRIORITIES; i++) {
+			if (ctdb->freeze_mode[i] == CTDB_FREEZE_FROZEN) {
+				ctdb->statistics.frozen = 1;
+			}
+		}
 		ctdb->statistics.recovering = (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE);
 		outdata->dptr = (uint8_t *)&ctdb->statistics;
 		outdata->dsize = sizeof(ctdb->statistics);
@@ -249,7 +255,7 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 
 	case CTDB_CONTROL_THAW:
 		CHECK_CONTROL_DATA_SIZE(0);
-		return ctdb_control_thaw(ctdb, c);
+		return ctdb_control_thaw(ctdb, (uint32_t)c->srvid);
 
 	case CTDB_CONTROL_SET_RECMODE:
 		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));		
diff --git a/ctdb/server/ctdb_daemon.c b/ctdb/server/ctdb_daemon.c
index fcfba669a0..0af3bc1836 100644
--- a/ctdb/server/ctdb_daemon.c
+++ b/ctdb/server/ctdb_daemon.c
@@ -733,7 +733,7 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork)
 	}
 
 	/* start frozen, then let the first election sort things out */
-	if (!ctdb_blocking_freeze(ctdb)) {
+	if (ctdb_blocking_freeze(ctdb)) {
 		ctdb_fatal(ctdb, "Failed to get initial freeze\n");
 	}
 
diff --git a/ctdb/server/ctdb_freeze.c b/ctdb/server/ctdb_freeze.c
index 5a775ce681..2cc39aa274 100644
--- a/ctdb/server/ctdb_freeze.c
+++ b/ctdb/server/ctdb_freeze.c
@@ -30,10 +30,13 @@
 /*
   lock all databases
  */
-static int ctdb_lock_all_databases(struct ctdb_context *ctdb)
+static int ctdb_lock_all_databases(struct ctdb_context *ctdb, uint32_t priority)
 {
 	struct ctdb_db_context *ctdb_db;
 	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+		if (ctdb_db->priority != priority) {
+			continue;
+		}
 		DEBUG(DEBUG_INFO,("locking database 0x%08x priority:%u %s\n", ctdb_db->db_id, ctdb_db->priority, ctdb_db->db_name));
 		if (tdb_lockall(ctdb_db->ltdb->tdb) != 0) {
 			return -1;
@@ -50,17 +53,17 @@ struct ctdb_freeze_waiter {
 	struct ctdb_freeze_waiter *next, *prev;
 	struct ctdb_context *ctdb;
 	struct ctdb_req_control *c;
+	uint32_t priority;
 	int32_t status;
 };
 
 /* a handle to a freeze lock child process */
 struct ctdb_freeze_handle {
 	struct ctdb_context *ctdb;
+	uint32_t priority;
 	pid_t child;
 	int fd;
 	struct ctdb_freeze_waiter *waiters;
-	bool transaction_started;
-	uint32_t transaction_id;
 };
 
 /*
@@ -71,9 +74,14 @@ static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
 	struct ctdb_context *ctdb = h->ctdb;
 	struct ctdb_db_context *ctdb_db;
 
+	DEBUG(DEBUG_ERR,("Release freeze handler for prio %u\n", h->priority));
+
 	/* cancel any pending transactions */
-	if (ctdb->freeze_handle && ctdb->freeze_handle->transaction_started) {
+	if (ctdb->freeze_transaction_started) {
 		for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+			if (ctdb_db->priority != h->priority) {
+				continue;
+			}
 			tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 			if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 				DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
@@ -81,11 +89,11 @@ static int ctdb_freeze_handle_destructor(struct ctdb_freeze_handle *h)
 			}
 			tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 		}
-		ctdb->freeze_handle->transaction_started = false;
+		ctdb->freeze_transaction_started = false;
 	}
 
-	ctdb->freeze_mode = CTDB_FREEZE_NONE;
-	ctdb->freeze_handle = NULL;
+	ctdb->freeze_mode[h->priority]    = CTDB_FREEZE_NONE;
+	ctdb->freeze_handles[h->priority] = NULL;
 
 	kill(h->child, SIGKILL);
 	return 0;
@@ -101,11 +109,8 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
 	int32_t status;
 	struct ctdb_freeze_waiter *w;
 
-	if (h->ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+	if (h->ctdb->freeze_mode[h->priority] == CTDB_FREEZE_FROZEN) {
 		DEBUG(DEBUG_INFO,("freeze child died - unfreezing\n"));
-		if (h->ctdb->freeze_handle == h) {
-			h->ctdb->freeze_handle = NULL;
-		}
 		talloc_free(h);
 		return;
 	}
@@ -122,12 +127,12 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
 		return;
 	}
 
-	h->ctdb->freeze_mode = CTDB_FREEZE_FROZEN;
+	h->ctdb->freeze_mode[h->priority] = CTDB_FREEZE_FROZEN;
 
 	/* notify the waiters */
-	while ((w = h->ctdb->freeze_handle->waiters)) {
+	while ((w = h->ctdb->freeze_handles[h->priority]->waiters)) {
 		w->status = status;
-		DLIST_REMOVE(h->ctdb->freeze_handle->waiters, w);
+		DLIST_REMOVE(h->ctdb->freeze_handles[h->priority]->waiters, w);
 		talloc_free(w);
 	}
 }
@@ -136,7 +141,7 @@ static void ctdb_freeze_lock_handler(struct event_context *ev, struct fd_event *
   create a child which gets locks on all the open databases, then calls the callback telling the parent
   that it is done
  */
-static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
+static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb, uint32_t priority)
 {
 	struct ctdb_freeze_handle *h;
 	int fd[2];
@@ -145,7 +150,8 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
 	h = talloc_zero(ctdb, struct ctdb_freeze_handle);
 	CTDB_NO_MEMORY_NULL(ctdb, h);
 
-	h->ctdb = ctdb;
+	h->ctdb     = ctdb;
+	h->priority = priority;
 
 	/* use socketpair() instead of pipe() so we have bi-directional fds */
 	if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) != 0) {
@@ -166,7 +172,7 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
 		int count = 0;
 		/* in the child */
 		close(fd[0]);
-		ret = ctdb_lock_all_databases(ctdb);
+		ret = ctdb_lock_all_databases(ctdb, priority);
 		if (ret != 0) {
 			_exit(0);
 		}
@@ -214,27 +220,34 @@ static struct ctdb_freeze_handle *ctdb_freeze_lock(struct ctdb_context *ctdb)
  */
 static int ctdb_freeze_waiter_destructor(struct ctdb_freeze_waiter *w)
 {
-	DLIST_REMOVE(w->ctdb->freeze_handle->waiters, w);
+	DLIST_REMOVE(w->ctdb->freeze_handles[w->priority]->waiters, w);
 	ctdb_request_control_reply(w->ctdb, w->c, NULL, w->status, NULL);
 	return 0;
 }
 
 /*
-  start the freeze process
+  start the freeze process for a certain priority
  */
-void ctdb_start_freeze(struct ctdb_context *ctdb)
+int ctdb_start_freeze(struct ctdb_context *ctdb, uint32_t priority)
 {
-	if (ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+	if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+		DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
+		return -1;
+	}
+
+	if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 		/* we're already frozen */
-		return;
+		return 0;
 	}
 
 	/* if there isn't a freeze lock child then create one */
-	if (!ctdb->freeze_handle) {
-		ctdb->freeze_handle = ctdb_freeze_lock(ctdb);
-		CTDB_NO_MEMORY_VOID(ctdb, ctdb->freeze_handle);
-		ctdb->freeze_mode = CTDB_FREEZE_PENDING;
+	if (!ctdb->freeze_handles[priority]) {
+		ctdb->freeze_handles[priority] = ctdb_freeze_lock(ctdb, priority);
+		CTDB_NO_MEMORY(ctdb, ctdb->freeze_handles[priority]);
+		ctdb->freeze_mode[priority] = CTDB_FREEZE_PENDING;
 	}
+
+	return 0;
 }
 
 /*
@@ -247,21 +260,32 @@ int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *
 
 	priority = (uint32_t)c->srvid;
 
-	if (ctdb->freeze_mode == CTDB_FREEZE_FROZEN) {
+	DEBUG(DEBUG_ERR, ("Freeze priority %u\n", priority));
+
+	if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+		DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
+		return -1;
+	}
+
+	if (ctdb->freeze_mode[priority] == CTDB_FREEZE_FROZEN) {
 		/* we're already frozen */
 		return 0;
 	}
 
-	ctdb_start_freeze(ctdb);
+	if (ctdb_start_freeze(ctdb, priority) != 0) {
+		DEBUG(DEBUG_ERR,(__location__ " Failed to start freezing databases with priority %u\n", priority));
+		return -1;
+	}
 
 	/* add ourselves to list of waiters */
-	w = talloc(ctdb->freeze_handle, struct ctdb_freeze_waiter);
+	w = talloc(ctdb->freeze_handles[priority], struct ctdb_freeze_waiter);
 	CTDB_NO_MEMORY(ctdb, w);
-	w->ctdb   = ctdb;
-	w->c      = talloc_steal(w, c);
-	w->status = -1;
+	w->ctdb     = ctdb;
+	w->c        = talloc_steal(w, c);
+	w->priority = priority;
+	w->status   = -1;
 	talloc_set_destructor(w, ctdb_freeze_waiter_destructor);
-	DLIST_ADD(ctdb->freeze_handle->waiters, w);
+	DLIST_ADD(ctdb->freeze_handles[priority]->waiters, w);
 
 	/* we won't reply till later */
 	*async_reply = True;
@@ -274,29 +298,30 @@ int32_t ctdb_control_freeze(struct ctdb_context *ctdb, struct ctdb_req_control *
  */
 bool ctdb_blocking_freeze(struct ctdb_context *ctdb)
 {
-	ctdb_start_freeze(ctdb);
+	int i;
+
+	for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb_start_freeze(ctdb, i)) {
+			DEBUG(DEBUG_ERR,(__location__ " Failed to freeze databases of prio %u\n", i));
+			continue;
+		}
 
-	/* block until frozen */
-	while (ctdb->freeze_mode == CTDB_FREEZE_PENDING) {
-		event_loop_once(ctdb->ev);
+		/* block until frozen */
+		while (ctdb->freeze_mode[i] == CTDB_FREEZE_PENDING) {
+			event_loop_once(ctdb->ev);
+		}
 	}
 
-	return ctdb->freeze_mode == CTDB_FREEZE_FROZEN;
+	return 0;
 }
 
 
-
-/*
-  thaw the databases
- */
-int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
+static void thaw_priority(struct ctdb_context *ctdb, uint32_t priority)
 {
-	uint32_t priority;
-
-	priority = (uint32_t)c->srvid;
+	DEBUG(DEBUG_ERR,("Thawing priority %u\n", priority));
 
 	/* cancel any pending transactions */
-	if (ctdb->freeze_handle && ctdb->freeze_handle->transaction_started) {
+	if (ctdb->freeze_transaction_started) {
 		struct ctdb_db_context *ctdb_db;
 
 		for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
@@ -308,6 +333,7 @@ int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
 			tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 		}
 	}
+	ctdb->freeze_transaction_started = false;
 
 #if 0
 	/* this hack can be used to get a copy of the databases at the end of a recovery */
@@ -319,9 +345,30 @@ int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
 	system("mkdir -p test.db.saved; /usr/bin/rsync --delete -a test.db/ test.db.saved/$$ 2>&1 > /dev/null");
 #endif
 
+	talloc_free(ctdb->freeze_handles[priority]);
+	ctdb->freeze_handles[priority] = NULL;
+}
+
+/*
+  thaw the databases
+ */
+int32_t ctdb_control_thaw(struct ctdb_context *ctdb, uint32_t priority)
+{
+
+	if (priority > NUM_DB_PRIORITIES) {
+		DEBUG(DEBUG_ERR,(__location__ " Invalid db priority : %u\n", priority));
+		return -1;
+	}
+
+	if (priority == 0) {
+		int i;
+		for (i=1;i<=NUM_DB_PRIORITIES; i++) {
+			thaw_priority(ctdb, i);
+		}
+	} else {
+		thaw_priority(ctdb, priority);
+	}
 
-	talloc_free(ctdb->freeze_handle);
-	ctdb->freeze_handle = NULL;
 	ctdb_call_resend_all(ctdb);
 	return 0;
 }
@@ -333,19 +380,21 @@ int32_t ctdb_control_thaw(struct ctdb_context *ctdb, struct ctdb_req_control *c)
 int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 {
 	struct ctdb_db_context *ctdb_db;
+	int i;
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
-		return -1;
+	for (i=1;i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+			DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
+			return -1;
+		}
 	}
 
-
 	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
 		int ret;
 
 		tdb_add_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 
-		if (ctdb->freeze_handle->transaction_started) {
+		if (ctdb->freeze_transaction_started) {
 			if (tdb_transaction_cancel(ctdb_db->ltdb->tdb) != 0) {
 				DEBUG(DEBUG_ERR,(__location__ " Failed to cancel transaction for db '%s'\n",
 					 ctdb_db->db_name));
@@ -364,8 +413,8 @@ int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 		}
 	}
 
-	ctdb->freeze_handle->transaction_started = true;
-	ctdb->freeze_handle->transaction_id = id;
+	ctdb->freeze_transaction_started = true;
+	ctdb->freeze_transaction_id = id;
 
 	return 0;
 }
@@ -376,18 +425,21 @@ int32_t ctdb_control_transaction_start(struct ctdb_context *ctdb, uint32_t id)
 int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 {
 	struct ctdb_db_context *ctdb_db;
-
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
-		return -1;
+	int i;
+	
+	for (i=1;i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+			DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
+			return -1;
+		}
 	}
 
-	if (!ctdb->freeze_handle->transaction_started) {
+	if (!ctdb->freeze_transaction_started) {
 		DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 		return -1;
 	}
 
-	if (id != ctdb->freeze_handle->transaction_id) {
+	if (id != ctdb->freeze_transaction_id) {
 		DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", id));
 		return -1;
 	}
@@ -407,15 +459,15 @@ int32_t ctdb_control_transaction_commit(struct ctdb_context *ctdb, uint32_t id)
 				}
 				tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 			}
-			ctdb->freeze_handle->transaction_started = false;
+			ctdb->freeze_transaction_started = false;
 
 			return -1;
 		}
 		tdb_remove_flags(ctdb_db->ltdb->tdb, TDB_NOLOCK);
 	}
 
-	ctdb->freeze_handle->transaction_started = false;
-	ctdb->freeze_handle->transaction_id = 0;
+	ctdb->freeze_transaction_started = false;
+	ctdb->freeze_transaction_id = 0;
 
 	return 0;
 }
@@ -428,24 +480,24 @@ int32_t ctdb_control_wipe_database(struct ctdb_context *ctdb, TDB_DATA indata)
 	struct ctdb_control_wipe_database w = *(struct ctdb_control_wipe_database *)indata.dptr;
 	struct ctdb_db_context *ctdb_db;
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
+	ctdb_db = find_ctdb_db(ctdb, w.db_id);
+	if (!ctdb_db) {
+		DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
 		return -1;
 	}
 
-	if (!ctdb->freeze_handle->transaction_started) {
-		DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
+	if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+		DEBUG(DEBUG_ERR,(__location__ " Failed transaction_start while not frozen\n"));
 		return -1;
 	}
 
-	if (w.transaction_id != ctdb->freeze_handle->transaction_id) {
-		DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
+	if (!ctdb->freeze_transaction_started) {
+		DEBUG(DEBUG_ERR,(__location__ " transaction not started\n"));
 		return -1;
 	}
 
-	ctdb_db = find_ctdb_db(ctdb, w.db_id);
-	if (!ctdb_db) {
-		DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%x\n", w.db_id));
+	if (w.transaction_id != ctdb->freeze_transaction_id) {
+		DEBUG(DEBUG_ERR,(__location__ " incorrect transaction id 0x%x in commit\n", w.transaction_id));
 		return -1;
 	}
 
diff --git a/ctdb/server/ctdb_ltdb_server.c b/ctdb/server/ctdb_ltdb_server.c
index 2a881c5d5a..ec29076c8d 100644
--- a/ctdb/server/ctdb_ltdb_server.c
+++ b/ctdb/server/ctdb_ltdb_server.c
@@ -509,6 +509,11 @@ int32_t ctdb_control_set_db_priority(struct ctdb_context *ctdb, TDB_DATA indata)
 		return -1;
 	}
 
+	if ((db_prio->priority<1) || (db_prio->priority>NUM_DB_PRIORITIES)) {
+		DEBUG(DEBUG_ERR,("Trying to set invalid priority : %u\n", db_prio->priority));
+		return -1;
+	}
+
 	ctdb_db->priority = db_prio->priority;
 	DEBUG(DEBUG_INFO,("Setting DB priority to %u for db 0x%08x\n", db_prio->priority, db_prio->db_id));
 
diff --git a/ctdb/server/ctdb_monitor.c b/ctdb/server/ctdb_monitor.c
index c230c0cd8d..4a554d4555 100644
--- a/ctdb/server/ctdb_monitor.c
+++ b/ctdb/server/ctdb_monitor.c
@@ -287,7 +287,8 @@ int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
 	struct ctdb_node_flag_change *c = (struct ctdb_node_flag_change *)indata.dptr;
 	struct ctdb_node *node;
 	uint32_t old_flags;
-	
+	int i;
+
 	if (c->pnn >= ctdb->num_nodes) {
 		DEBUG(DEBUG_ERR,(__location__ " Node %d is invalid, num_nodes :%d\n", c->pnn, ctdb->num_nodes));
 		return -1;
@@ -338,7 +339,11 @@ int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
 		*/
 		ctdb->vnn_map->generation = INVALID_GENERATION;
 
-		ctdb_start_freeze(ctdb);
+		for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+			if (ctdb_start_freeze(ctdb, i) != 0) {
+				DEBUG(DEBUG_ERR,(__location__ " Failed to freeze db priority %u\n", i));
+			}
+		}
 		ctdb_release_all_ips(ctdb);
 		ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
 	}
diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c
index b77bf58917..451773dea3 100644
--- a/ctdb/server/ctdb_recover.c
+++ b/ctdb/server/ctdb_recover.c
@@ -31,14 +31,23 @@
 /*
   lock all databases - mark only
  */
-static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb)
+static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb, uint32_t priority)
 {
 	struct ctdb_db_context *ctdb_db;
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+
+	if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+		DEBUG(DEBUG_ERR,(__location__ " Illegal priority when trying to mark all databases Prio:%u\n", priority));
+		return -1;
+	}
+
+	if (ctdb->freeze_mode[priority] != CTDB_FREEZE_FROZEN) {
 		DEBUG(DEBUG_ERR,("Attempt to mark all databases locked when not frozen\n"));
 		return -1;
 	}
 	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+		if (ctdb_db->priority != priority) {
+			continue;
+		}
 		if (tdb_lockall_mark(ctdb_db->ltdb->tdb) != 0) {
 			return -1;
 		}
@@ -49,14 +58,23 @@ static int ctdb_lock_all_databases_mark(struct ctdb_context *ctdb)
 /*
   lock all databases - unmark only
  */
-static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb)
+static int ctdb_lock_all_databases_unmark(struct ctdb_context *ctdb, uint32_t priority)
 {
 	struct ctdb_db_context *ctdb_db;
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
+
+	if ((priority < 1) || (priority > NUM_DB_PRIORITIES)) {
+		DEBUG(DEBUG_ERR,(__location__ " Illegal priority when trying to mark all databases Prio:%u\n", priority));
+		return -1;
+	}
+
+	if (ctdb->freeze_mode[priority] != CTDB_FREEZE_FROZEN) {
 		DEBUG(DEBUG_ERR,("Attempt to unmark all databases locked when not frozen\n"));
 		return -1;
 	}
 	for (ctdb_db=ctdb->db_list;ctdb_db;ctdb_db=ctdb_db->next) {
+		if (ctdb_db->priority != priority) {
+			continue;
+		}
 		if (tdb_lockall_unmark(ctdb_db->ltdb->tdb) != 0) {
 			return -1;
 		}
@@ -90,10 +108,13 @@ int
 ctdb_control_setvnnmap(struct ctdb_context *ctdb, uint32_t opcode, TDB_DATA indata, TDB_DATA *outdata)
 {
 	struct ctdb_vnn_map_wire *map = (struct ctdb_vnn_map_wire *)indata.dptr;
+	int i;
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_ERR,("Attempt to set vnnmap when not frozen\n"));
-		return -1;
+	for(i=1; i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+			DEBUG(DEBUG_ERR,("Attempt to set vnnmap when not frozen\n"));
+			return -1;
+		}
 	}
 
 	talloc_free(ctdb->vnn_map);
@@ -322,11 +343,6 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
 	struct pulldb_data params;
 	struct ctdb_marshall_buffer *reply;
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_pull_db when not frozen\n"));
-		return -1;
-	}
-
 	pull = (struct ctdb_control_pulldb *)indata.dptr;
 	
 	ctdb_db = find_ctdb_db(ctdb, pull->db_id);
@@ -335,6 +351,11 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
 		return -1;
 	}
 
+	if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+		DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_pull_db when not frozen\n"));
+		return -1;
+	}
+
 	reply = talloc_zero(outdata, struct ctdb_marshall_buffer);
 	CTDB_NO_MEMORY(ctdb, reply);
 
@@ -345,19 +366,19 @@ int32_t ctdb_control_pull_db(struct ctdb_context *ctdb, TDB_DATA indata, TDB_DAT
 	params.len = offsetof(struct ctdb_marshall_buffer, data);
 	params.failed = false;
 
-	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+	if (ctdb_lock_all_databases_mark(ctdb, ctdb_db->priority) != 0) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
 		return -1;
 	}
 
 	if (tdb_traverse_read(ctdb_db->ltdb->tdb, traverse_pulldb, &params) == -1) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to get traverse db '%s'\n", ctdb_db->db_name));
-		ctdb_lock_all_databases_unmark(ctdb);
+		ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
 		talloc_free(params.pulldata);
 		return -1;
 	}
 
-	ctdb_lock_all_databases_unmark(ctdb);
+	ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
 
 	outdata->dptr = (uint8_t *)params.pulldata;
 	outdata->dsize = params.len;
@@ -375,11 +396,6 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
 	int i, ret;
 	struct ctdb_rec_data *rec;
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_push_db when not frozen\n"));
-		return -1;
-	}
-
 	if (indata.dsize < offsetof(struct ctdb_marshall_buffer, data)) {
 		DEBUG(DEBUG_ERR,(__location__ " invalid data in pulldb reply\n"));
 		return -1;
@@ -391,7 +407,12 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
 		return -1;
 	}
 
-	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+	if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+		DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_push_db when not frozen\n"));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb, ctdb_db->priority) != 0) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
 		return -1;
 	}
@@ -430,11 +451,11 @@ int32_t ctdb_control_push_db(struct ctdb_context *ctdb, TDB_DATA indata)
 	DEBUG(DEBUG_DEBUG,("finished push of %u records for dbid 0x%x\n",
 		 reply->count, reply->db_id));
 
-	ctdb_lock_all_databases_unmark(ctdb);
+	ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
 	return 0;
 
 failed:
-	ctdb_lock_all_databases_unmark(ctdb);
+	ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
 	return -1;
 }
 
@@ -468,25 +489,25 @@ int32_t ctdb_control_set_dmaster(struct ctdb_context *ctdb, TDB_DATA indata)
 	struct ctdb_control_set_dmaster *p = (struct ctdb_control_set_dmaster *)indata.dptr;
 	struct ctdb_db_context *ctdb_db;
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_set_dmaster when not frozen\n"));
-		return -1;
-	}
-
 	ctdb_db = find_ctdb_db(ctdb, p->db_id);
 	if (!ctdb_db) {
 		DEBUG(DEBUG_ERR,(__location__ " Unknown db 0x%08x\n", p->db_id));
 		return -1;
 	}
 
-	if (ctdb_lock_all_databases_mark(ctdb) != 0) {
+	if (ctdb->freeze_mode[ctdb_db->priority] != CTDB_FREEZE_FROZEN) {
+		DEBUG(DEBUG_DEBUG,("rejecting ctdb_control_set_dmaster when not frozen\n"));
+		return -1;
+	}
+
+	if (ctdb_lock_all_databases_mark(ctdb, 	ctdb_db->priority) != 0) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to get lock on entired db - failing\n"));
 		return -1;
 	}
 
 	tdb_traverse(ctdb_db->ltdb->tdb, traverse_setdmaster, &p->dmaster);
 
-	ctdb_lock_all_databases_unmark(ctdb);
+	ctdb_lock_all_databases_unmark(ctdb, ctdb_db->priority);
 	
 	return 0;
 }
@@ -603,7 +624,7 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
 				 const char **errormsg)
 {
 	uint32_t recmode = *(uint32_t *)indata.dptr;
-	int ret;
+	int i, ret;
 	struct ctdb_set_recmode_state *state;
 	pid_t parent = getpid();
 
@@ -622,11 +643,13 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
 	}
 
 
-	if (ctdb->freeze_mode != CTDB_FREEZE_FROZEN) {
-		DEBUG(DEBUG_ERR,("Attempt to change recovery mode to %u when not frozen\n", 
-			 recmode));
-		(*errormsg) = "Cannot change recovery mode while not frozen";
-		return -1;
+	for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb->freeze_mode[i] != CTDB_FREEZE_FROZEN) {
+			DEBUG(DEBUG_ERR,("Attempt to change recovery mode to %u when not frozen\n", 
+				 recmode));
+			(*errormsg) = "Cannot change recovery mode while not frozen";
+			return -1;
+		}
 	}
 
 	if (recmode != ctdb->recovery_mode) {
@@ -642,9 +665,11 @@ int32_t ctdb_control_set_recmode(struct ctdb_context *ctdb,
 
 	/* some special handling when ending recovery mode */
 
-	/* force the databased to thaw */
-	if (ctdb->freeze_handle) {
-		ctdb_control_thaw(ctdb);
+	/* force the databases to thaw */
+	for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+		if (ctdb->freeze_handles[i] != NULL) {
+			ctdb_control_thaw(ctdb, i);
+		}
 	}
 
 	state = talloc(ctdb, struct ctdb_set_recmode_state);
diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index 15181224b8..879f83db2a 100644
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -113,7 +113,7 @@ static int run_recovered_eventscript(struct ctdb_context *ctdb, struct ctdb_node
 
 	nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_END_RECOVERY,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, tdb_null,
 					NULL, NULL,
 					NULL) != 0) {
@@ -194,7 +194,7 @@ static int run_startrecovery_eventscript(struct ctdb_recoverd *rec, struct ctdb_
 
 	nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_START_RECOVERY,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, tdb_null,
 					NULL,
 					startrecovery_fail_callback,
@@ -232,7 +232,8 @@ static int update_capabilities(struct ctdb_context *ctdb, struct ctdb_node_map *
 
 	nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_CAPABILITIES,
-					nodes, CONTROL_TIMEOUT(),
+					nodes, 0,
+					CONTROL_TIMEOUT(),
 					false, tdb_null,
 					async_getcap_callback, NULL,
 					NULL) != 0) {
@@ -268,15 +269,20 @@ static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_recoverd *re
 	/* freeze all nodes */
 	nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
 	if (rec_mode == CTDB_RECOVERY_ACTIVE) {
-		if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
-						nodes, CONTROL_TIMEOUT(),
+		int i;
+
+		for (i=1; i<=NUM_DB_PRIORITIES; i++) {
+			if (ctdb_client_async_control(ctdb, CTDB_CONTROL_FREEZE,
+						nodes, i,
+						CONTROL_TIMEOUT(),
 						false, tdb_null,
 						NULL,
 						set_recmode_fail_callback,
 						rec) != 0) {
-			DEBUG(DEBUG_ERR, (__location__ " Unable to freeze nodes. Recovery failed.\n"));
-			talloc_free(tmp_ctx);
-			return -1;
+				DEBUG(DEBUG_ERR, (__location__ " Unable to freeze nodes. Recovery failed.\n"));
+				talloc_free(tmp_ctx);
+				return -1;
+			}
 		}
 	}
 
@@ -285,7 +291,8 @@ static int set_recovery_mode(struct ctdb_context *ctdb, struct ctdb_recoverd *re
 	data.dptr = (unsigned char *)&rec_mode;
 
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECMODE,
-					nodes, CONTROL_TIMEOUT(),
+					nodes, 0,
+					CONTROL_TIMEOUT(),
 					false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -315,7 +322,7 @@ static int set_recovery_master(struct ctdb_context *ctdb, struct ctdb_node_map *
 
 	nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_SET_RECMASTER,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -362,7 +369,7 @@ static int update_db_priority_on_remote_nodes(struct ctdb_context *ctdb,
 
 		if (ctdb_client_async_control(ctdb,
 					CTDB_CONTROL_SET_DB_PRIORITY,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -1108,7 +1115,7 @@ static int push_recdb_database(struct ctdb_context *ctdb, uint32_t dbid,
 
 	nodes = list_of_active_nodes(ctdb, nodemap, tmp_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_PUSH_DB,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, outdata,
 					NULL, NULL,
 					NULL) != 0) {
@@ -1168,7 +1175,7 @@ static int recover_database(struct ctdb_recoverd *rec,
 
 	nodes = list_of_active_nodes(ctdb, nodemap, recdb, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_WIPE_DATABASE,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -1325,7 +1332,7 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
 	nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_START,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -1346,7 +1353,7 @@ static int do_recovery(struct ctdb_recoverd *rec,
 
 	/* commit all the changes */
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_TRANSACTION_COMMIT,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, data,
 					NULL, NULL,
 					NULL) != 0) {
@@ -2351,7 +2358,7 @@ static int get_remote_nodemaps(struct ctdb_context *ctdb, TALLOC_CTX *mem_ctx,
 
 	nodes = list_of_active_nodes(ctdb, nodemap, mem_ctx, true);
 	if (ctdb_client_async_control(ctdb, CTDB_CONTROL_GET_NODEMAP,
-					nodes,
+					nodes, 0,
 					CONTROL_TIMEOUT(), false, tdb_null,
 					async_getnodemap_callback,
 					NULL,