add a new command "ctdb ipreallocate", this command will force the recovery master to perform a full ip reallocation process.

the ctdb command will block until the ip reallocation has comleted

(This used to be ctdb commit abad7b97fe0c066b33f6e75d0953bbed892a3216)

3 files changed, 144 insertions, 0 deletions

diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h
index 53669b255f..c410923fa9 100644
--- a/ctdb/include/ctdb.h
+++ b/ctdb/include/ctdb.h
@@ -106,6 +106,11 @@ struct ctdb_call_info {
  */
 #define CTDB_SRVID_RELOAD_NODES 0xFA00000000000000LL
 
+/* 
+   a message ID to get the recovery daemon to perform a takeover run
+ */
+#define CTDB_SRVID_TAKEOVER_RUN 0xFB00000000000000LL
+
 
 
 /* used on the domain socket, send a pdu to the local daemon */
diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index a367630f32..fdb2881ba7 100644
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -36,6 +36,14 @@ struct ban_state {
 	uint32_t banned_node;
 };
 
+/* list of "ctdb ipreallocate" processes to call back when we have
+   finished the takeover run.
+*/
+struct ip_reallocate_list {
+	struct ip_reallocate_list *next;
+	struct rd_memdump_reply *rd;
+};
+
 /*
   private state of recovery daemon
  */
@@ -56,6 +64,8 @@ struct ctdb_recoverd {
 	struct timed_event *send_election_te;
 	struct timed_event *election_timeout;
 	struct vacuum_info *vacuum_info;
+	TALLOC_CTX *ip_reallocate_ctx;
+	struct ip_reallocate_list *reallocate_callers;
 };
 
 #define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
@@ -1813,6 +1823,63 @@ static void reload_nodes_handler(struct ctdb_context *ctdb, uint64_t srvid,
 	reload_nodes_file(rec->ctdb);
 }
 
+/*
+  handler for ip reallocate, just add it to the list of callers and 
+  handle this later in the monitor_cluster loop so we do not recurse
+  with other callers to takeover_run()
+*/
+static void ip_reallocate_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			     TDB_DATA data, void *private_data)
+{
+	struct ctdb_recoverd *rec = talloc_get_type(private_data, struct ctdb_recoverd);
+	struct ip_reallocate_list *caller;
+
+	if (data.dsize != sizeof(struct rd_memdump_reply)) {
+		DEBUG(DEBUG_ERR, (__location__ " Wrong size of return address.\n"));
+		return;
+	}
+
+	if (rec->ip_reallocate_ctx == NULL) {
+		rec->ip_reallocate_ctx = talloc_new(rec);
+		CTDB_NO_MEMORY_FATAL(ctdb, caller);
+	}
+
+	caller = talloc(rec->ip_reallocate_ctx, struct ip_reallocate_list);
+	CTDB_NO_MEMORY_FATAL(ctdb, caller);
+
+	caller->rd   = (struct rd_memdump_reply *)talloc_steal(caller, data.dptr);
+	caller->next = rec->reallocate_callers;
+	rec->reallocate_callers = caller;
+
+	return;
+}
+
+static void process_ipreallocate_requests(struct ctdb_context *ctdb, struct ctdb_recoverd *rec)
+{
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+	TDB_DATA result;
+	int32_t ret;
+	struct ip_reallocate_list *callers;
+
+	DEBUG(DEBUG_INFO, ("recovery master forced ip reallocation\n"));
+	ret = ctdb_takeover_run(ctdb, rec->nodemap);
+	result.dsize = sizeof(int32_t);
+	result.dptr  = (uint8_t *)&ret;
+
+	for (callers=rec->reallocate_callers; callers; callers=callers->next) {
+		DEBUG(DEBUG_INFO,("Sending ip reallocate reply message to %u:%lu\n", callers->rd->pnn, callers->rd->srvid));
+		ret = ctdb_send_message(ctdb, callers->rd->pnn, callers->rd->srvid, result);
+		if (ret != 0) {
+			DEBUG(DEBUG_ERR,("Failed to send ip reallocate reply message to %u:%lu\n", callers->rd->pnn, callers->rd->srvid));
+		}
+	}
+
+	talloc_free(tmp_ctx);
+	talloc_free(rec->ip_reallocate_ctx);
+	rec->ip_reallocate_ctx = NULL;
+	rec->reallocate_callers = NULL;
+	
+}
 
 
 /*
@@ -2611,6 +2678,9 @@ static void monitor_cluster(struct ctdb_context *ctdb)
 	/* register a message port for reloadnodes  */
 	ctdb_set_message_handler(ctdb, CTDB_SRVID_RELOAD_NODES, reload_nodes_handler, rec);
 
+	/* register a message port for performing a takeover run */
+	ctdb_set_message_handler(ctdb, CTDB_SRVID_TAKEOVER_RUN, ip_reallocate_handler, rec);
+
 again:
 	if (mem_ctx) {
 		talloc_free(mem_ctx);
@@ -2716,6 +2786,19 @@ again:
 		goto again;
 	}
 
+	/* if we are not the recmaster we can safely ignore any ip reallocate requests */
+	if (rec->recmaster != pnn) {
+		if (rec->ip_reallocate_ctx != NULL) {
+			talloc_free(rec->ip_reallocate_ctx);
+			rec->ip_reallocate_ctx = NULL;
+			rec->reallocate_callers = NULL;
+		}
+	}
+	/* if there are takeovers requested, perform it and notify the waiters */
+	if (rec->reallocate_callers) {
+		process_ipreallocate_requests(ctdb, rec);
+	}
+
 	if (rec->recmaster == (uint32_t)-1) {
 		DEBUG(DEBUG_NOTICE,(__location__ " Initial recovery master set - forcing election\n"));
 		force_election(rec, pnn, nodemap);
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index 5ca013cb50..5d00da9eea 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -2859,6 +2859,61 @@ static int control_rddumpmemory(struct ctdb_context *ctdb, int argc, const char
 }
 
 /*
+  handler for receiving the response to ipreallocate
+*/
+static void ip_reallocate_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			     TDB_DATA data, void *private_data)
+{
+	printf("IP Reallocation completed\n");
+	exit(0);
+}
+
+/*
+  ask the recovery daemon on the recovery master to perform a ip reallocation
+ */
+static int control_ipreallocate(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	TDB_DATA data;
+	struct rd_memdump_reply rd;
+	uint32_t recmaster;
+
+	rd.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
+	if (rd.pnn == -1) {
+		DEBUG(DEBUG_ERR, ("Failed to get pnn of local node\n"));
+		return -1;
+	}
+	rd.srvid = getpid();
+
+	/* register a message port for receiveing the reply so that we
+	   can receive the reply
+	*/
+	ctdb_set_message_handler(ctdb, rd.srvid, ip_reallocate_handler, NULL);
+
+	data.dptr = (uint8_t *)&rd;
+	data.dsize = sizeof(rd);
+
+	ret = ctdb_ctrl_getrecmaster(ctdb, ctdb, TIMELIMIT(), options.pnn, &recmaster);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR, ("Unable to get recmaster from node %u\n", options.pnn));
+		return ret;
+	}
+
+	ret = ctdb_send_message(ctdb, recmaster, CTDB_SRVID_TAKEOVER_RUN, data);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR,("Failed to send ip takeover run request message to %u\n", options.pnn));
+		return -1;
+	}
+
+	/* this loop will terminate when we have received the reply */
+	while (1) {	
+		event_loop_once(ctdb->ev);
+	}
+
+	return 0;
+}
+
+/*
   list all nodes in the cluster
   if the daemon is running, we read the data from the daemon.
   if the daemon is not running we parse the nodes file directly
@@ -3008,6 +3063,7 @@ static const struct {
 	{ "unban",           control_unban,             true,	false,  "unban a node from the cluster" },
 	{ "shutdown",        control_shutdown,          true,	false,  "shutdown ctdbd" },
 	{ "recover",         control_recover,           true,	false,  "force recovery" },
+	{ "ipreallocate",    control_ipreallocate,      true,	false,  "force the recovery daemon to perform a ip reallocation procedure" },
 	{ "freeze",          control_freeze,            true,	false,  "freeze all databases" },
 	{ "thaw",            control_thaw,              true,	false,  "thaw all databases" },
 	{ "isnotrecmaster",  control_isnotrecmaster,    false,	false,  "check if the local node is recmaster or not" },