ctdbd: Add new runstate CTDB_RUNSTATE_FIRST_RECOVERY

This adds more serialisation to the startup, ensuring that the
"startup" event runs after everything to do with the first recovery
(including the "recovered" event).

Given that it now takes longer to get to the "startup" state, the
initscript needs to wait until ctdbd gets to "first_recovery".

Signed-off-by: Martin Schwenke <martin@meltin.net>
Pair-programmed-with: Amitay Isaacs <amitay@gmail.com>

(This used to be ctdb commit ed6814ff0a59ddbb1c1b3128b505380f60d7aeb7)

8 files changed, 19 insertions, 5 deletions

diff --git a/ctdb/common/ctdb_util.c b/ctdb/common/ctdb_util.c
index 270ad625c19..a910a0cd79c 100644
--- a/ctdb/common/ctdb_util.c
+++ b/ctdb/common/ctdb_util.c
@@ -711,6 +711,7 @@ static struct {
 	{ CTDB_RUNSTATE_UNKNOWN, "UNKNOWN" },
 	{ CTDB_RUNSTATE_INIT, "INIT" },
 	{ CTDB_RUNSTATE_SETUP, "SETUP" },
+	{ CTDB_RUNSTATE_FIRST_RECOVERY, "FIRST_RECOVERY" },
 	{ CTDB_RUNSTATE_STARTUP, "STARTUP" },
 	{ CTDB_RUNSTATE_RUNNING, "RUNNING" },
 	{ CTDB_RUNSTATE_SHUTDOWN, "SHUTDOWN" },
diff --git a/ctdb/config/ctdb.init b/ctdb/config/ctdb.init
index 70dcfa64216..2ceb45ff191 100755
--- a/ctdb/config/ctdb.init
+++ b/ctdb/config/ctdb.init
@@ -220,7 +220,7 @@ wait_until_ready () {
     _timeout="${1:-10}" # default is 10 seconds
 
     _count=0
-    while ! ctdb runstate startup running >/dev/null 2>&1 ; do
+    while ! ctdb runstate first_recovery startup running >/dev/null 2>&1 ; do
 	if [ $_count -ge $_timeout ] ; then
 	    return 1
 	fi
diff --git a/ctdb/doc/ctdb.1.xml b/ctdb/doc/ctdb.1.xml
index 7242f3aa29a..ce83a3efa0b 100644
--- a/ctdb/doc/ctdb.1.xml
+++ b/ctdb/doc/ctdb.1.xml
@@ -382,7 +382,7 @@ response from 3 time=0.000114 sec  (2 clients)
       </screen>
     </refsect2>
 
-    <refsect2><title>runstate [setup|startup|running]</title>
+    <refsect2><title>runstate [setup|first_recovery|startup|running]</title>
       <para>
         Print the runstate of the specified node.  Runstates are used
         to serialise important state transitions in CTDB, particularly
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 2698785f0be..eadd9637155 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -440,6 +440,7 @@ enum ctdb_runstate {
 	CTDB_RUNSTATE_UNKNOWN,
 	CTDB_RUNSTATE_INIT,
 	CTDB_RUNSTATE_SETUP,
+	CTDB_RUNSTATE_FIRST_RECOVERY,
 	CTDB_RUNSTATE_STARTUP,
 	CTDB_RUNSTATE_RUNNING,
 	CTDB_RUNSTATE_SHUTDOWN,
diff --git a/ctdb/server/ctdb_daemon.c b/ctdb/server/ctdb_daemon.c
index edbc34c7a1f..32c30243ea6 100644
--- a/ctdb/server/ctdb_daemon.c
+++ b/ctdb/server/ctdb_daemon.c
@@ -1037,7 +1037,7 @@ static void ctdb_setup_event_callback(struct ctdb_context *ctdb, int status,
 	}
 	ctdb_run_notification_script(ctdb, "setup");
 
-	ctdb_set_runstate(ctdb, CTDB_RUNSTATE_STARTUP);
+	ctdb_set_runstate(ctdb, CTDB_RUNSTATE_FIRST_RECOVERY);
 
 	/* tell all other nodes we've just started up */
 	ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL,
diff --git a/ctdb/server/ctdb_monitor.c b/ctdb/server/ctdb_monitor.c
index 1e556e00d87..1608804c308 100644
--- a/ctdb/server/ctdb_monitor.c
+++ b/ctdb/server/ctdb_monitor.c
@@ -307,7 +307,6 @@ static void ctdb_wait_until_recovered(struct event_context *ev, struct timed_eve
 	}
 	ctdb->db_persistent_check_errors = 0;
 
-	DEBUG(DEBUG_NOTICE,(__location__ " Recoveries finished. Running the \"startup\" event.\n"));
 	event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 			     timeval_current(),
 			     ctdb_check_health, ctdb);
@@ -323,6 +322,14 @@ static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
 	struct ctdb_context *ctdb = talloc_get_type(private_data, struct ctdb_context);
 	int ret = 0;
 
+	if (ctdb->runstate < CTDB_RUNSTATE_STARTUP) {
+		DEBUG(DEBUG_NOTICE,("Not yet in startup runstate. Wait one more second\n"));
+		event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
+				timeval_current_ofs(1, 0), 
+				ctdb_check_health, ctdb);
+		return;
+	}
+	
 	if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL ||
 	    (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED &&
 	     ctdb->runstate == CTDB_RUNSTATE_RUNNING)) {
@@ -333,6 +340,7 @@ static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
 	}
 	
 	if (ctdb->runstate == CTDB_RUNSTATE_STARTUP) {
+		DEBUG(DEBUG_NOTICE,("Recoveries finished. Running the \"startup\" event.\n"));
 		ret = ctdb_event_script_callback(ctdb, 
 						 ctdb->monitor->monitor_context, ctdb_startup_callback, 
 						 ctdb, false,
diff --git a/ctdb/server/ctdb_recover.c b/ctdb/server/ctdb_recover.c
index 7fd0ac36ea1..6506f8bbce3 100644
--- a/ctdb/server/ctdb_recover.c
+++ b/ctdb/server/ctdb_recover.c
@@ -911,6 +911,10 @@ static void ctdb_end_recovery_callback(struct ctdb_context *ctdb, int status, vo
 	talloc_free(state);
 
 	gettimeofday(&ctdb->last_recovery_finished, NULL);
+
+	if (ctdb->runstate == CTDB_RUNSTATE_FIRST_RECOVERY) {
+		ctdb_set_runstate(ctdb, CTDB_RUNSTATE_STARTUP);
+	}
 }
 
 /*
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index e21e8450736..48ca93ba0ee 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -5847,7 +5847,7 @@ static const struct {
 	{ "status",          control_status,            true,	false,  "show node status" },
 	{ "uptime",          control_uptime,            true,	false,  "show node uptime" },
 	{ "ping",            control_ping,              true,	false,  "ping all nodes" },
-	{ "runstate",        control_runstate,          true,	false,  "get/check runstate of a node", "[setup|startup|running]" },
+	{ "runstate",        control_runstate,          true,	false,  "get/check runstate of a node", "[setup|first_recovery|startup|running]" },
 	{ "getvar",          control_getvar,            true,	false,  "get a tunable variable",               "<name>"},
 	{ "setvar",          control_setvar,            true,	false,  "set a tunable variable",               "<name> <value>"},
 	{ "listvars",        control_listvars,          true,	false,  "list tunable variables"},