ctdbd: Replace ctdb->done_startup with ctdb->runstate

This allows states, including startup and shutdown states, to be
clearly tracked.  This doesn't include regular runtime "states", which
are handled by node flags.

Introduce new functions ctdb_set_runstate(), runstate_to_string() and
runstate_from_string().

Signed-off-by: Martin Schwenke <martin@meltin.net>
Pair-programmed-with: Amitay Isaacs <amitay@gmail.com>

(This used to be ctdb commit 8076773a9924dcf8aff16f7d96b2b9ac383ecc28)

7 files changed, 80 insertions, 11 deletions

diff --git a/ctdb/common/ctdb_util.c b/ctdb/common/ctdb_util.c
index 71dee2b9544..270ad625c19 100644
--- a/ctdb/common/ctdb_util.c
+++ b/ctdb/common/ctdb_util.c
@@ -702,3 +702,52 @@ const char *ctdb_eventscript_call_names[] = {
 	"updateip",
 	"ipreallocated"
 };
+
+/* Runstate handling */
+static struct {
+	enum ctdb_runstate runstate;
+	const char * label;
+} runstate_map[] = {
+	{ CTDB_RUNSTATE_UNKNOWN, "UNKNOWN" },
+	{ CTDB_RUNSTATE_INIT, "INIT" },
+	{ CTDB_RUNSTATE_SETUP, "SETUP" },
+	{ CTDB_RUNSTATE_STARTUP, "STARTUP" },
+	{ CTDB_RUNSTATE_RUNNING, "RUNNING" },
+	{ CTDB_RUNSTATE_SHUTDOWN, "SHUTDOWN" },
+	{ -1, NULL },
+};
+
+const char *runstate_to_string(enum ctdb_runstate runstate)
+{
+	int i;
+	for (i=0; runstate_map[i].label != NULL ; i++) {
+		if (runstate_map[i].runstate == runstate) {
+			return runstate_map[i].label;
+		}
+	}
+
+	return runstate_map[0].label;
+}
+
+enum ctdb_runstate runstate_from_string(const char *label)
+{
+	int i;
+	for (i=0; runstate_map[i].label != NULL; i++) {
+		if (strcasecmp(runstate_map[i].label, label) == 0) {
+			return runstate_map[i].runstate;
+		}
+	}
+
+	return CTDB_RUNSTATE_UNKNOWN;
+}
+
+void ctdb_set_runstate(struct ctdb_context *ctdb, enum ctdb_runstate runstate)
+{
+	if (runstate <= ctdb->runstate) {
+		ctdb_fatal(ctdb, "runstate must always increase");
+	}
+
+	DEBUG(DEBUG_NOTICE,("Set runstate to %s (%d)\n",
+			    runstate_to_string(runstate), runstate));
+	ctdb->runstate = runstate;
+}
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index c47210e1f0a..2698785f0be 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -436,6 +436,19 @@ struct ctdb_write_record {
 
 enum ctdb_freeze_mode {CTDB_FREEZE_NONE, CTDB_FREEZE_PENDING, CTDB_FREEZE_FROZEN};
 
+enum ctdb_runstate {
+	CTDB_RUNSTATE_UNKNOWN,
+	CTDB_RUNSTATE_INIT,
+	CTDB_RUNSTATE_SETUP,
+	CTDB_RUNSTATE_STARTUP,
+	CTDB_RUNSTATE_RUNNING,
+	CTDB_RUNSTATE_SHUTDOWN,
+};
+
+const char *runstate_to_string(enum ctdb_runstate runstate);
+enum ctdb_runstate runstate_from_string(const char *label);
+void ctdb_set_runstate(struct ctdb_context *ctdb, enum ctdb_runstate runstate);
+
 #define CTDB_MONITORING_ACTIVE		0
 #define CTDB_MONITORING_DISABLED	1
 
@@ -505,7 +518,7 @@ struct ctdb_context {
 	pid_t ctdbd_pid;
 	pid_t recoverd_pid;
 	pid_t syslogd_pid;
-	bool done_startup;
+	enum ctdb_runstate runstate;
 	struct ctdb_monitor_state *monitor;
 	struct ctdb_log_state *log;
 	int start_as_disabled;
diff --git a/ctdb/server/ctdb_control.c b/ctdb/server/ctdb_control.c
index 72a602d98d7..5e0c27581f5 100644
--- a/ctdb/server/ctdb_control.c
+++ b/ctdb/server/ctdb_control.c
@@ -325,6 +325,7 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 
 	case CTDB_CONTROL_SHUTDOWN:
 		DEBUG(DEBUG_NOTICE,("Received SHUTDOWN command. Stopping CTDB daemon.\n"));
+		ctdb_set_runstate(ctdb, CTDB_RUNSTATE_SHUTDOWN);
 		ctdb_stop_recoverd(ctdb);
 		ctdb_stop_keepalive(ctdb);
 		ctdb_stop_monitoring(ctdb);
diff --git a/ctdb/server/ctdb_daemon.c b/ctdb/server/ctdb_daemon.c
index d2df1150a7c..bcead6f4699 100644
--- a/ctdb/server/ctdb_daemon.c
+++ b/ctdb/server/ctdb_daemon.c
@@ -1054,6 +1054,8 @@ static void ctdb_setup_event_callback(struct ctdb_context *ctdb, int status,
 	}
 	ctdb_run_notification_script(ctdb, "setup");
 
+	ctdb_set_runstate(ctdb, CTDB_RUNSTATE_STARTUP);
+
 	/* tell all other nodes we've just started up */
 	ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_ALL,
 				 0, CTDB_CONTROL_STARTUP, 0,
@@ -1259,6 +1261,7 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog,
 		ctdb_fatal(ctdb, "Failed to attach to databases\n");
 	}
 
+	ctdb_set_runstate(ctdb, CTDB_RUNSTATE_INIT);
 	ret = ctdb_event_script(ctdb, CTDB_EVENT_INIT);
 	if (ret != 0) {
 		ctdb_fatal(ctdb, "Failed to run init event\n");
@@ -1284,6 +1287,8 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog,
 	/* start the transport going */
 	ctdb_start_transport(ctdb);
 
+	ctdb_set_runstate(ctdb, CTDB_RUNSTATE_SETUP);
+
 	ret = ctdb_event_script_callback(ctdb,
 					 ctdb,
 					 ctdb_setup_event_callback,
diff --git a/ctdb/server/ctdb_ltdb_server.c b/ctdb/server/ctdb_ltdb_server.c
index 8b067039235..0426d96bdbc 100644
--- a/ctdb/server/ctdb_ltdb_server.c
+++ b/ctdb/server/ctdb_ltdb_server.c
@@ -656,7 +656,7 @@ int32_t ctdb_control_db_set_healthy(struct ctdb_context *ctdb, TDB_DATA indata)
 		return -1;
 	}
 
-	if (may_recover && !ctdb->done_startup) {
+	if (may_recover && ctdb->runstate == CTDB_RUNSTATE_STARTUP) {
 		DEBUG(DEBUG_ERR, (__location__ " db %s become healthy  - force recovery for startup\n",
 				  ctdb_db->db_name));
 		ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
@@ -794,7 +794,7 @@ static int ctdb_local_attach(struct ctdb_context *ctdb, const char *db_name,
 		if (ctdb->max_persistent_check_errors > 0) {
 			remaining_tries = 1;
 		}
-		if (ctdb->done_startup) {
+		if (ctdb->runstate == CTDB_RUNSTATE_RUNNING) {
 			remaining_tries = 0;
 		}
 
@@ -1086,9 +1086,9 @@ int32_t ctdb_control_db_attach(struct ctdb_context *ctdb, TDB_DATA indata,
 			return -1;
 		}
 
-		if (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE
-		 && client->pid != ctdb->recoverd_pid
-		 && !ctdb->done_startup) {
+		if (ctdb->recovery_mode == CTDB_RECOVERY_ACTIVE &&
+		    client->pid != ctdb->recoverd_pid &&
+		    ctdb->runstate < CTDB_RUNSTATE_RUNNING) {
 			struct ctdb_deferred_attach_context *da_ctx = talloc(client, struct ctdb_deferred_attach_context);
 
 			if (da_ctx == NULL) {
diff --git a/ctdb/server/ctdb_monitor.c b/ctdb/server/ctdb_monitor.c
index 984f947a3b4..1e556e00d87 100644
--- a/ctdb/server/ctdb_monitor.c
+++ b/ctdb/server/ctdb_monitor.c
@@ -204,7 +204,7 @@ static void ctdb_startup_callback(struct ctdb_context *ctdb, int status, void *p
 		DEBUG(DEBUG_ERR,("startup event failed\n"));
 	} else if (status == 0) {
 		DEBUG(DEBUG_NOTICE,("startup event OK - enabling monitoring\n"));
-		ctdb->done_startup = true;
+		ctdb_set_runstate(ctdb, CTDB_RUNSTATE_RUNNING);
 		ctdb->monitor->next_interval = 2;
 		ctdb_run_notification_script(ctdb, "startup");
 	}
@@ -324,14 +324,15 @@ static void ctdb_check_health(struct event_context *ev, struct timed_event *te,
 	int ret = 0;
 
 	if (ctdb->recovery_mode != CTDB_RECOVERY_NORMAL ||
-	    (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED && ctdb->done_startup)) {
+	    (ctdb->monitor->monitoring_mode == CTDB_MONITORING_DISABLED &&
+	     ctdb->runstate == CTDB_RUNSTATE_RUNNING)) {
 		event_add_timed(ctdb->ev, ctdb->monitor->monitor_context,
 				timeval_current_ofs(ctdb->monitor->next_interval, 0), 
 				ctdb_check_health, ctdb);
 		return;
 	}
 	
-	if (!ctdb->done_startup) {
+	if (ctdb->runstate == CTDB_RUNSTATE_STARTUP) {
 		ret = ctdb_event_script_callback(ctdb, 
 						 ctdb->monitor->monitor_context, ctdb_startup_callback, 
 						 ctdb, false,
@@ -477,7 +478,7 @@ int32_t ctdb_control_modflags(struct ctdb_context *ctdb, TDB_DATA indata)
 
 	DEBUG(DEBUG_INFO, ("Control modflags on node %u - flags now 0x%x\n", c->pnn, node->flags));
 
-	if (node->flags == 0 && !ctdb->done_startup) {
+	if (node->flags == 0 && ctdb->runstate == CTDB_RUNSTATE_STARTUP) {
 		DEBUG(DEBUG_ERR, (__location__ " Node %u became healthy - force recovery for startup\n",
 				  c->pnn));
 		ctdb->recovery_mode = CTDB_RECOVERY_ACTIVE;
diff --git a/ctdb/server/ctdb_takeover.c b/ctdb/server/ctdb_takeover.c
index 3d7820584f0..d8e77dfb1f0 100644
--- a/ctdb/server/ctdb_takeover.c
+++ b/ctdb/server/ctdb_takeover.c
@@ -84,7 +84,7 @@ static int ctdb_add_local_iface(struct ctdb_context *ctdb, const char *iface)
 	 * IPs can't be assigned, and after startup IPs can be
 	 * assigned immediately.
 	 */
-	i->link_up = ctdb->done_startup;
+	i->link_up = (ctdb->runstate == CTDB_RUNSTATE_RUNNING);
 
 	DLIST_ADD(ctdb->ifaces, i);