cpgx: support up to 16 nodes

Signed-off-by: David Teigland <teigland@redhat.com>

1 files changed, 51 insertions, 23 deletions

diff --git a/cpgx/cpgx.c b/cpgx/cpgx.c
index bb041bc..cf228eb 100644
--- a/cpgx/cpgx.c
+++ b/cpgx/cpgx.c
@@ -48,11 +48,20 @@ static char *exec_name = "aisexec";
 static char *exec_name = "corosync";
 #endif
 
+#define NODES16
+
+#ifdef NODES16
+#define MAX_NODES		16
+#define EVENT_BUF_LEN		60 /* + 4 = 64 */
+#else
+#define MAX_NODES		8
+#define EVENT_BUF_LEN		28 /* + 4 = 32 */
+#endif
+
 #define CLIENT_NALLOC		2
-#define MAX_NODES		8 /* not easily changed */
 #define DUMP_SIZE		(1024 * 1024)
-#define HISTORY_EVENTS		(1024 * 1024)
-#define DEFAULT_SYNC_MAX	2000 /* sync up to this many events */
+#define HISTORY_EVENTS		(1024 * 4)
+#define DEFAULT_SYNC_MAX	1000 /* sync up to this many events */
 #define DEFAULT_TIMEOUT_SEC	60
 #define DEFAULT_PORT		5405
 #define DEFAULT_RESTART_SEC	10
@@ -78,29 +87,29 @@ struct dct_header {
 	uint8_t nodeid;		/* sender */
 	uint8_t to_nodeid;	/* for MSGSYNC */
 	uint8_t pad;
-	uint8_t synced_nodes[8];/* for MSGSYNC, nodes in sync */
+	uint8_t synced_nodes[MAX_NODES];/* for MSGSYNC, nodes in sync */
 	uint32_t tv_sec;
 	uint32_t tv_usec;
 	uint32_t last_config;	/* last config eventid we've seen */
 	uint32_t event_count;	/* this many events after header */
-}; /* 28 bytes */
+}; /* 28 bytes, 36 bytes */
 
 struct dct_config {
 	uint8_t type;		/* EV_CONFCHG */
 	uint8_t memb_count;
 	uint8_t join_count;
 	uint8_t left_count;
-	uint8_t memb[8]; 
-	uint8_t join[8]; 
-	uint8_t left[8]; 
-}; /* 28 bytes */
+	uint8_t memb[MAX_NODES]; 
+	uint8_t join[MAX_NODES]; 
+	uint8_t left[MAX_NODES]; 
+}; /* 28 bytes, 52 bytes */
 
 struct event {
 	uint32_t eventid;
 	union {
 		struct dct_header header;
 		struct dct_config config;
-		char buf[28];
+		char buf[EVENT_BUF_LEN];
 	};
 }; /* 32 bytes */
 
@@ -116,7 +125,7 @@ struct node {
 	uint32_t last_check_eventid; /* the eventid from the check event in the
 					most recent time message we received
 					from this node. not used for anything */
-	uint8_t synced_nodes[8];
+	uint8_t synced_nodes[MAX_NODES];
 };
 
 struct save_event {
@@ -221,9 +230,9 @@ do { \
 
 static void _log_config(struct dct_config *c, uint32_t id, int error)
 {
-	char m_buf[32];
-	char j_buf[32];
-	char l_buf[32];
+	char m_buf[64];
+	char j_buf[64];
+	char l_buf[64];
 	int i, off;
 
 	memset(m_buf, 0, sizeof(m_buf));
@@ -502,7 +511,7 @@ void init_nodes_list(struct dct_header *hd, struct dct_config *c)
 		node->needs_sync = 1;
 	}
 
-	for (i = 0; i < 8; i++) {
+	for (i = 0; i < MAX_NODES; i++) {
 		nodeid = hd->synced_nodes[i];
 		if (!nodeid)
 			break;
@@ -538,7 +547,7 @@ void free_nodes_list(void)
 void update_nodes_list(struct dct_config *c, uint32_t id)
 {
 	struct node *node;
-	uint8_t synced_nodes[8];
+	uint8_t synced_nodes[MAX_NODES];
 	uint8_t low = 0;
 	int i, is_memb, is_join, is_left;
 
@@ -600,7 +609,7 @@ void update_nodes_list(struct dct_config *c, uint32_t id)
 	/* figure out who should send syncs and what synced_nodes are */
 
 	i = 0;
-	memset(&synced_nodes, 0, 8);
+	memset(&synced_nodes, 0, MAX_NODES);
 
 	list_for_each_entry(node, &nodes, list) {
 		if (!node->is_member)
@@ -634,7 +643,7 @@ void update_nodes_list(struct dct_config *c, uint32_t id)
 		if (node->is_member && node->needs_sync &&
 		    node->join_eventid == id) {
 			node->sync_from = low;
-			memcpy(&node->synced_nodes, &synced_nodes, 8);
+			memcpy(&node->synced_nodes, &synced_nodes, MAX_NODES);
 		}
 	}
 
@@ -650,7 +659,7 @@ void update_nodes_list(struct dct_config *c, uint32_t id)
 		node->needs_sync = 1;
 		node->join_eventid = id;
 		node->sync_from = low;
-		memcpy(&node->synced_nodes, &synced_nodes, 8);
+		memcpy(&node->synced_nodes, &synced_nodes, MAX_NODES);
 
 		is_join = in_join(c, node->nodeid);
 		if (!is_join)
@@ -845,7 +854,7 @@ void send_sync(uint8_t nodeid)
 	/* which nodes are synced is part of the replicated state that needs to
 	   be synced to new nodes */
 
-	memcpy(hd->synced_nodes, node->synced_nodes, 8);
+	memcpy(hd->synced_nodes, node->synced_nodes, MAX_NODES);
 
 	for (i = start_eventid; i < end_eventid + 1; i++) {
 		memcpy(ev, history(i), sizeof(struct event));
@@ -862,7 +871,23 @@ void send_sync(uint8_t nodeid)
 
 void check_event(struct event *ev_buf)
 {
-	if (memcmp(ev_buf, history(ev_buf->eventid), sizeof(struct event))) {
+	struct dct_config *c = &ev_buf->config;
+	int len = 0;
+
+	switch (c->type) {
+	case EV_CONFCHG:
+		len = sizeof(struct dct_config);
+		break;
+	case EV_MSGTIME:
+	case EV_MSGSYNC:
+		len = sizeof(struct dct_header);
+		break;
+	default:
+		log_error("check_event unknown type %d", c->type);
+		return;
+	}
+
+	if (memcmp(ev_buf, history(ev_buf->eventid), len)) {
 		log_error("check_event %u", ev_buf->eventid);
 		log_event(ev_buf, 1);
 		log_event(history(ev_buf->eventid), 1);
@@ -1082,7 +1107,9 @@ static void confchg_cb(cpg_handle_t handle, const struct cpg_name *group_name,
 		       size_t join_list_entries)
 #endif
 {
-	uint8_t memb_sort[8], left_sort[8], join_sort[8];
+	uint8_t memb_sort[MAX_NODES];
+        uint8_t left_sort[MAX_NODES];
+	uint8_t join_sort[MAX_NODES];
 	struct dct_config c;
 	struct node *node;
 	int we_join = 0;
@@ -1741,8 +1768,9 @@ void print_usage(void)
 	printf("Notes:\n");
 	printf("  - when cpgx is started the node must be a cluster member\n");
 	printf("  - to prevent history from periodically restarting from 0,\n"
+	       "    or sometimes all nodes being new without any to sync,\n"
 	       "    keep one node from leaving/exiting/dieing with -l0 -e0 -d0\n");
-	printf("  - 8 nodes max, nodeids beteen 1 and 255\n");
+	printf("  - %d nodes max, nodeids beteen 1 and 255\n", MAX_NODES);
 	printf("  - debug dump on error: %s\n", DUMP_WRITE_PATH);
 }