added code to ctdb to send a tcp 'tickle' ack when we takeover an

IP. A raw tcp ack is sent for each tcp connection held by clients
before the IP takeover.

These acks have a deliberately incorrect sequence number, and should
cause the windows client to send its own ack which will in turn cause
a tcp reset and thus cause windows clients to much more quickly
reconnect to the new node.

(This used to be ctdb commit eef38bfe8461b47489d169c61895d6bb8a8f79a1)

5 files changed, 212 insertions, 14 deletions

diff --git a/ctdb/common/ctdb_control.c b/ctdb/common/ctdb_control.c
index cb990d52c2..fd3dcf9988 100644
--- a/ctdb/common/ctdb_control.c
+++ b/ctdb/common/ctdb_control.c
@@ -272,6 +272,18 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_delete_low_rsn));
 		return ctdb_control_delete_low_rsn(ctdb, indata, outdata);
 
+	case CTDB_CONTROL_TCP_CLIENT: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+		return ctdb_control_tcp_client(ctdb, client_id, indata);
+
+	case CTDB_CONTROL_TCP_ADD: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+		return ctdb_control_tcp_add(ctdb, indata);
+
+	case CTDB_CONTROL_TCP_REMOVE: 
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_control_tcp));
+		return ctdb_control_tcp_remove(ctdb, indata);
+
 	default:
 		DEBUG(0,(__location__ " Unknown CTDB control opcode %u\n", opcode));
 		return -1;
diff --git a/ctdb/common/ctdb_daemon.c b/ctdb/common/ctdb_daemon.c
index 40d5da6e6b..4bf1d1a605 100644
--- a/ctdb/common/ctdb_daemon.c
+++ b/ctdb/common/ctdb_daemon.c
@@ -29,17 +29,6 @@
 #include "../include/ctdb.h"
 #include "../include/ctdb_private.h"
 
-/*
-  structure describing a connected client in the daemon
- */
-struct ctdb_client {
-	struct ctdb_context *ctdb;
-	int fd;
-	struct ctdb_queue *queue;
-	uint32_t client_id;
-};
-
-
 static void daemon_incoming_packet(void *, struct ctdb_req_header *);
 
 static void ctdb_main_loop(struct ctdb_context *ctdb)
@@ -250,6 +239,7 @@ static void daemon_request_connect_wait(struct ctdb_client *client,
 */
 static int ctdb_client_destructor(struct ctdb_client *client)
 {
+	ctdb_takeover_client_destructor_hook(client);
 	ctdb_reqid_remove(client->ctdb, client->client_id);
 	client->ctdb->status.num_clients--;
 	return 0;
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 31751c24ae..61b459108c 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -81,6 +81,18 @@ typedef void (*ctdb_control_callback_fn_t)(struct ctdb_context *,
 					   void *private_data);
 
 /*
+  structure describing a connected client in the daemon
+ */
+struct ctdb_client {
+	struct ctdb_context *ctdb;
+	int fd;
+	struct ctdb_queue *queue;
+	uint32_t client_id;
+	struct ctdb_tcp_list *tcp_list;
+};
+
+
+/*
   state associated with one node
 */
 struct ctdb_node {
@@ -291,6 +303,7 @@ struct ctdb_context {
 	uint32_t recovery_master;
 	struct ctdb_call_state *pending_calls;
 	struct ctdb_takeover takeover;
+	struct ctdb_tcp_list *tcp_list;
 };
 
 struct ctdb_db_context {
@@ -398,6 +411,9 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS,
 		    CTDB_CONTROL_DELETE_LOW_RSN,
 		    CTDB_CONTROL_TAKEOVER_IP,
 		    CTDB_CONTROL_RELEASE_IP,
+		    CTDB_CONTROL_TCP_CLIENT,
+		    CTDB_CONTROL_TCP_ADD,
+		    CTDB_CONTROL_TCP_REMOVE,
 };
 
 /*
@@ -425,6 +441,14 @@ struct ctdb_control_set_call {
 	uint32_t id;
 };
 
+/*
+  struct for tcp_client, tcp_add and tcp_remove controls
+ */
+struct ctdb_control_tcp {
+	struct sockaddr_in src;
+	struct sockaddr_in dest;
+};
+
 enum call_state {CTDB_CALL_WAIT, CTDB_CALL_DONE, CTDB_CALL_ERROR};
 
 #define CTDB_LMASTER_ANY	0xffffffff
@@ -896,9 +920,18 @@ int ctdb_ctrl_release_ip(struct ctdb_context *ctdb, struct timeval timeout,
 int ctdb_sys_send_arp(const struct sockaddr_in *saddr, const char *iface);
 int ctdb_sys_take_ip(const char *ip, const char *interface);
 int ctdb_sys_release_ip(const char *ip, const char *interface);
+int ctdb_sys_send_ack(const struct sockaddr_in *dest, 
+		      const struct sockaddr_in *src);
 
 int ctdb_set_public_addresses(struct ctdb_context *ctdb, const char *alist);
 
 int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap);
 
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, 
+				TDB_DATA indata);
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata);
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata);
+
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client);
+
 #endif
diff --git a/ctdb/takeover/ctdb_takeover.c b/ctdb/takeover/ctdb_takeover.c
index 63b5ed7dab..220cd56ae7 100644
--- a/ctdb/takeover/ctdb_takeover.c
+++ b/ctdb/takeover/ctdb_takeover.c
@@ -29,15 +29,26 @@
 
 #define TAKEOVER_TIMEOUT() timeval_current_ofs(5,0)
 
-#define CTDB_ARP_INTERVAL 5
-#define CTDB_ARP_REPEAT  24
+#define CTDB_ARP_INTERVAL 1
+#define CTDB_ARP_REPEAT   3
 
 struct ctdb_takeover_arp {
 	struct ctdb_context *ctdb;
 	uint32_t count;
 	struct sockaddr_in sin;
+	struct ctdb_tcp_list *tcp_list;
 };
 
+/*
+  lists of tcp endpoints
+ */
+struct ctdb_tcp_list {
+	struct ctdb_tcp_list *prev, *next;
+	struct sockaddr_in saddr;
+	struct sockaddr_in daddr;	
+};
+
+
 
 /*
   send a gratuitous arp
@@ -48,12 +59,25 @@ static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *
 	struct ctdb_takeover_arp *arp = talloc_get_type(private_data, 
 							struct ctdb_takeover_arp);
 	int ret;
+	struct ctdb_tcp_list *tcp;
 
 	ret = ctdb_sys_send_arp(&arp->sin, arp->ctdb->takeover.interface);
 	if (ret != 0) {
 		DEBUG(0,(__location__ "sending of arp failed (%s)\n", strerror(errno)));
 	}
 
+	for (tcp=arp->tcp_list;tcp;tcp=tcp->next) {
+		DEBUG(0,("sending tcp tickle ack for %u->%s:%u\n",
+			 (unsigned)tcp->daddr.sin_port, 
+			 inet_ntoa(tcp->saddr.sin_addr),
+			 (unsigned)tcp->saddr.sin_port));			 
+		ret = ctdb_sys_send_ack(&tcp->daddr, &tcp->saddr);
+		if (ret != 0) {
+			DEBUG(0,(__location__ " Failed to send tcp tickle ack for %s\n",
+				 inet_ntoa(tcp->saddr.sin_addr)));
+		}
+	}
+
 	arp->count++;
 
 	if (arp->count == CTDB_ARP_REPEAT) {
@@ -66,6 +90,7 @@ static void ctdb_control_send_arp(struct event_context *ev, struct timed_event *
 			ctdb_control_send_arp, arp);
 }
 
+
 /*
   take over an ip address
  */
@@ -75,6 +100,7 @@ int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, TDB_DATA indata)
 	struct sockaddr_in *sin = (struct sockaddr_in *)indata.dptr;
 	struct ctdb_takeover_arp *arp;
 	char *ip = inet_ntoa(sin->sin_addr);
+	struct ctdb_tcp_list *tcp;
 
 	DEBUG(0,("Takover of IP %s on interface %s\n", ip, ctdb->takeover.interface));
 	ret = ctdb_sys_take_ip(ip, ctdb->takeover.interface);
@@ -95,6 +121,17 @@ int32_t ctdb_control_takeover_ip(struct ctdb_context *ctdb, TDB_DATA indata)
 	arp->ctdb = ctdb;
 	arp->sin = *sin;
 
+	/* add all of the known tcp connections for this IP to the
+	   list of tcp connections to send tickle acks for */
+	for (tcp=ctdb->tcp_list;tcp;tcp=tcp->next) {
+		if (sin->sin_addr.s_addr == tcp->daddr.sin_addr.s_addr) {
+			struct ctdb_tcp_list *t2 = talloc(arp, struct ctdb_tcp_list);
+			CTDB_NO_MEMORY(ctdb, t2);
+			*t2 = *tcp;
+			DLIST_ADD(arp->tcp_list, t2);
+		}
+	}
+
 	event_add_timed(arp->ctdb->ev, arp->ctdb->takeover.last_ctx, 
 			timeval_zero(), ctdb_control_send_arp, arp);
 
@@ -238,3 +275,129 @@ int ctdb_takeover_run(struct ctdb_context *ctdb, struct ctdb_node_map *nodemap)
 
 	return 0;
 }
+
+
+/*
+  called by a client to inform us of a TCP connection that it is managing
+  that should tickled with an ACK when IP takeover is done
+ */
+int32_t ctdb_control_tcp_client(struct ctdb_context *ctdb, uint32_t client_id, 
+				TDB_DATA indata)
+{
+	struct ctdb_client *client = ctdb_reqid_find(ctdb, client_id, struct ctdb_client);
+	struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+	struct ctdb_tcp_list *tcp;
+	int ret;
+
+	tcp = talloc(client, struct ctdb_tcp_list);
+	CTDB_NO_MEMORY(ctdb, tcp);
+
+	tcp->saddr = p->src;
+	tcp->daddr = p->dest;
+
+	DLIST_ADD(client->tcp_list, tcp);
+
+	/* tell all nodes about this tcp connection */
+	ret = ctdb_daemon_send_control(ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+				       CTDB_CONTROL_TCP_ADD,
+				       0, CTDB_CTRL_FLAG_NOREPLY, indata, NULL, NULL);
+	if (ret != 0) {
+		DEBUG(0,(__location__ " Failed to send CTDB_CONTROL_TCP_ADD\n"));
+		return -1;
+	}
+
+	return 0;
+}
+
+/*
+  see if two sockaddr_in are the same
+ */
+static bool same_sockaddr_in(struct sockaddr_in *in1, struct sockaddr_in *in2)
+{
+	return in1->sin_family == in2->sin_family &&
+		in1->sin_port == in2->sin_port &&
+		in1->sin_addr.s_addr == in2->sin_addr.s_addr;
+}
+
+/*
+  find a tcp address on a list
+ */
+static struct ctdb_tcp_list *ctdb_tcp_find(struct ctdb_tcp_list *list, 
+					   struct ctdb_tcp_list *tcp)
+{
+	while (list) {
+		if (same_sockaddr_in(&list->saddr, &tcp->saddr) &&
+		    same_sockaddr_in(&list->daddr, &tcp->daddr)) {
+			return list;
+		}
+		list = list->next;
+	}
+	return NULL;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_add(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+	struct ctdb_tcp_list *tcp;
+
+	tcp = talloc(ctdb, struct ctdb_tcp_list);
+	CTDB_NO_MEMORY(ctdb, tcp);
+
+	tcp->saddr = p->src;
+	tcp->daddr = p->dest;
+
+	if (NULL == ctdb_tcp_find(ctdb->tcp_list, tcp)) {
+		DLIST_ADD(ctdb->tcp_list, tcp);
+	}
+
+	return 0;
+}
+
+/*
+  called by a daemon to inform us of a TCP connection that one of its
+  clients managing that should tickled with an ACK when IP takeover is
+  done
+ */
+int32_t ctdb_control_tcp_remove(struct ctdb_context *ctdb, TDB_DATA indata)
+{
+	struct ctdb_control_tcp *p = (struct ctdb_control_tcp *)indata.dptr;
+	struct ctdb_tcp_list t, *tcp;
+
+	t.saddr = p->src;
+	t.daddr = p->dest;
+
+	tcp = ctdb_tcp_find(ctdb->tcp_list, &t);
+	if (tcp) {
+		DLIST_REMOVE(ctdb->tcp_list, tcp);
+	}
+
+	return 0;
+}
+
+
+/*
+  called when a client structure goes away - hook to remove
+  elements from the tcp_list in all daemons
+ */
+void ctdb_takeover_client_destructor_hook(struct ctdb_client *client)
+{
+	while (client->tcp_list) {
+		TDB_DATA data;
+		struct ctdb_control_tcp p;
+		struct ctdb_tcp_list *tcp = client->tcp_list;
+		DLIST_REMOVE(client->tcp_list, tcp);
+		p.src = tcp->saddr;
+		p.dest = tcp->daddr;
+		data.dptr = (uint8_t *)&p;
+		data.dsize = sizeof(p);
+		ctdb_daemon_send_control(client->ctdb, CTDB_BROADCAST_VNNMAP, 0, 
+					 CTDB_CONTROL_TCP_REMOVE,
+					 0, CTDB_CTRL_FLAG_NOREPLY, data, NULL, NULL);
+		talloc_free(tcp);
+	}
+}
diff --git a/ctdb/takeover/system.c b/ctdb/takeover/system.c
index bddd6a636e..b16b96343b 100644
--- a/ctdb/takeover/system.c
+++ b/ctdb/takeover/system.c
@@ -195,7 +195,7 @@ int ctdb_sys_send_ack(const struct sockaddr_in *dest,
 	pkt.tcp.source   = src->sin_port;
 	pkt.tcp.dest     = dest->sin_port;
 	pkt.tcp.ack      = 1;
-	pkt.tcp.check    = ip_checksum((uint16_t *)&pkt.tcp, sizeof(pkt.tcp)/2);
+	pkt.tcp.check    = 0;
 
 	ret = sendto(3, &pkt, sizeof(pkt), 0, dest, sizeof(*dest));
 	if (ret != 0) {