diff options
Diffstat (limited to 'daemons/clvmd')
-rw-r--r-- | daemons/clvmd/Makefile.in | 22 | ||||
-rw-r--r-- | daemons/clvmd/clvmd-cman.c | 1 | ||||
-rw-r--r-- | daemons/clvmd/clvmd-gulm.c | 80 | ||||
-rw-r--r-- | daemons/clvmd/tcp-comms.c | 160 | ||||
-rw-r--r-- | daemons/clvmd/tcp-comms.h | 3 |
5 files changed, 137 insertions, 129 deletions
diff --git a/daemons/clvmd/Makefile.in b/daemons/clvmd/Makefile.in index da1d0167..a5ab680b 100644 --- a/daemons/clvmd/Makefile.in +++ b/daemons/clvmd/Makefile.in @@ -15,6 +15,18 @@ srcdir = @srcdir@ top_srcdir = @top_srcdir@ VPATH = @srcdir@ +ifeq ("@CLVMD@", "gulm") +SOURCES = \ + clvmd-gulm.c \ + tcp-comms.c \ + clvmd-command.c \ + clvmd.c \ + libclvm.c \ + lvm-functions.c \ + system-lv.c +LM_LIBS = -lccs -lgulm +LM_FLAGS= -DUSE_GULM +else SOURCES = \ clvmd-cman.c \ clvmd-command.c \ @@ -22,20 +34,24 @@ SOURCES = \ libclvm.c \ lvm-functions.c \ system-lv.c +LM_LIBS = -ldlm +LM_FLAGS= +endif TARGETS = \ clvmd include $(top_srcdir)/make.tmpl -CFLAGS += -D_REENTRANT -fno-strict-aliasing -LIBS += -ldevmapper -ldlm -llvm -lpthread +CFLAGS += -D_REENTRANT -fno-strict-aliasing $(LM_FLAGS) +LIBS += -ldevmapper -llvm -lpthread + INSTALL_TARGETS = \ install_clvmd clvmd: $(OBJECTS) $(top_srcdir)/lib/liblvm.a - $(CC) -o clvmd $(OBJECTS) $(LDFLAGS) $(LVMLIBS) $(LIBS) + $(CC) -o clvmd $(OBJECTS) $(LDFLAGS) $(LVMLIBS) $(LM_LIBS) $(LIBS) .PHONY: install_clvmd diff --git a/daemons/clvmd/clvmd-cman.c b/daemons/clvmd/clvmd-cman.c index fcc20a00..76d6f4bb 100644 --- a/daemons/clvmd/clvmd-cman.c +++ b/daemons/clvmd/clvmd-cman.c @@ -55,7 +55,6 @@ static int max_updown_nodes = 50; /* Current size of the allocated array */ static int *node_updown = NULL; static dlm_lshandle_t *lockspace; -static void sigusr1_handler(int sig); static void count_clvmds_running(void); static void get_members(void); static int nodeid_from_csid(char *csid); diff --git a/daemons/clvmd/clvmd-gulm.c b/daemons/clvmd/clvmd-gulm.c index bef4cbe7..f3be3855 100644 --- a/daemons/clvmd/clvmd-gulm.c +++ b/daemons/clvmd/clvmd-gulm.c @@ -139,11 +139,28 @@ int init_cluster() { int status; int ccs_h; + int port = 0; + char *portstr; /* Get cluster name from CCS */ /* TODO: is this right? */ - ccs_h = ccs_connect(); + ccs_h = ccs_force_connect(NULL, 1); // PJC + if (!ccs_h) + return -1; + ccs_get(ccs_h, "//cluster/@name", &cluster_name); + DEBUGLOG("got cluster name %s\n", cluster_name); + + if (!ccs_get(ccs_h, "//clvm/@port", &portstr)) + { + port = atoi(portstr); + free(portstr); + DEBUGLOG("got port number %d\n", port); + + if (port <= 0 && port >= 65536) + port = 0; + } + ccs_disconnect(ccs_h); /* Block locking until we are logged in */ @@ -155,7 +172,8 @@ int init_cluster() lock_hash = hash_create(10); /* Get all nodes from CCS */ - get_all_cluster_nodes(); + if (get_all_cluster_nodes()) + return -1; /* Initialise GULM library */ status = lg_initialize(&gulm_if, cluster_name, "clvmd"); @@ -174,7 +192,7 @@ int init_cluster() } /* Initialise the inter-node comms */ - status = init_comms(); + status = init_comms(port); if (status) return status; @@ -316,11 +334,11 @@ static void set_node_state(struct node_info *ninfo, char *csid, uint8_t nodestat ninfo->name, ninfo->state, num_nodes); } -static struct node_info *add_or_set_node(char *name, uint32_t ip, uint8_t state) +static struct node_info *add_or_set_node(char *name, struct in6_addr *ip, uint8_t state) { struct node_info *ninfo; - ninfo = hash_lookup_binary(node_hash, (char *)&ip, MAX_CSID_LEN); + ninfo = hash_lookup_binary(node_hash, (char *)ip, MAX_CSID_LEN); if (!ninfo) { /* If we can't find that node then re-read the config file in case it @@ -329,7 +347,7 @@ static struct node_info *add_or_set_node(char *name, uint32_t ip, uint8_t state) get_all_cluster_nodes(); /* Now try again */ - ninfo = hash_lookup_binary(node_hash, (char *)&ip, MAX_CSID_LEN); + ninfo = hash_lookup_binary(node_hash, (char *)ip, MAX_CSID_LEN); if (!ninfo) { DEBUGLOG("Ignoring node %s, not part of the SAN cluster\n", name); @@ -342,7 +360,7 @@ static struct node_info *add_or_set_node(char *name, uint32_t ip, uint8_t state) return ninfo; } -static int core_nodelist(void *misc, lglcb_t type, char *name, uint32_t ip, uint8_t state) +static int core_nodelist(void *misc, lglcb_t type, char *name, struct in6_addr *ip, uint8_t state) { DEBUGLOG("CORE nodelist\n"); @@ -354,7 +372,7 @@ static int core_nodelist(void *misc, lglcb_t type, char *name, uint32_t ip, uint { if (type == lglcb_item) { - DEBUGLOG("Got nodelist, item: %s, %#x, %#x\n", name, ip, state); + DEBUGLOG("Got nodelist, item: %s, %#x\n", name, state); add_or_set_node(name, ip, state); } @@ -381,24 +399,24 @@ static int core_nodelist(void *misc, lglcb_t type, char *name, uint32_t ip, uint return 0; } -static int core_statechange(void *misc, uint8_t corestate, uint32_t masterip, char *mastername) +static int core_statechange(void *misc, uint8_t corestate, uint8_t quorate, struct in6_addr *masterip, char *mastername) { - DEBUGLOG("CORE Got statechange corestate:%#x masterip:%#x mastername:%s\n", - corestate, masterip, mastername); + DEBUGLOG("CORE Got statechange corestate:%#x mastername:%s\n", + corestate, mastername); current_corestate = corestate; return 0; } -static int core_nodechange(void *misc, char *nodename, uint32_t nodeip, uint8_t nodestate) +static int core_nodechange(void *misc, char *nodename, struct in6_addr *nodeip, uint8_t nodestate) { struct node_info *ninfo; - DEBUGLOG("CORE node change, name=%s, ip=%x, state = %d\n", nodename, nodeip, nodestate); + DEBUGLOG("CORE node change, name=%s, state = %d\n", nodename, nodestate); /* If we don't get nodeip here, try a lookup by name */ if (!nodeip) - csid_from_name((char *)&nodeip, nodename); + csid_from_name((char *)nodeip, nodename); if (!nodeip) return 0; @@ -443,7 +461,9 @@ static int lock_login_reply(void *misc, uint32_t error, uint8_t which) return 0; } -static int lock_lock_state(void *misc, uint8_t *key, uint16_t keylen, uint8_t state, uint32_t flags, uint32_t error, +static int lock_lock_state(void *misc, uint8_t *key, uint16_t keylen, + uint64_t subid, uint64_t start, uint64_t stop, + uint8_t state, uint32_t flags, uint32_t error, uint8_t *LVB, uint16_t LVBlen) { struct lock_wait *lwait; @@ -530,8 +550,7 @@ int name_from_csid(char *csid, char *name) ninfo = hash_lookup_binary(node_hash, csid, MAX_CSID_LEN); if (!ninfo) { - sprintf(name, "UNKNOWN [%d.%d.%d.%d]", - csid[0], csid[1], csid[2], csid[3]); + sprintf(name, "UNKNOWN %s", print_csid(csid)); return -1; } @@ -661,6 +680,7 @@ static int _lock_resource(char *resource, int mode, int flags, int *lockid) DEBUGLOG("lock_resource '%s', flags=%d, mode=%d\n", resource, flags, mode); status = lg_lock_state_req(gulm_if, resource, strlen(resource)+1, + 0, 0, 0, mode, flags, NULL, 0); if (status) { @@ -692,6 +712,7 @@ static int _unlock_resource(char *resource, int lockid) DEBUGLOG("unlock_resource %s\n", resource); status = lg_lock_state_req(gulm_if, resource, strlen(resource)+1, + 0, 0, 0, lg_lock_state_Unlock, 0, NULL, 0); if (status) @@ -819,26 +840,38 @@ static int get_all_cluster_nodes() int ctree; char *nodename; int error; + int i; /* Open the config file */ - ctree = ccs_connect(); + ctree = ccs_force_connect(NULL, 1); if (ctree <= 0) { log_error("Error connecting to CCS"); return -1; } - error = ccs_get(ctree, "//nodes/node/@name", &nodename); - while (nodename) + for (i=1; i++;) { + char nodekey[256]; char nodeip[MAX_CSID_LEN]; - char *clvmflag; + int clvmflag = 1; + char *clvmflagstr; char key[256]; + sprintf(nodekey, "//cluster/nodes/node[%d]/@name", i); + error = ccs_get(ctree, nodekey, &nodename); + if (error) + break; + sprintf(key, "//nodes/node[@name=\"%s\"]/clvm", nodename); - ccs_get(ctree, key, &clvmflag); + if (!ccs_get(ctree, key, &clvmflagstr)) + { + clvmflag = atoi(clvmflagstr); + free(clvmflagstr); + } - if ((get_ip_address(nodename, nodeip) == 0) && atoi(clvmflag)) + DEBUGLOG("Got node %s from ccs(clvmflag = %d)\n", nodename, clvmflag); + if ((get_ip_address(nodename, nodeip) == 0) && clvmflag) { struct node_info *ninfo; @@ -863,7 +896,6 @@ static int get_all_cluster_nodes() { DEBUGLOG("node %s has clvm disabled\n", nodename); } - if (clvmflag) free(clvmflag); free(nodename); error = ccs_get(ctree, "//nodes/node/@name", &nodename); } diff --git a/daemons/clvmd/tcp-comms.c b/daemons/clvmd/tcp-comms.c index 2e0406b3..e0a53c0c 100644 --- a/daemons/clvmd/tcp-comms.c +++ b/daemons/clvmd/tcp-comms.c @@ -34,7 +34,6 @@ #include <netdb.h> #include <assert.h> -#include "ccs.h" #include "clvm.h" #include "clvmd-comms.h" #include "clvmd.h" @@ -47,43 +46,19 @@ static int listen_fd = -1; static int tcp_port; struct hash_table *sock_hash; -static int get_tcp_port(int default_port); static int get_our_ip_address(char *addr, int *family); static int read_from_tcpsock(struct local_client *fd, char *buf, int len, char *csid, struct local_client **new_client); /* Called by init_cluster() to open up the listening socket */ -// TODO: IPv6 compat. -int init_comms() +int init_comms(unsigned short port) { - struct sockaddr *addr = NULL; - struct sockaddr_in addr4; - struct sockaddr_in6 addr6; - int addr_len; - int family; - char address[MAX_CSID_LEN]; + struct sockaddr_in6 addr; sock_hash = hash_create(100); - tcp_port = get_tcp_port(DEFAULT_TCP_PORT); + tcp_port = port ? port : DEFAULT_TCP_PORT; - /* Get IP address and IP type */ - get_our_ip_address(address, &family); - if (family == AF_INET) - { - memcpy(&addr4.sin_addr, addr, sizeof(struct in_addr)); - addr = (struct sockaddr *)&addr4; - addr4.sin_port = htons(tcp_port); - addr_len = sizeof(addr4); - } - else - { - memcpy(&addr6.sin6_addr, addr, sizeof(struct in6_addr)); - addr = (struct sockaddr *)&addr6; - addr6.sin6_port = htons(tcp_port); - addr_len = sizeof(addr6); - } - - listen_fd = socket(family, SOCK_STREAM, 0); + listen_fd = socket(AF_INET6, SOCK_STREAM, 0); if (listen_fd < 0) { @@ -95,11 +70,13 @@ int init_comms() setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(int)); } - addr->sa_family = family; + memset(&addr, 0, sizeof(addr)); // Bind to INADDR_ANY + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(tcp_port); - if (bind(listen_fd, addr, addr_len) < 0) + if (bind(listen_fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { - DEBUGLOG("Can't bind to port\n"); + DEBUGLOG("Can't bind to port: %s\n", strerror(errno)); syslog(LOG_ERR, "Can't bind to port %d, is clvmd already running ?", tcp_port); close(listen_fd); return -1; @@ -142,7 +119,7 @@ int alloc_client(int fd, char *csid, struct local_client **new_client) { struct local_client *client; - DEBUGLOG("alloc_client %d csid = [%d.%d.%d.%d]\n", fd,csid[0],csid[1],csid[2],csid[3]); + DEBUGLOG("alloc_client %d csid = %s\n", fd, print_csid(csid)); /* Create a local_client and return it */ client = malloc(sizeof(struct local_client)); @@ -199,7 +176,7 @@ int cluster_fd_callback(struct local_client *fd, char *buf, int len, char *csid, struct local_client **new_client) { int newfd; - struct sockaddr_in addr; + struct sockaddr_in6 addr; socklen_t addrlen = sizeof(addr); int status; char name[MAX_CLUSTER_MEMBER_NAME_LEN]; @@ -218,22 +195,20 @@ int cluster_fd_callback(struct local_client *fd, char *buf, int len, char *csid, /* Check that the client is a member of the cluster and reject if not. - // FIXME: IPv4 specific */ - if (name_from_csid((char *)&addr.sin_addr.s_addr, name) < 0) + if (name_from_csid((char *)&addr.sin6_addr, name) < 0) { - char *ip = (char *)&addr.sin_addr.s_addr; - syslog(LOG_ERR, "Got connect from non-cluster node %d.%d.%d.%d\n", - ip[0], ip[1], ip[2], ip[3]); - DEBUGLOG("Got connect from non-cluster node %d.%d.%d.%d\n", - ip[0], ip[1], ip[2], ip[3]); + syslog(LOG_ERR, "Got connect from non-cluster node %s\n", + print_csid((char *)&addr.sin6_addr)); + DEBUGLOG("Got connect from non-cluster node %s\n", + print_csid((char *)&addr.sin6_addr)); close(newfd); errno = EAGAIN; return -1; } - status = alloc_client(newfd, (char *)&addr.sin_addr.s_addr, new_client); + status = alloc_client(newfd, (char *)&addr.sin6_addr, new_client); if (status) { DEBUGLOG("cluster_fd_callback, alloc_client failed, status = %d\n", status); @@ -250,7 +225,7 @@ int cluster_fd_callback(struct local_client *fd, char *buf, int len, char *csid, static int read_from_tcpsock(struct local_client *client, char *buf, int len, char *csid, struct local_client **new_client) { - struct sockaddr_in addr; + struct sockaddr_in6 addr; socklen_t slen = sizeof(addr); int status; @@ -259,7 +234,7 @@ static int read_from_tcpsock(struct local_client *client, char *buf, int len, ch /* Get "csid" */ getpeername(client->fd, (struct sockaddr *)&addr, &slen); - memcpy(csid, &addr.sin_addr.s_addr, MAX_CSID_LEN); + memcpy(csid, &addr.sin6_addr, MAX_CSID_LEN); status = read(client->fd, buf, len); @@ -289,11 +264,11 @@ static int read_from_tcpsock(struct local_client *client, char *buf, int len, ch static int connect_csid(char *csid, struct local_client **newclient) { int fd; - struct sockaddr_in addr; + struct sockaddr_in6 addr; int status; DEBUGLOG("Connecting socket\n"); - fd = socket(PF_INET, SOCK_STREAM, 0); + fd = socket(PF_INET6, SOCK_STREAM, 0); if (fd < 0) { @@ -301,12 +276,12 @@ static int connect_csid(char *csid, struct local_client **newclient) return -1; } - addr.sin_family = AF_INET; - memcpy(&addr.sin_addr.s_addr, csid, MAX_CSID_LEN); - addr.sin_port = htons(tcp_port); + addr.sin6_family = AF_INET6; + memcpy(&addr.sin6_addr, csid, MAX_CSID_LEN); + addr.sin6_port = htons(tcp_port); DEBUGLOG("Connecting socket %d\n", fd); - if (connect(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) < 0) + if (connect(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) < 0) { syslog(LOG_ERR, "Unable to connect to remote node: %m"); DEBUGLOG("Unable to connect to remote node: %s\n", strerror(errno)); @@ -334,7 +309,7 @@ static int tcp_send_message(void *buf, int msglen, unsigned char *csid, const ch assert(csid); - DEBUGLOG("tcp_send_message, csid = [%d.%d.%d.%d], msglen = %d\n", csid[0],csid[1],csid[2],csid[3], msglen); + DEBUGLOG("tcp_send_message, csid = %s, msglen = %d\n", print_csid(csid), msglen); /* Don't connect to ourself */ get_our_csid(ourcsid); @@ -383,55 +358,17 @@ int cluster_send_message(void *buf, int msglen, char *csid, const char *errtext) return status; } -static int get_tcp_port(int default_port) -{ - int ccs_handle; - int port = default_port; - char *portstr; - - ccs_handle = ccs_connect(); - if (ccs_handle) - { - return port; - } - - if (!ccs_get(ccs_handle, "//clvm/@port", &portstr)) - { - port = atoi(portstr); - free(portstr); - - if (port <= 0 && port >= 65536) - port = default_port; - } - ccs_disconnect(ccs_handle); - - DEBUGLOG("Using port %d for communications\n", port); - return port; -} - /* To get our own IP address we get the locally bound address of the socket that's talking to GULM in the assumption(eek) that it will be on the "right" network in a multi-homed system */ static int get_our_ip_address(char *addr, int *family) { - /* Use a sockaddr_in6 to make sure it's big enough */ - struct sockaddr_in6 saddr; - int socklen = sizeof(saddr); + struct utsname info; + + uname(&info); + get_ip_address(info.nodename, addr); - if (!getsockname(gulm_fd(), (struct sockaddr *)&saddr, &socklen)) - { - if (saddr.sin6_family == AF_INET6) - { - memcpy(addr, &saddr.sin6_addr, sizeof(saddr.sin6_addr)); - } - else - { - struct sockaddr_in *sin4 = (struct sockaddr_in *)&saddr; - memcpy(addr, &sin4->sin_addr, sizeof(sin4->sin_addr)); - } return 0; - } - return -1; } /* Public version of above for those that don't care what protocol @@ -454,6 +391,14 @@ void get_our_csid(char *csid) memcpy(csid, our_csid, MAX_CSID_LEN); } +static void map_v4_to_v6(struct in_addr *ip4, struct in6_addr *ip6) +{ + ip6->s6_addr32[0] = 0; + ip6->s6_addr32[1] = 0; + ip6->s6_addr32[2] = htonl(0xffff); + ip6->s6_addr32[3] = ip4->s_addr; +} + /* Get someone else's IP address from DNS */ int get_ip_address(char *node, char *addr) { @@ -467,14 +412,29 @@ int get_ip_address(char *node, char *addr) /* Try IPv6 first. The man page for gethostbyname implies that it will lookup ip6 & ip4 names, but it seems not to */ he = gethostbyname2(node, AF_INET6); - if (!he) + if (he) + { + memcpy(addr, he->h_addr_list[0], + he->h_length); + } + else + { he = gethostbyname2(node, AF_INET); - if (!he) - return -1; - - /* For IPv4 address just use the lower 4 bytes */ - memcpy(&addr, he->h_addr_list[0], - he->h_length); + if (!he) + return -1; + map_v4_to_v6((struct in_addr *)he->h_addr_list[0], (struct in6_addr *)addr); + } return 0; } + +char *print_csid(char *csid) +{ + static char buf[128]; + int *icsid = (int *)csid; + + sprintf(buf, "[%x.%x.%x.%x]", + icsid[0],icsid[1],icsid[2],icsid[3]); + + return buf; +} diff --git a/daemons/clvmd/tcp-comms.h b/daemons/clvmd/tcp-comms.h index 8dafd441..49ac479a 100644 --- a/daemons/clvmd/tcp-comms.h +++ b/daemons/clvmd/tcp-comms.h @@ -4,4 +4,5 @@ #define MAX_CSID_LEN sizeof(struct in6_addr) #define MAX_CLUSTER_MEMBER_NAME_LEN 128 -extern int init_comms(void); +extern int init_comms(unsigned short); +extern char *print_csid(char *); |