dlm_kill: add dlm_lock_alt test

dlm_lock_alt is like dlm_lock_loop, but it enforces a
back-and-forth pattern (using a separate dlm lock),
ensuring that each node gets the EX in turn after a kill.

Signed-off-by: David Teigland <teigland@redhat.com>

3 files changed, 322 insertions, 11 deletions

diff --git a/dlm/dlm_kill/Makefile b/dlm/dlm_kill/Makefile
index 28c4e0f..b56452e 100644
--- a/dlm/dlm_kill/Makefile
+++ b/dlm/dlm_kill/Makefile
@@ -1,7 +1,7 @@
 
 CFLAGS := -Wall -g -I ~/rhel4.git/dlm-kernel/src/
 
-all: dlm_lockspace dlm_lock dlm_lock_loop
+all: dlm_lockspace dlm_lock dlm_lock_loop dlm_lock_alt
 
 dlm_lockspace:		dlm_lockspace.o
 	gcc -o $@ $^ -ldlm_lt
@@ -12,7 +12,7 @@ dlm_lockspace.o:	dlm_lockspace.c
 dlm_lock:		dlm_lock.o
 	gcc -o $@ $^ -ldlm_lt
 
-dlm_lock.o:	dlm_lock.c
+dlm_lock.o:		dlm_lock.c
 	gcc $(CFLAGS) -c -o $@ $<
 
 dlm_lock_loop:		dlm_lock_loop.o
@@ -21,6 +21,12 @@ dlm_lock_loop:		dlm_lock_loop.o
 dlm_lock_loop.o:	dlm_lock_loop.c
 	gcc $(CFLAGS) -c -o $@ $<
 
+dlm_lock_alt:		dlm_lock_alt.o
+	gcc -o $@ $^ -ldlm -lpthread
+
+dlm_lock_alt.o:		dlm_lock_alt.c
+	gcc $(CFLAGS) -D_REENTRANT -c -o $@ $<
+
 clean:
-	rm -f *.o dlm_lockspace dlm_lock dlm_lock_loop
+	rm -f *.o dlm_lockspace dlm_lock dlm_lock_loop dlm_lock_alt
 
diff --git a/dlm/dlm_kill/dlm_lock_alt.c b/dlm/dlm_kill/dlm_lock_alt.c
new file mode 100644
index 0000000..a757c6c
--- /dev/null
+++ b/dlm/dlm_kill/dlm_lock_alt.c
@@ -0,0 +1,304 @@
+/*
+ * Copyright (c) 2010 David Teigland
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License V2
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it would be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <string.h>
+#include <limits.h>
+#include <time.h>
+#include <errno.h>
+#include <signal.h>
+#include <syslog.h>
+#include <sys/poll.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <sys/types.h>
+
+#include "libdlm.h"
+
+/* 
+ * The general idea of this test is
+ * - dlm_lock on node1 gets EX
+ * - dlm_lock on node2 blocks on EX
+ * - dlm_lock on node1 is killed
+ * - dlm_lock on node2 should get EX
+ * repeat
+ *
+ * The dlm_lock process is intentionally killed without an explicit
+ * unlock to test the lock cleanup done by the kernel when a process
+ * exits while holding locks.
+ */
+
+/* even/odd limits this to two instances, but requires no config;
+   "offset max" args (like alternate) would allow multiple */
+
+static dlm_lshandle_t *dh;
+static struct dlm_lksb counter_lksb;
+static char counter_lvb[32];
+static uint32_t counter;
+static int we_are_even;
+
+static int do_release(void)
+{
+	dh = dlm_create_lockspace("dlm_lock_alt", 0600);
+	if (!dh) {
+		printf("alt create lockspace error\n");
+		return -1;
+	}
+
+	dlm_release_lockspace("dlm_lock_alt", dh, 1);
+	return 0;
+}
+
+static int get_counter(void)
+{
+	int rv;
+
+	dh = dlm_create_lockspace("dlm_lock_alt", 0600);
+	if (!dh) {
+		return -1;
+	}
+
+	rv = dlm_ls_pthread_init(dh);
+	if (rv < 0) {
+		return -1;
+	}
+
+	memset(&counter_lksb, 0, sizeof(counter_lksb));
+	memset(&counter_lvb, 0, sizeof(counter_lvb));
+	counter_lksb.sb_lvbptr = counter_lvb;
+
+	printf("alt counter request NL ...\n");
+	rv = dlm_ls_lock_wait(dh, LKM_NLMODE, &counter_lksb,
+				LKF_VALBLK,
+				"counter", strlen("counter"),
+				0, NULL, NULL, NULL);
+
+	printf("alt counter request NL done status %d lkid %x\n",
+	       counter_lksb.sb_status, counter_lksb.sb_lkid);
+
+	if (rv) {
+		return -1;
+	}
+	return 0;
+}
+
+/* increment lvb of counter_lock */
+/* if we are even, counter++ should make counter odd */
+
+static int inc_counter(void)
+{
+	int rv;
+
+	/* NL->EX */
+
+	rv = dlm_ls_lock_wait(dh, LKM_EXMODE, &counter_lksb,
+				LKF_VALBLK | LKF_CONVERT,
+				"counter", strlen("counter"),
+				0, NULL, NULL, NULL);
+	if (rv) {
+		return -1;
+	}
+
+	counter++;
+	memcpy(&counter_lvb, &counter, sizeof(counter));
+
+	/* EX->NL */
+
+	rv = dlm_ls_lock_wait(dh, LKM_NLMODE, &counter_lksb,
+				LKF_VALBLK | LKF_CONVERT,
+				"counter", strlen("counter"),
+				0, NULL, NULL, NULL);
+	if (rv) {
+		return -1;
+	}
+
+	return 0;
+}
+
+/* if we are even, wait for counter lock to be even */
+
+static int wait_counter(void)
+{
+	int rv;
+
+	while (1) {
+		/* NL->PR */
+
+		rv = dlm_ls_lock_wait(dh, LKM_PRMODE, &counter_lksb,
+					LKF_VALBLK | LKF_CONVERT,
+					"counter", strlen("counter"),
+					0, NULL, NULL, NULL);
+		if (rv) {
+			return -1;
+		}
+
+		memcpy(&counter, &counter_lvb, sizeof(counter));
+
+		/* PR->NL */
+
+		rv = dlm_ls_lock_wait(dh, LKM_NLMODE, &counter_lksb,
+					LKF_VALBLK | LKF_CONVERT,
+					"counter", strlen("counter"),
+					0, NULL, NULL, NULL);
+		if (rv) {
+			return -1;
+		}
+
+		if (!counter) {
+			we_are_even = 1;
+			break;
+		}
+
+		if (we_are_even && !(counter % 2))
+			break;
+
+		if (!we_are_even && (counter % 2))
+			break;
+
+		usleep(500000);
+	}
+
+	return 0;
+}
+
+static int rand_int(int a, int b)
+{
+	return a + (int) (((float)(b - a + 1)) * random() / (RAND_MAX+1.0)); 
+}
+
+#define LINE_LEN 64
+
+static int wait_child_lock(int pr_fd)
+{
+	char *done_str = "dlm_lock convert EX done";
+	char line[LINE_LEN], prog[32], act[32], mode[32], done[32], status[32];
+	FILE *file;
+	int n, val, rv = -1;
+
+	file = fdopen(pr_fd, "r");
+	if (!file) {
+		printf("alt fdopen error %d\n", errno);
+		return -1;
+	}
+
+	while (fgets(line, LINE_LEN, file)) {
+		printf("%s", line);
+		fflush(stdout);
+
+		n = sscanf(line, "%s %s %s %s %s %d",
+			   prog, act, mode, done, status, &val);
+
+		if (n < 6)
+			continue;
+
+		if (!strncmp(line, done_str, strlen(done_str))) {
+			if (!strcmp(status, "status") && val == 0)
+				rv = 0;
+			break;
+		}
+	}
+	fclose(file);
+	return rv;
+}
+
+int main(int argc, char *argv[])
+{
+	int fda[2];
+	int pr_fd, cw_fd; /* parent read fd, child write fd */
+	int pid, status, n, rv;
+
+	printf("alt pid %d\n", getpid());
+
+	if (argc > 1 && !strcmp(argv[1], "release")) {
+		do_release();
+		return 0;
+	}
+
+	rv = get_counter();
+	if (rv < 0) {
+		return -1;
+	}
+
+	while (1) {
+		/* when this returns, it's our turn (other node has EX) */
+		printf("alt wait counter %s %u\n", we_are_even ? "even" : "odd", counter);
+		rv = wait_counter();
+		if (rv < 0) {
+			printf("alt wait_counter error\n");
+			break;
+		}
+
+		pipe(fda);
+		pr_fd = fda[0];
+		cw_fd = fda[1];
+
+		pid = fork();
+		if (!pid) {
+			/* child's stdout and stderr sent to parent's pr_fd */
+			close(STDOUT_FILENO);
+			dup(cw_fd);
+			close(STDERR_FILENO);
+			dup(cw_fd);
+			close(STDIN_FILENO);
+
+			execl("./dlm_lock", "./dlm_lock", NULL);
+			exit(EXIT_FAILURE);
+
+		} else {
+			printf("alt fork pid %d\n", pid);
+
+			/* wait for child to get EX */
+			printf("alt wait child lock\n");
+			rv = wait_child_lock(pr_fd);
+			close(pr_fd);
+			close(cw_fd);
+			if (rv < 0) {
+				printf("alt child lock error\n");
+				break;
+			}
+
+			/* let other node start another dlm_lock process */
+			rv = inc_counter();
+			if (rv < 0) {
+				printf("alt inc_counter error\n");
+				break;
+			}
+
+			/* sleep random number of seconds */
+			n = rand_int(0, 4);
+			printf("alt sleep %d\n", n);
+			sleep(rand_int(0, 10));
+
+			/* kill child */
+			printf("alt kill pid %d\n", pid);
+			kill(pid, SIGKILL);
+
+			/* wait for child to exit */
+			printf("alt wait pid %d\n", pid);
+			waitpid(pid, &status, 0);
+
+			/* sleep random number of seconds */
+			n = rand_int(0, 2);
+			printf("alt sleep %d\n", n);
+			sleep(n);
+		}
+	}
+
+	return 0;
+}
+
diff --git a/dlm/dlm_kill/dlm_lock_loop.c b/dlm/dlm_kill/dlm_lock_loop.c
index 3ff8c13..0e57b4b 100644
--- a/dlm/dlm_kill/dlm_lock_loop.c
+++ b/dlm/dlm_kill/dlm_lock_loop.c
@@ -57,7 +57,7 @@ static int wait_child_lock(int pr_fd)
 
 	file = fdopen(pr_fd, "r");
 	if (!file) {
-		printf("fdopen error %d\n", errno);
+		printf("loop: fdopen error %d\n", errno);
 		return -1;
 	}
 
@@ -105,33 +105,34 @@ int main(int argc, char *argv[])
 			exit(EXIT_FAILURE);
 
 		} else {
-			printf("fork pid %d\n", pid);
+			printf("loop fork pid %d\n", pid);
 
 			/* wait for child to get EX */
-			printf("wait child lock\n");
+			printf("loop wait child lock\n");
 			rv = wait_child_lock(pr_fd);
 			close(pr_fd);
+			close(cw_fd);
 			if (rv < 0) {
-				printf("child lock error\n");
+				printf("loop child lock error\n");
 				break;
 			}
 
 			/* sleep random number of seconds */
 			n = rand_int(0, 4);
-			printf("sleep %d\n", n);
+			printf("loop sleep %d\n", n);
 			sleep(rand_int(0, 10));
 
 			/* kill child */
-			printf("kill pid %d\n", pid);
+			printf("loop kill pid %d\n", pid);
 			kill(pid, SIGKILL);
 
 			/* wait for child to exit */
-			printf("wait pid %d\n", pid);
+			printf("loop wait pid %d\n", pid);
 			waitpid(pid, &status, 0);
 
 			/* sleep random number of seconds */
 			n = rand_int(0, 2);
-			printf("sleep %d\n", n);
+			printf("loop sleep %d\n", n);
 			sleep(n);
 		}
 	}