path: root/dlm/dlm_master.c

/*
 * Copyright 2012 David Teigland
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License V2
 * as published by the Free Software Foundation.
 */

#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <dirent.h>
#include <inttypes.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include <signal.h>
#include <syslog.h>
#include <sys/time.h>
#include <asm/types.h>
#include <sys/socket.h>
#include <sys/poll.h>
#include <sys/un.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/errno.h>

#include "libdlm.h"

#define MAX_NODES 16

static int nodes[MAX_NODES];
static int nodes_count;
static int our_nodeid;

static dlm_lshandle_t *dh;
static int openclose = 0;
static int quiet = 0;
static int verbose = 0;
static int sleep_msec = 1000;

#define log_debug(fmt, args...) \
do { \
	if (!quiet) \
		printf(fmt "\n", ##args); \
} while (0)

#define log_error(fmt, args...) \
do { \
	printf("ERROR " fmt "\n", ##args); \
	exit(-1); \
} while (0)

static int rand_int(int a, int b)
{
	return a + (int) (((float)(b - a + 1)) * random() / (RAND_MAX+1.0)); 
}

static const char *mode_str(int mode)
{
        switch (mode) {
        case -1:
                return "IV";
        case LKM_NLMODE:
                return "NL";
        case LKM_CRMODE:
                return "CR";
        case LKM_CWMODE:
                return "CW";
        case LKM_PRMODE:
                return "PR";
        case LKM_PWMODE:
                return "PW";
        case LKM_EXMODE:
                return "EX";
        }
        return "??";
}

static void do_master(int acquire_mode, char *suffix)
{
	struct dlm_lksb lksb;
	char name[DLM_RESNAME_MAXLEN];
	int rv;

	snprintf(name, sizeof(name), "nodeid%d-%s", our_nodeid, suffix);

	memset(&lksb, 0, sizeof(lksb));

	printf("master \"%s\" ", name);
	fflush(stdout);

	rv = dlm_ls_lock_wait(dh, acquire_mode, &lksb, 0,
				name, strlen(name),
				0, NULL, NULL, NULL);

	if (rv < 0) {
		printf("lock_wait error %d\n", rv);
		return;
	}

	printf("0x%08x %s\n", lksb.sb_lkid, mode_str(acquire_mode));
}

static void ping_masters(int acquire_mode, int convert_mode, char *suffix,
			 uint32_t *lkids)
{
	struct dlm_lksb lksb;
	char name[DLM_RESNAME_MAXLEN];
	int rv, i;

	for (i = 0; i < nodes_count; i++) {
		if (nodes[i] == our_nodeid)
			continue;

		snprintf(name, sizeof(name), "nodeid%d-%s", nodes[i], suffix);

		memset(&lksb, 0, sizeof(lksb));

		if (acquire_mode > -1) {
			rv = dlm_ls_lock_wait(dh, acquire_mode, &lksb,
						LKF_NOQUEUE,
						name, strlen(name),
						0, NULL, NULL, NULL);
			if (rv < 0) {
				printf("lock_wait error %d\n", rv);
				return;
			}

			lkids[i] = lksb.sb_lkid;

			printf("remote \"%s\" 0x%08x %s\n", name, lksb.sb_lkid,
				mode_str(acquire_mode));
			continue;
		}

		if (convert_mode > -1) {
			lksb.sb_lkid = lkids[i];

			printf("ping \"%s\" 0x%08x %s", name, lksb.sb_lkid,
				mode_str(convert_mode));
			fflush(stdout);

			rv = dlm_ls_lock_wait(dh, convert_mode, &lksb,
						LKF_NOQUEUE | LKF_CONVERT,
						name, strlen(name),
						0, NULL, NULL, NULL);
			if (rv < 0)
				printf(" error %d\n", rv);
			else
				printf(" .\n");
			continue;
		}
	}
}

static int setup_nodes(void)
{
	char path[PATH_MAX];
	DIR *d;
	struct dirent *de;
	char local[8];
	int i, fd, rv, nodeid;

	memset(path, 0, PATH_MAX);
	snprintf(path, PATH_MAX, "/sys/kernel/config/dlm/cluster/comms");

	d = opendir(path);
	if (!d) {
		log_debug("%s: opendir failed: %d", path, errno);
		return -1;
	}

	while ((de = readdir(d))) {
		if (de->d_name[0] == '.')
			continue;

		nodeid = atoi(de->d_name);

		nodes[nodes_count++] = nodeid;

		memset(path, 0, PATH_MAX);
		snprintf(path, PATH_MAX, "/sys/kernel/config/dlm/cluster/comms/%s/local", de->d_name);

		fd = open(path, O_RDONLY);
		if (fd < 0) {
			perror("open");
			return -1;
		}

		rv = read(fd, local, sizeof(local));
		if (rv < 0) {
			perror("read local");
			return -1;
		}

		close(fd);

		if (atoi(local))
			our_nodeid = nodeid;
	}
	closedir(d);

	for (i = 0; i < nodes_count; i++)
		printf("nodeid %d\n", nodes[i]);
	printf("our_nodeid %d\n", our_nodeid);

	return 0;

}

static void print_usage(void)
{
	printf("dlm_master [ping]\n");
	printf("Options:\n");
	printf("\n");
	printf("  -s <ms>      Sleep milliseconds\n");
	printf("  -o           Open/close existing lockspace\n");
	printf("  -v           Verbose output\n");
	printf("  -q           Quiet output\n");
}

static void decode_arguments(int argc, char **argv)
{
	int cont = 1;
	int optchar;

	while (cont) {
		optchar = getopt(argc, argv, "vqohs:");

		switch (optchar) {

		case 's':
			sleep_msec = atoi(optarg);
			break;

		case 'o':
			openclose = 1;
			break;

		case 'v':
			verbose = 1;
			break;

		case 'q':
			quiet = 1;
			break;

		case 'h':
			print_usage();
			exit(EXIT_SUCCESS);
			break;

		case 'V':
			printf("%s (built %s %s)\n", argv[0], __DATE__, __TIME__);
			exit(EXIT_SUCCESS);
			break;

		case ':':
		case '?':
			fprintf(stderr, "Please use '-h' for usage.\n");
			exit(EXIT_FAILURE);
			break;

		case EOF:
			cont = 0;
			break;

		default:
			fprintf(stderr, "unknown option: %c\n", optchar);
			exit(EXIT_FAILURE);
			break;
		};
	}
}

int main(int argc, char *argv[])
{
	uint32_t *lkids_A;
	uint32_t *lkids_B;
	uint32_t *lkids_C;
	int rv, quit = 0;
	int do_ping = 0;

	if (argc > 1 && !strcmp(argv[1], "ping")) {
		do_ping = 1;
		argc--;
		argv++;
	}

	lkids_A = malloc(sizeof(uint32_t) * MAX_NODES);
	lkids_B = malloc(sizeof(uint32_t) * MAX_NODES);
	lkids_C = malloc(sizeof(uint32_t) * MAX_NODES);

	decode_arguments(argc, argv);

	if (openclose) {
		log_debug("dlm_open_lockspace...");

		dh = dlm_open_lockspace("dlm_master");
		if (!dh) {
			log_error("dlm_open_lockspace error %lu %d",
				  (unsigned long)dh, errno);
			return -ENOTCONN;
		}
	} else {
		log_debug("dlm_new_lockspace...");

		dh = dlm_new_lockspace("dlm_master", 0600, 0);
		if (!dh) {
			log_error("dlm_new_lockspace error %lu %d",
				  (unsigned long)dh, errno);
			return -ENOTCONN;
		}
	}

	rv = dlm_ls_pthread_init(dh);
	if (rv < 0) {
		log_error("dlm_ls_pthread_init error %d %d", rv, errno);
		goto done;
	}

	setup_nodes();

	/* We do both upconvert/downconvert on A/B for each cycle because
	   the downconvert does not include a reply from the master and we
	   want there to be messages back and forth for each cycle. */

	if (do_ping) {
		ping_masters(LKM_NLMODE, -1, "A", lkids_A);
		ping_masters(LKM_PRMODE, -1, "B", lkids_B);
		ping_masters(LKM_NLMODE, -1, "C", lkids_C);

		while (1) {
			usleep(sleep_msec*1000);
			ping_masters(-1, LKM_PRMODE, "A", lkids_A);
			ping_masters(-1, LKM_NLMODE, "B", lkids_B);
			ping_masters(-1, LKM_EXMODE, "C", lkids_C);
			usleep(sleep_msec*1000);
			ping_masters(-1, LKM_PRMODE, "B", lkids_B);
			ping_masters(-1, LKM_NLMODE, "A", lkids_A);
			ping_masters(-1, LKM_EXMODE, "C", lkids_C);
		}
	} else {
		do_master(LKM_CRMODE, "A");
		do_master(LKM_CRMODE, "B");
		do_master(LKM_EXMODE, "C");

		while (1) {
			sleep(10);
		}
	}

 done:
	if (openclose) {
		log_debug("dlm_close_lockspace");

		rv = dlm_close_lockspace(dh);
		if (rv < 0)
			log_error("dlm_close_lockspace error %d %d",
				  rv, errno);
	} else {
		log_debug("dlm_release_lockspace");

		rv = dlm_release_lockspace("dlm_master", dh, 1);
		if (rv < 0)
			log_error("dlm_release_lockspace error %d %d",
				  rv, errno);
	}

	return 0;
}