ctdbd: Replace lockwait with locking API and remove ctdb_lockwait.c

Signed-off-by: Amitay Isaacs <amitay@gmail.com>

(This used to be ctdb commit 2126795153dacb255e441abcb36ee05107b6282a)

7 files changed, 7 insertions, 262 deletions

diff --git a/ctdb/Makefile.in b/ctdb/Makefile.in
index 2d5ef6253a..0e365f671e 100755
--- a/ctdb/Makefile.in
+++ b/ctdb/Makefile.in
@@ -80,7 +80,7 @@ CTDB_CLIENT_OBJ = client/ctdb_client.o \
 	$(CTDB_COMMON_OBJ) $(POPT_OBJ) $(UTIL_OBJ) $(TALLOC_OBJ) $(TDB_OBJ) \
 	@LIBREPLACEOBJ@ $(EXTRA_OBJ) $(TEVENT_OBJ) $(SOCKET_WRAPPER_OBJ)
 
-CTDB_SERVER_OBJ = server/ctdbd.o server/ctdb_daemon.o server/ctdb_lockwait.o \
+CTDB_SERVER_OBJ = server/ctdbd.o server/ctdb_daemon.o \
 	server/ctdb_recoverd.o server/ctdb_recover.o server/ctdb_freeze.o \
 	server/ctdb_tunables.o server/ctdb_monitor.o server/ctdb_server.o \
 	server/ctdb_control.o server/ctdb_call.o server/ctdb_ltdb_server.o \
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 35a1210c8e..7e6f9dea21 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -567,8 +567,6 @@ struct ctdb_db_context {
 	struct ctdb_vacuum_handle *vacuum_handle;
 	char *unhealthy_reason;
 	int pending_requests;
-	struct lockwait_handle *lockwait_active;
-	struct lockwait_handle *lockwait_overflow;
 	struct revokechild_handle *revokechild_active;
 	struct ctdb_persistent_state *persistent_state;
 	struct trbt_tree *delete_queue;
@@ -840,10 +838,6 @@ int ctdb_daemon_send_message(struct ctdb_context *ctdb, uint32_t pnn,
 			     uint64_t srvid, TDB_DATA data);
 
 
-struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
-				      TDB_DATA key,
-				      void (*callback)(void *), void *private_data);
-
 struct ctdb_call_state *ctdb_daemon_call_send(struct ctdb_db_context *ctdb_db, 
 					      struct ctdb_call *call);
 
diff --git a/ctdb/include/ctdb_protocol.h b/ctdb/include/ctdb_protocol.h
index 78e7437f1d..14b44d5c27 100644
--- a/ctdb/include/ctdb_protocol.h
+++ b/ctdb/include/ctdb_protocol.h
@@ -671,8 +671,6 @@ struct ctdb_statistics {
 	} locks;
 	uint32_t total_calls;
 	uint32_t pending_calls;
-	uint32_t lockwait_calls;
-	uint32_t pending_lockwait_calls;
 	uint32_t childwrite_calls;
 	uint32_t pending_childwrite_calls;
 	uint32_t memory_used;
@@ -680,7 +678,6 @@ struct ctdb_statistics {
 	uint32_t max_hop_count;
 	uint32_t hop_count_bucket[MAX_COUNT_BUCKETS];
 	struct latency_counter call_latency;
-	struct latency_counter lockwait_latency;
 	struct latency_counter childwrite_latency;
 	uint32_t num_recoveries;
 	struct timeval statistics_start_time;
diff --git a/ctdb/server/ctdb_lockwait.c b/ctdb/server/ctdb_lockwait.c
deleted file mode 100644
index 2339dc861c..0000000000
--- a/ctdb/server/ctdb_lockwait.c
+++ /dev/null
@@ -1,234 +0,0 @@
-/* 
-   wait for a tdb chain lock
-
-   Copyright (C) Andrew Tridgell  2006
-
-   This program is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published by
-   the Free Software Foundation; either version 3 of the License, or
-   (at your option) any later version.
-   
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU General Public License for more details.
-   
-   You should have received a copy of the GNU General Public License
-   along with this program; if not, see <http://www.gnu.org/licenses/>.
-*/
-
-#include "includes.h"
-#include "system/filesys.h"
-#include "system/wait.h"
-#include "db_wrap.h"
-#include "lib/tdb/include/tdb.h"
-#include "lib/util/dlinklist.h"
-#include "../include/ctdb_private.h"
-
-
-struct lockwait_handle {
-	struct lockwait_handle *next, *prev;
-	struct ctdb_context *ctdb;
-	struct ctdb_db_context *ctdb_db;
-	struct fd_event *fde;
-	int fd[2];
-	pid_t child;
-	void *private_data;
-	void (*callback)(void *);
-	TDB_DATA key;
-	struct timeval start_time;
-};
-
-/* If we managed to obtain a lock, find any overflow records which wanted the
- * same one and do all the callbacks at once. */
-static void do_overflow(struct ctdb_db_context *ctdb_db,
-			TDB_DATA key)
-{
-	struct lockwait_handle *i, *next;
-	TALLOC_CTX *tmp_ctx = talloc_new(ctdb_db);
-
-	for (i = ctdb_db->lockwait_overflow; i; i = next) {
-		/* Careful: destructor removes it from list! */
-		next = i->next;
-		if (key.dsize == i->key.dsize
-		    && memcmp(key.dptr, i->key.dptr, key.dsize) == 0) {
-			/* Callback might free them, so reparent. */
-			talloc_steal(tmp_ctx, i);
-			i->callback(i->private_data);
-		}
-	}
-
-	/* This will free them if callback didn't. */
-	talloc_free(tmp_ctx);
-
-	/* Remove one from the overflow queue if there is one. */
-	if (ctdb_db->lockwait_overflow) {
-		i = ctdb_db->lockwait_overflow;
-		ctdb_lockwait(ctdb_db, i->key, i->callback, i->private_data);
-		talloc_free(i);
-	}
-}
-
-static int lockwait_destructor(struct lockwait_handle *h)
-{
-	CTDB_DECREMENT_STAT(h->ctdb, pending_lockwait_calls);
-	ctdb_kill(h->ctdb, h->child, SIGKILL);
-	h->ctdb_db->pending_requests--;
-	DLIST_REMOVE(h->ctdb_db->lockwait_active, h);
-	return 0;
-}
-
-static void lockwait_handler(struct event_context *ev, struct fd_event *fde, 
-			     uint16_t flags, void *private_data)
-{
-	struct lockwait_handle *h = talloc_get_type(private_data, 
-						     struct lockwait_handle);
-	void (*callback)(void *) = h->callback;
-	void *p = h->private_data;
-	TDB_DATA key = h->key;
-	struct tdb_context *tdb = h->ctdb_db->ltdb->tdb;
-	TALLOC_CTX *tmp_ctx = talloc_new(ev);
-
-	key.dptr = talloc_memdup(tmp_ctx, key.dptr, key.dsize);
-	h->ctdb_db->pending_requests--;
-
-	CTDB_UPDATE_LATENCY(h->ctdb, h->ctdb_db, "lockwait", lockwait_latency, h->start_time);
-
-	/* the handle needs to go away when the context is gone - when
-	   the handle goes away this implicitly closes the pipe, which
-	   kills the child holding the lock */
-	talloc_steal(tmp_ctx, h);
-
-	if (h->ctdb->flags & CTDB_FLAG_TORTURE) {
-		if (tdb_chainlock_nonblock(tdb, key) == 0) {
-			ctdb_fatal(h->ctdb, "got chain lock while lockwait child active");
-		}
-	}
-
-	tdb_chainlock_mark(tdb, key);
-	callback(p);
-	if (h->ctdb_db->lockwait_overflow) {
-		do_overflow(h->ctdb_db, key);
-	}
-	tdb_chainlock_unmark(tdb, key);
-
-	talloc_free(tmp_ctx);
-}
-
-
-static int overflow_lockwait_destructor(struct lockwait_handle *h)
-{
-	CTDB_DECREMENT_STAT(h->ctdb, pending_lockwait_calls);
-	DLIST_REMOVE(h->ctdb_db->lockwait_overflow, h);
-	return 0;
-}
-
-/*
-  setup a non-blocking chainlock on a tdb record. If this function
-  returns NULL then it could not get the chainlock. Otherwise it
-  returns a opaque handle, and will call callback() once it has
-  managed to get the chainlock. You can cancel it by using talloc_free
-  on the returned handle.
-
-  It is the callers responsibility to unlock the chainlock once
-  acquired
- */
-struct lockwait_handle *ctdb_lockwait(struct ctdb_db_context *ctdb_db,
-				      TDB_DATA key,
-				      void (*callback)(void *private_data),
-				      void *private_data)
-{
-	struct lockwait_handle *result, *i;
-	int ret;
-	pid_t parent = getpid();
-
-	CTDB_INCREMENT_STAT(ctdb_db->ctdb, lockwait_calls);
-	CTDB_INCREMENT_STAT(ctdb_db->ctdb, pending_lockwait_calls);
-
-	if (!(result = talloc_zero(private_data, struct lockwait_handle))) {
-		CTDB_DECREMENT_STAT(ctdb_db->ctdb, pending_lockwait_calls);
-		return NULL;
-	}
-
-	result->callback = callback;
-	result->private_data = private_data;
-	result->ctdb = ctdb_db->ctdb;
-	result->ctdb_db = ctdb_db;
-	result->key = key;
-
-	/* If we already have a lockwait child for this request, then put this
-	   request on the overflow queue straight away
-	 */
-	for (i = ctdb_db->lockwait_active; i; i = i->next) {
-		if (key.dsize == i->key.dsize
-		    && memcmp(key.dptr, i->key.dptr, key.dsize) == 0) {
-			DLIST_ADD_END(ctdb_db->lockwait_overflow, result, NULL);
-			talloc_set_destructor(result, overflow_lockwait_destructor);
-			return result;
-		}
-	}
-
-	/* Don't fire off too many children at once! */
-	if (ctdb_db->pending_requests > 200) {
-		DLIST_ADD_END(ctdb_db->lockwait_overflow, result, NULL);
-		talloc_set_destructor(result, overflow_lockwait_destructor);
-		DEBUG(DEBUG_DEBUG, (__location__ " Created overflow for %s\n",
-				    ctdb_db->db_name));
-		return result;
-	}
-
-	ret = pipe(result->fd);
-
-	if (ret != 0) {
-		talloc_free(result);
-		CTDB_DECREMENT_STAT(ctdb_db->ctdb, pending_lockwait_calls);
-		return NULL;
-	}
-
-	result->child = ctdb_fork(ctdb_db->ctdb);
-
-	if (result->child == (pid_t)-1) {
-		close(result->fd[0]);
-		close(result->fd[1]);
-		talloc_free(result);
-		CTDB_DECREMENT_STAT(ctdb_db->ctdb, pending_lockwait_calls);
-		return NULL;
-	}
-
-	if (result->child == 0) {
-		char c = 0;
-		close(result->fd[0]);
-		debug_extra = talloc_asprintf(NULL, "chainlock-%s:", ctdb_db->db_name);
-		tdb_chainlock(ctdb_db->ltdb->tdb, key);
-		write(result->fd[1], &c, 1);
-		/* make sure we die when our parent dies */
-		while (ctdb_kill(ctdb_db->ctdb, parent, 0) == 0 || errno != ESRCH) {
-			sleep(5);
-		}
-		_exit(0);
-	}
-
-	close(result->fd[1]);
-	set_close_on_exec(result->fd[0]);
-
-	/* This is an active lockwait child process */
-	DLIST_ADD_END(ctdb_db->lockwait_active, result, NULL);
-
-	DEBUG(DEBUG_DEBUG, (__location__ " Created PIPE FD:%d to child lockwait process\n", result->fd[0]));
-
-	ctdb_db->pending_requests++;
-	talloc_set_destructor(result, lockwait_destructor);
-
-	result->fde = event_add_fd(ctdb_db->ctdb->ev, result, result->fd[0],
-				   EVENT_FD_READ, lockwait_handler,
-				   (void *)result);
-	if (result->fde == NULL) {
-		talloc_free(result);
-		CTDB_DECREMENT_STAT(ctdb_db->ctdb, pending_lockwait_calls);
-		return NULL;
-	}
-	tevent_fd_set_auto_close(result->fde);
-
-	result->start_time = timeval_current();
-	return result;
-}
diff --git a/ctdb/server/ctdb_ltdb_server.c b/ctdb/server/ctdb_ltdb_server.c
index 00d5566dc2..e01206790f 100644
--- a/ctdb/server/ctdb_ltdb_server.c
+++ b/ctdb/server/ctdb_ltdb_server.c
@@ -248,7 +248,7 @@ struct lock_fetch_state {
 /*
   called when we should retry the operation
  */
-static void lock_fetch_callback(void *p)
+static void lock_fetch_callback(void *p, bool locked)
 {
 	struct lock_fetch_state *state = talloc_get_type(p, struct lock_fetch_state);
 	if (!state->ignore_generation &&
@@ -271,9 +271,9 @@ static void lock_fetch_callback(void *p)
    1) tries to get the chainlock. If it succeeds, then it returns 0
 
    2) if it fails to get a chainlock immediately then it sets up a
-   non-blocking chainlock via ctdb_lockwait, and when it gets the
+   non-blocking chainlock via ctdb_lock_record, and when it gets the
    chainlock it re-submits this ctdb request to the main packet
-   receive function
+   receive function.
 
    This effectively queues all ctdb requests that cannot be
    immediately satisfied until it can get the lock. This means that
@@ -293,7 +293,7 @@ int ctdb_ltdb_lock_requeue(struct ctdb_db_context *ctdb_db,
 {
 	int ret;
 	struct tdb_context *tdb = ctdb_db->ltdb->tdb;
-	struct lockwait_handle *h;
+	struct lock_request *lreq;
 	struct lock_fetch_state *state;
 	
 	ret = tdb_chainlock_nonblock(tdb, key);
@@ -325,15 +325,14 @@ int ctdb_ltdb_lock_requeue(struct ctdb_db_context *ctdb_db,
 	state->ignore_generation = ignore_generation;
 
 	/* now the contended path */
-	h = ctdb_lockwait(ctdb_db, key, lock_fetch_callback, state);
-	if (h == NULL) {
+	lreq = ctdb_lock_record(ctdb_db, key, true, lock_fetch_callback, state);
+	if (lreq == NULL) {
 		return -1;
 	}
 
 	/* we need to move the packet off the temporary context in ctdb_input_pkt(),
 	   so it won't be freed yet */
 	talloc_steal(state, hdr);
-	talloc_steal(state, h);
 
 	/* now tell the caller than we will retry asynchronously */
 	return -2;
diff --git a/ctdb/tests/src/ctdbd_test.c b/ctdb/tests/src/ctdbd_test.c
index 5638a77e59..6f045a3f7a 100644
--- a/ctdb/tests/src/ctdbd_test.c
+++ b/ctdb/tests/src/ctdbd_test.c
@@ -58,7 +58,6 @@ void ctdb_load_nodes_file(struct ctdb_context *ctdb) {}
 
 /* CTDB_SERVER_OBJ */
 #include "server/ctdb_daemon.c"
-#include "server/ctdb_lockwait.c"
 #include "server/ctdb_recoverd.c"
 #include "server/ctdb_recover.c"
 #include "server/ctdb_freeze.c"
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index fcc44bb2b6..5e884df339 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -340,8 +340,6 @@ static void show_statistics(struct ctdb_statistics *s, int show_header)
 		STATISTICS_FIELD(locks.num_failed),
 		STATISTICS_FIELD(total_calls),
 		STATISTICS_FIELD(pending_calls),
-		STATISTICS_FIELD(lockwait_calls),
-		STATISTICS_FIELD(pending_lockwait_calls),
 		STATISTICS_FIELD(childwrite_calls),
 		STATISTICS_FIELD(pending_childwrite_calls),
 		STATISTICS_FIELD(memory_used),
@@ -414,11 +412,6 @@ static void show_statistics(struct ctdb_statistics *s, int show_header)
 		printf("%.6f:", s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0);
 		printf("%.6f:", s->call_latency.max);
 
-		printf("%d:", s->lockwait_latency.num);
-		printf("%.6f:", s->lockwait_latency.min);
-		printf("%.6f:", s->lockwait_latency.num?s->lockwait_latency.total/s->lockwait_latency.num:0.0);
-		printf("%.6f:", s->lockwait_latency.max);
-
 		printf("%d:", s->childwrite_latency.num);
 		printf("%.6f:", s->childwrite_latency.min);
 		printf("%.6f:", s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0);
@@ -462,7 +455,6 @@ static void show_statistics(struct ctdb_statistics *s, int show_header)
 		printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "reclock_recd       MIN/AVG/MAX", s->reclock.recd.min, s->reclock.recd.num?s->reclock.recd.total/s->reclock.recd.num:0.0, s->reclock.recd.max, s->reclock.recd.num);
 
 		printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "call_latency       MIN/AVG/MAX", s->call_latency.min, s->call_latency.num?s->call_latency.total/s->call_latency.num:0.0, s->call_latency.max, s->call_latency.num);
-		printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "lockwait_latency   MIN/AVG/MAX", s->lockwait_latency.min, s->lockwait_latency.num?s->lockwait_latency.total/s->lockwait_latency.num:0.0, s->lockwait_latency.max, s->lockwait_latency.num);
 		printf(" %-30s     %.6f/%.6f/%.6f sec out of %d\n", "childwrite_latency MIN/AVG/MAX", s->childwrite_latency.min, s->childwrite_latency.num?s->childwrite_latency.total/s->childwrite_latency.num:0.0, s->childwrite_latency.max, s->childwrite_latency.num);
 	}
 
@@ -503,8 +495,6 @@ static int control_statistics_all(struct ctdb_context *ctdb)
 			MAX(statistics.max_hop_count, s1.max_hop_count);
 		statistics.call_latency.max = 
 			MAX(statistics.call_latency.max, s1.call_latency.max);
-		statistics.lockwait_latency.max = 
-			MAX(statistics.lockwait_latency.max, s1.lockwait_latency.max);
 	}
 	talloc_free(nodes);
 	printf("Gathered statistics for %u nodes\n", num_nodes);