libctdb: test infrastructure

This introduces 'ctdb-test', a program for testing libctdb.  It takes
commands on standard input (with reduced functionality) or an input file.

It still needs some cleaning up, but you can uncover a bug in libctdb
today simply by running a simple attachdb test:

	$ ctdb-test tests/attachdb1.txt

It will print out a crash, and the path of successful and failed
operations which lead to it:

	...
	Child signalled 11 on failure path: [malloc]:1:S[socket]:1:S[connect]:1:S[malloc]:1:S[malloc]:1:S[malloc]:1:S[malloc]:4:S[malloc]:4:F

Feed that failure path into ctdb-test using --failpath (under a debugger):

	gdb --args ctdb-test tests/attachdb1.txt --failpath=[malloc]:1:S[socket]:1:S[connect]:1:S[malloc]:1:S[malloc]:1:S[malloc]:1:S[malloc]:4:S[malloc]:4:F

And you hit the exact error.

It is based on the fork-to-fail model of nfsim.  The relevant parts are
from page 154 of the proceedings of 2005 Ottawa Linux Symposium Volume II:
	http://www.linuxsymposium.org/2005/linuxsymposium_procv2.pdf

Or our presentation of same (from slide 21):
	http://ozlabs.org/~jk/projects/nfsim/nfsim.sxi

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>


(This used to be ctdb commit b4aab4199a57898877b6545a54f212087ed4b35a)

22 files changed, 2487 insertions, 0 deletions

diff --git a/ctdb/libctdb/test/Makefile b/ctdb/libctdb/test/Makefile
new file mode 100644
index 0000000000..e5aa09b37d
--- /dev/null
+++ b/ctdb/libctdb/test/Makefile
@@ -0,0 +1,23 @@
+CFLAGS=-Wall -g -I../../include/ -I../../lib/talloc/ -I../../lib/tdb/include/ -I../../lib/util/
+LDLIBS=-lreadline
+
+USAGE_SOURCES := $(shell grep -l 'XML Argument' *.c)
+HELP_SOURCES := $(shell grep -l 'XML Help' *.c)
+
+ctdb-test: $(patsubst %.c,%.o,$(wildcard *.c)) generated-usage.o ../../talloc.o ../../common/check.o ../../common/error.o ../../common/freelist.o ../../common/io.o ../../common/lock.o ../../common/open.o ../../common/tdb.o ../../common/transaction.o ../../common/traverse.o
+
+$(patsubst %.c,%.o,$(wildcard *.c)): .help-files
+
+.PHONY: links
+links:
+	cd tools && ./create-links
+
+generated-usage.o: generated-usage.c links .help-files
+generated-usage.c: $(USAGE_SOURCES) tools/gen-usage links
+	tools/gen-usage $(USAGE_SOURCES) >$@
+
+.help-files: $(HELP_SOURCES) links
+	set -e; for f in $(HELP_SOURCES); do tools/gen-help $$f; done; touch .help-files
+
+clean:
+	rm -f ctdb-test .help-files generated-* *.o
diff --git a/ctdb/libctdb/test/attachdb.c b/ctdb/libctdb/test/attachdb.c
new file mode 100644
index 0000000000..bb4ce2d0bf
--- /dev/null
+++ b/ctdb/libctdb/test/attachdb.c
@@ -0,0 +1,166 @@
+#include "utils.h"
+#include "log.h"
+#include "tui.h"
+#include "ctdb-test.h"
+#include <ctdb.h>
+#include <tdb.h>
+#include <talloc.h>
+#include <dlinklist.h>
+#include <errno.h>
+
+static unsigned int db_num;
+static struct db *dbs;
+
+struct db {
+	struct db *next, *prev;
+	struct ctdb_db *db;
+	const char *name;
+	unsigned int num;
+	bool persistent;
+	uint32_t tdb_flags;
+};
+
+static void attachdb_help(int agc, char **argv)
+{
+#include "generated-attachdb-help:attachdb"
+/*** XML Help:
+    <section id="c:attachdb">
+     <title><command>attachdb</command></title>
+     <para>Attach to a ctdb database</para>
+     <cmdsynopsis>
+      <command>attachdb</command>
+      <arg choice="req"><replaceable>name</replaceable></arg>
+      <arg choice="req"><replaceable>persistent</replaceable></arg>
+      <arg choice="opt"><replaceable>tdb-flags</replaceable></arg>
+     </cmdsynopsis>
+     <para>Attach to the database of the given <replaceable>name</replaceable>.
+	<replaceable>persistent</replaceable> is 'true' or 'false', an
+
+	<replaceable>tdb-flags</replaceable> an optional one or more
+	comma-separated values:</para>
+     <variablelist>
+      <varlistentry>
+       <term>SEQNUM</term>
+       <listitem>
+        <para>Use sequence numbers on the tdb</para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+
+     <para>It uses a consecutive number for each attached db to
+     identify it for other ctdb-test commands, starting with 1.</para>
+
+     <para>Without any options, the <command>attachdb</command>
+      command lists all databases attached.</para>
+     </section>
+*/
+}
+
+static void detachdb_help(int agc, char **argv)
+{
+#include "generated-attachdb-help:detachdb"
+/*** XML Help:
+    <section id="c:detachdb">
+     <title><command>detachdb</command></title>
+     <para>Detach from a ctdb database</para>
+     <cmdsynopsis>
+      <command>detachdb</command>
+      <arg choice="req"><replaceable>number</replaceable></arg>
+     </cmdsynopsis>
+     <para>Detach from the database returned by <command>attachdb</command>.
+     </para>
+     </section>
+*/
+}
+static int db_destructor(struct db *db)
+{
+	ctdb_detachdb(get_ctdb(), db->db);
+	DLIST_REMOVE(dbs, db);
+	return 0;
+}
+
+static bool detachdb(int argc, char **argv)
+{
+	struct db *db;
+
+	if (argc != 2) {
+		log_line(LOG_ALWAYS, "Need database number");
+		return false;
+	}
+
+	for (db = dbs; db; db = db->next) {
+		if (db->num == atoi(argv[1]))
+			break;
+	}
+	if (!db) {
+		log_line(LOG_ALWAYS, "Unknown db number %s", argv[1]);
+		return false;
+	}
+	talloc_free(db);
+	return true;
+}
+
+static bool attachdb(int argc, char **argv)
+{
+	struct db *db;
+
+	if (!get_ctdb()) {
+		log_line(LOG_ALWAYS, "No ctdb connection");
+		return false;
+	}
+
+	if (argc == 1) {
+		log_line(LOG_UI, "Databases currently attached:");
+		for (db = dbs; db; db = db->next) {
+			log_line(LOG_ALWAYS, "  %i: %s: %s %u",
+				 db->num, db->name,
+				 db->persistent
+				 ? "persistent" : "not persistent",
+				 db->tdb_flags);
+		}
+		return true;
+	}
+	if (argc != 3 && argc != 4) {
+		log_line(LOG_ALWAYS, "Need 2 or 3 args");
+		return false;
+	}
+	db = talloc(working, struct db);
+	db->name = talloc_strdup(db, argv[1]);
+	if (strcasecmp(argv[2], "true") == 0)
+		db->persistent = true;
+	else if (strcasecmp(argv[2], "false") == 0)
+		db->persistent = false;
+	else {
+		log_line(LOG_ALWAYS, "persistent should be true or false");
+		talloc_free(db);
+		return false;
+	}
+	db->tdb_flags = 0;
+	if (argc == 4) {
+		if (strcasecmp(argv[3], "seqnum") == 0)
+			db->tdb_flags |= TDB_SEQNUM;
+		else {
+			log_line(LOG_ALWAYS, "invalid tdb-flags");
+			talloc_free(db);
+			return false;
+		}
+	}
+	db->db = ctdb_attachdb(get_ctdb(), db->name, db->persistent,
+			       db->tdb_flags);
+	if (!db->db) {
+		log_line(LOG_UI, "ctdb_attachdb: %s", strerror(errno));
+		return false;
+	}
+	db->num = ++db_num;
+	DLIST_ADD(dbs, db);
+	talloc_set_destructor(db, db_destructor);
+	log_line(LOG_UI, "attached: %u", db->num);
+	return true;
+}
+
+static void attachdb_init(void)
+{
+	tui_register_command("attachdb", attachdb, attachdb_help);
+	tui_register_command("detachdb", detachdb, detachdb_help);
+}
+init_call(attachdb_init);
diff --git a/ctdb/libctdb/test/ctdb-test.c b/ctdb/libctdb/test/ctdb-test.c
new file mode 100644
index 0000000000..b4c12ff9ab
--- /dev/null
+++ b/ctdb/libctdb/test/ctdb-test.c
@@ -0,0 +1,453 @@
+/*
+   test driver for libctdb
+
+   Copyright (C) Rusty Russell 2010
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <poll.h>
+#include <talloc.h>
+#include <tdb.h>
+
+/* We replace the following functions, for finer control. */
+#define poll(fds, nfds, timeout) ctdb_test_poll((fds), (nfds), (timeout), __location__)
+#define malloc(size) ctdb_test_malloc((size), __location__)
+#define free(ptr) ctdb_test_free((ptr), __location__)
+#define realloc(ptr, size) ctdb_test_realloc((ptr), (size), __location__)
+#define read(fd, buf, count) ctdb_test_read((fd), (buf), (count), __location__)
+#define write(fd, buf, count) ctdb_test_write((fd), (buf), (count), __location__)
+#define socket(domain, type, protocol) ctdb_test_socket((domain), (type), (protocol), __location__)
+#define connect(sockfd, addr, addrlen) ctdb_test_connect((sockfd), (addr), (addrlen), __location__)
+
+#define tdb_open_ex(name, hash_size, tdb_flags, open_flags, mode, log_ctx, hash_fn) ctdb_test_tdb_open_ex((name), (hash_size), (tdb_flags), (open_flags), (mode), (log_ctx), (hash_fn), __location__)
+#define tdb_fetch(tdb, key) ctdb_test_tdb_fetch((tdb), (key))
+
+/* Implement these if they're ever used. */
+#define calloc ctdb_test_calloc
+#define select ctdb_test_select
+#define epoll_wait ctdb_test_epoll_wait
+#define epoll_ctl ctdb_test_epoll_ctl
+#define tdb_open ctdb_test_tdb_open
+
+static int ctdb_test_poll(struct pollfd *fds, nfds_t nfds, int timeout, const char *location);
+static void *ctdb_test_malloc(size_t size, const char *location);
+static void ctdb_test_free(void *ptr, const char *location);
+static void *ctdb_test_realloc(void *ptr, size_t size, const char *location);
+static ssize_t ctdb_test_read(int fd, void *buf, size_t count, const char *location);
+static ssize_t ctdb_test_write(int fd, const void *buf, size_t count, const char *location);
+static int ctdb_test_socket(int domain, int type, int protocol, const char *location);
+static int ctdb_test_connect(int sockfd, const struct sockaddr *addr,
+			     socklen_t addrlen, const char *location);
+static struct tdb_context *ctdb_test_tdb_open_ex(const char *name,
+						 int hash_size, int tdb_flags,
+						 int open_flags, mode_t mode,
+						 const struct tdb_logging_context *log_ctx,
+						 tdb_hash_func hash_fn, const char *location);
+static TDB_DATA ctdb_test_tdb_fetch(struct tdb_context *tdb, TDB_DATA key);
+
+#include "../sync.c"
+#include "../control.c"
+#include "../ctdb.c"
+#include "../io_elem.c"
+#include "../local_tdb.c"
+#include "../logging.c"
+#include "../messages.c"
+
+#undef poll
+#undef malloc
+#undef realloc
+#undef read
+#undef write
+#undef socket
+#undef connect
+#undef tdb_open_ex
+#undef calloc
+#undef select
+#undef epoll_wait
+#undef epoll_ctl
+#undef tdb_open
+#undef tdb_fetch
+
+#include "ctdb-test.h"
+#include "utils.h"
+#include "tui.h"
+#include "log.h"
+#include "failtest.h"
+#include "expect.h"
+#include <err.h>
+
+/* Talloc contexts */
+void *allocations;
+void *working;
+
+static void run_inits(void)
+{
+	/* Linker magic creates these to delineate section. */
+	extern initcall_t __start_init_call[], __stop_init_call[];
+	initcall_t *p;
+
+	for (p = __start_init_call; p < __stop_init_call; p++)
+		(*p)();
+}
+
+static void print_license(void)
+{
+	printf("ctdb-test, Copyright (C) 2010 Jeremy Kerr, Rusty Russell\n"
+	       "ctdb-test comes with ABSOLUTELY NO WARRANTY; see COPYING.\n"
+	       "This is free software, and you are welcome to redistribute\n"
+	       "it under certain conditions; see COPYING for details.\n");
+}
+
+/*** XML Argument:
+    <section id="a:echo">
+     <title><option>--echo</option>, <option>-x</option></title>
+     <subtitle>Echo commands as they are executed</subtitle>
+     <para>ctdb-test will echo each command before it is executed. Useful when
+      commands are read from a file</para>
+    </section>
+*/
+static void cmdline_echo(struct option *opt)
+{
+	tui_echo_commands = 1;
+}
+cmdline_opt("echo", 0, 'x', cmdline_echo);
+
+/*** XML Argument:
+    <section id="a:quiet">
+     <title><option>--quiet</option>, <option>-q</option></title>
+     <subtitle>Run quietly</subtitle>
+     <para>Causes ctdb-test to reduce its output to the minimum possible - no prompt
+      is displayed, and most warning messages are suppressed
+     </para>
+    </section>
+*/
+static void cmdline_quiet(struct option *opt)
+{
+	tui_quiet = 1;
+}
+cmdline_opt("quiet", 0, 'q', cmdline_quiet);
+
+/*** XML Argument:
+    <section id="a:exit">
+     <title><option>--exit</option>, <option>-e</option></title>
+     <subtitle>Exit on error</subtitle>
+     <para>If <option>--exit</option> is specified, ctdb-test will exit (with a
+     non-zero error code) on the first script error it encounters (eg an
+     expect command does not match). This is the default when invoked as a
+     non-interactive script.</para>
+    </section>
+*/
+static void cmdline_abort_on_fail(struct option *opt)
+{
+	tui_abort_on_fail = 1;
+}
+cmdline_opt("exit", 0, 'e', cmdline_abort_on_fail);
+
+/*** XML Argument:
+    <section id="a:help">
+     <title><option>--help</option></title>
+     <subtitle>Print usage information</subtitle>
+     <para>Causes ctdb-test to print its command line arguments and then exit</para>
+    </section>
+*/
+static void cmdline_help(struct option *opt)
+{
+	print_license();
+	print_usage();
+	exit(EXIT_SUCCESS);
+}
+cmdline_opt("help", 0, 'h', cmdline_help);
+
+extern struct cmdline_option __start_cmdline[], __stop_cmdline[];
+
+static struct cmdline_option *get_cmdline_option(int opt)
+{
+	struct cmdline_option *copt;
+
+	/* if opt is < '0', we have been passed a long option, which is
+	 * indexed directly */
+	if (opt < '0')
+		return __start_cmdline + opt;
+
+	/* otherwise search for the short option in the .val member */
+	for (copt = __start_cmdline; copt < __stop_cmdline; copt++)
+		if (copt->opt.val == opt)
+			return copt;
+
+	return NULL;
+}
+
+static struct option *get_cmdline_options(void)
+{
+	struct cmdline_option *copts;
+	struct option *opts;
+	unsigned int x, n_opts;
+
+	n_opts = ((void *)__stop_cmdline - (void *)__start_cmdline) /
+		sizeof(struct cmdline_option);
+
+	opts = talloc_zero_array(NULL, struct option, n_opts + 1);
+	copts = __start_cmdline;
+
+	for (x = 0; x < n_opts; x++) {
+		unsigned int y;
+
+		if (copts[x].opt.has_arg > 2)
+			errx(1, "Bad argument `%s'", copts[x].opt.name);
+
+		for (y = 0; y < x; y++)
+			if ((copts[x].opt.val && copts[x].opt.val
+						== opts[y].val)
+					|| streq(copts[x].opt.name,
+						opts[y].name))
+				errx(1, "Conflicting arguments %s = %s\n",
+				     copts[x].opt.name, opts[y].name);
+
+		opts[x] = copts[x].opt;
+		opts[x].val = x;
+	}
+
+	return opts;
+}
+
+static char *get_cmdline_optstr(void)
+{
+	struct cmdline_option *copts;
+	unsigned int x, n_opts;
+	char *optstr, tmpstr[3], *colonstr = "::";
+
+	n_opts = ((void *)__stop_cmdline - (void *)__start_cmdline) /
+		sizeof(struct cmdline_option);
+
+	optstr = talloc_size(NULL, 3 * n_opts * sizeof(*optstr) + 1);
+	*optstr = '\0';
+
+	copts = __start_cmdline;
+
+	for (x = 0; x < n_opts; x++) {
+		if (!copts[x].opt.val)
+			continue;
+		snprintf(tmpstr, 4, "%c%s", copts[x].opt.val,
+			colonstr + 2 - copts[x].opt.has_arg);
+		strcat(optstr, tmpstr);
+	}
+	return optstr;
+}
+
+static int ctdb_test_poll(struct pollfd *fds, nfds_t nfds, int timeout,
+			  const char *location)
+{
+	if (should_i_fail("poll")) {
+		errno = EINVAL;
+		return -1;
+	}
+	return poll(fds, nfds, timeout);
+}
+
+static void *ctdb_test_malloc(size_t size, const char *location)
+{
+	if (should_i_fail("malloc")) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	return talloc_named_const(allocations, size, location);
+}
+
+static void ctdb_test_free(void *ptr, const char *location)
+{
+	talloc_free(ptr);
+}
+
+static void *ctdb_test_realloc(void *ptr, size_t size, const char *location)
+{
+	if (should_i_fail("realloc")) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	ptr = _talloc_realloc(allocations, ptr, size, location);
+	if (ptr)
+		talloc_set_name(ptr, "%s (reallocated to %u at %s)",
+				talloc_get_name(ptr), size, location);
+	return ptr;
+}
+
+static ssize_t ctdb_test_read(int fd, void *buf, size_t count,
+			      const char *location)
+{
+	if (should_i_fail("read")) {
+		errno = EBADF;
+		return -1;
+	}
+	/* FIXME: We only let parent read and write.
+	 * We should have child do short read, at least until whole packet is
+	 * read.  Then we terminate child. */
+	if (!am_parent()) {
+		log_line(LOG_DEBUG, "Child reading fd");
+		return 0;
+	}
+	return read(fd, buf, count);
+}
+
+static ssize_t ctdb_test_write(int fd, const void *buf, size_t count,
+			       const char *location)
+{
+	if (should_i_fail("write")) {
+		errno = EBADF;
+		return -1;
+	}
+	/* FIXME: We only let parent read and write.
+	 * We should have child do short write, at least until whole packet is
+	 * written, then terminate child.  Check that all children and parent
+	 * write the same data. */
+	if (!am_parent()) {
+		log_line(LOG_DEBUG, "Child writing fd");
+		return 0;
+	}
+	return write(fd, buf, count);
+}
+
+static int ctdb_test_socket(int domain, int type, int protocol,
+			    const char *location)
+{
+	if (should_i_fail("socket")) {
+		errno = EINVAL;
+		return -1;
+	}
+	return socket(domain, type, protocol);
+}
+
+static int ctdb_test_connect(int sockfd, const struct sockaddr *addr,
+			     socklen_t addrlen, const char *location)
+{
+	if (should_i_fail("connect")) {
+		errno = EINVAL;
+		return -1;
+	}
+	return connect(sockfd, addr, addrlen);
+}
+
+static struct tdb_context *ctdb_test_tdb_open_ex(const char *name,
+						 int hash_size, int tdb_flags,
+						 int open_flags, mode_t mode,
+						 const struct tdb_logging_context *log_ctx,
+						 tdb_hash_func hash_fn,
+						 const char *location)
+{
+	if (should_i_fail("tdb_open_ex")) {
+		errno = ENOENT;
+		return NULL;
+	}
+	return tdb_open_ex(name, hash_size, tdb_flags, open_flags, mode,
+			   log_ctx, hash_fn);
+}
+
+/* We don't need to fail this, but as library expects to be able to free()
+   dptr, we need to make sure it's talloced (see ctdb_test_free) */
+static TDB_DATA ctdb_test_tdb_fetch(struct tdb_context *tdb, TDB_DATA key)
+{
+	TDB_DATA ret = tdb_fetch(tdb, key);
+	if (ret.dptr) {
+		ret.dptr = talloc_memdup(allocations, ret.dptr, ret.dsize);
+		if (!ret.dptr) {
+			err(1, "Could not memdup %zu bytes", ret.dsize);
+		}
+	}
+	return ret;
+}
+
+void check_allocations(void)
+{
+	talloc_free(working);
+
+	if (talloc_total_blocks(allocations) != 1) {
+		log_line(LOG_ALWAYS, "Resource leak:");
+		talloc_report_full(allocations, stdout);
+		exit(1);
+	}
+}
+
+/* This version adds one byte (for nul term) */
+void *grab_fd(int fd, size_t *size)
+{
+	size_t max = 16384;
+	int ret;
+	void *buffer = talloc_array(NULL, char, max+1);
+
+	*size = 0;
+	while ((ret = read(fd, buffer + *size, max - *size)) > 0) {
+		*size += ret;
+		if (*size == max)
+			buffer = talloc_realloc(NULL, buffer, char, max *= 2 + 1);
+	}
+	if (ret < 0) {
+		talloc_free(buffer);
+		buffer = NULL;
+	}
+	return buffer;
+}
+
+int main(int argc, char *argv[])
+{
+	int input_fd, c;
+	const char *optstr;
+	struct option *options;
+
+	allocations = talloc_named_const(NULL, 1, "ctdb-test");
+	working = talloc_named_const(NULL, 1, "ctdb-test-working");
+
+	options = get_cmdline_options();
+	optstr = get_cmdline_optstr();
+
+	while ((c = getopt_long(argc, argv, optstr, options, NULL)) != EOF) {
+		struct cmdline_option *copt = get_cmdline_option(c);
+		if (!copt)
+			errx(1, "Unknown argument");
+
+		copt->parse(&copt->opt);
+	}
+
+	if (optind == argc) {
+		log_line(LOG_DEBUG, "Disabling failtest due to stdin.");
+		failtest = false;
+		input_fd = STDIN_FILENO;
+	} else if (optind + 1 != argc)
+		errx(1, "Need a single argument: input filename");
+	else {
+		input_fd = open(argv[optind], O_RDONLY);
+		if (input_fd < 0)
+			err(1, "Opening %s", argv[optind]);
+		tui_abort_on_fail = true;
+	}
+
+	run_inits();
+	if (!tui_quiet)
+		print_license();
+
+	log_line(LOG_VERBOSE, "initialisation done");
+
+	tui_run(input_fd);
+
+	/* Everyone loves a good error haiku! */
+	if (expects_remaining())
+		errx(1, "Expectations still / "
+		     "unfulfilled remaining. / "
+		     "Testing blossoms fail.");
+	check_allocations();
+	dump_failinfo();
+
+	return EXIT_SUCCESS;
+}
diff --git a/ctdb/libctdb/test/ctdb-test.h b/ctdb/libctdb/test/ctdb-test.h
new file mode 100644
index 0000000000..68ab2ea8f3
--- /dev/null
+++ b/ctdb/libctdb/test/ctdb-test.h
@@ -0,0 +1,19 @@
+#ifndef __HAVE_CTDB_TEST_H
+#define __HAVE_CTDB_TEST_H
+#include <stdlib.h>
+
+/* We hang all libctdb allocations off this talloc tree. */
+extern void *allocations;
+
+void check_allocations(void);
+
+/* Our own working state gets hung off this tree. */
+extern void *working;
+
+/* The ctdb connection; created by 'connect' command. */
+struct ctdb_connection *get_ctdb(void);
+
+/* Talloc bytes from an fd until EOF.  Nul terminate. */
+void *grab_fd(int fd, size_t *size);
+
+#endif /* __HAVE_CTDB_TEST_H */
diff --git a/ctdb/libctdb/test/expect.c b/ctdb/libctdb/test/expect.c
new file mode 100644
index 0000000000..8e23a6d90b
--- /dev/null
+++ b/ctdb/libctdb/test/expect.c
@@ -0,0 +1,300 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Jeremy Kerr & Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "tui.h"
+#include "log.h"
+#include "expect.h"
+#include <fnmatch.h>
+#include <talloc.h>
+#include "utils.h"
+
+/* Expect is used to set up expectations on logging, for automated
+ * testing. */
+struct expect {
+	struct expect *next;
+	int invert;
+	int matched;
+	char *command;
+	char *pattern;
+};
+
+static struct expect *expect;
+struct cmdstack
+{
+	struct cmdstack *prev;
+	char *command;
+};
+static struct cmdstack *current_command;
+
+/* We don't need to try to match every pattern length: we only need
+ * lengths where the next char matches. */
+static unsigned int maybe_skip(char next_char, const char *line)
+{
+	char str[2];
+
+	/* If next one is *, we can't skip any. */
+	if (next_char == '*')
+		return 1;
+
+	/* No more string? */
+	if (*line == '\0')
+		return 1;
+
+	/* Next one is space, skip up to next space. */
+	if (next_char == '\t' || next_char == ' ')
+		return strcspn(line+1, "\t ") + 1;
+
+	str[0] = next_char;
+	str[1] = '\0';
+	return strcspn(line+1, str) + 1;
+}
+
+/* Loose match for strings: whitespace can be any number of
+ * whitespace, and * matches anything.  Approximately. */
+static bool loose_match(const char *pattern, const char *line)
+{
+	/* Any whitespace in pattern matches any whitespace in line. */
+	if (*pattern == '\t' || *pattern == ' ') {
+		int i, j, pat_space, line_space;
+
+		pat_space = strspn(pattern, "\t ");
+		line_space = strspn(line, "\t ");
+
+		for (i = 1; i <= pat_space; i++)
+			for (j = 1; j <= line_space; j++)
+				if (loose_match(pattern+i, line+j))
+					return true;
+		return false;
+	}
+
+	if (*pattern == '*') {
+		int i, len = strlen(line);
+		for (i = 0; i <= len; i += maybe_skip(pattern[1],line+i))
+			if (loose_match(pattern+1, line+i))
+				return true;
+
+		return false;
+	}
+
+	if (*pattern == *line) {
+		if (*pattern == '\0')
+			return true;
+		return loose_match(pattern+1, line+1);
+	}
+
+	return false;
+}
+
+/* Pattern can't have whitespace at start and end, due to our parser.
+ * Strip ot here. */
+static bool matches(const char *pattern, const char *line)
+{
+	unsigned int len;
+
+	line += strspn(line, "\t ");
+	len = strlen(line);
+	if (len > 0 && (line[len-1] == '\t' || line[len-1] == ' ')) {
+		char trimmed[len];
+		memcpy(trimmed, line, len);
+		while (trimmed[len-1] == '\t' || trimmed[len-1] == ' ') {
+			if (len == 1)
+				break;
+			len--;
+		}
+		trimmed[len] = '\0';
+		return loose_match(pattern, trimmed);
+	}
+	return loose_match(pattern, line);
+}
+
+bool expect_log_hook(const char *line)
+{
+	struct expect *e;
+	bool ret = false;
+
+	if (current_command == NULL)
+		return false;
+
+	/* Only allow each pattern to match once, so we can easily
+	 * expect something to happen twice. */
+	for (e = expect; e; e = e->next) {
+		if (!e->matched
+		    && streq(current_command->command, e->command)
+		    && matches(e->pattern, line)) {
+			e->matched = 1;
+			ret = true;
+		}
+	}
+	return ret;
+}
+
+bool expects_remaining(void)
+{
+	return expect != NULL;
+}
+
+static void expect_pre_command(const char *command)
+{
+	struct cmdstack *new = talloc(NULL, struct cmdstack);
+	new->prev = current_command;
+	new->command = talloc_strdup(new, command);
+	current_command = new;
+}
+
+static bool expect_post_command(const char *command)
+{
+	struct expect **e, **next, *old;
+	bool ret = true;
+	struct cmdstack *oldcmd;
+
+	for (e = &expect; *e; e = next) {
+		next = &(*e)->next;
+
+		if (!streq(current_command->command, (*e)->command))
+			continue;
+
+		if (!(*e)->invert && !(*e)->matched) {
+			if (tui_abort_on_fail)
+				script_fail("Pattern '%s' did not match",
+					    (*e)->pattern);
+			log_line(LOG_VERBOSE, "Pattern '%s' did not match",
+				 (*e)->pattern);
+			ret = false;
+		} else if ((*e)->invert && (*e)->matched) {
+			if (tui_abort_on_fail)
+				script_fail("Pattern '%s' matched",
+					    (*e)->pattern);
+			log_line(LOG_VERBOSE, "Pattern '%s' matched",
+				 (*e)->pattern);
+			ret = false;
+		}
+
+		/* Unlink from list and free. */
+		old = *e;
+		*e = (*e)->next;
+		next = e;
+
+		talloc_free(old);
+	}
+
+	oldcmd = current_command;
+	current_command = current_command->prev;
+	talloc_free(oldcmd);
+	return ret;
+}
+
+static bool expect_cmd(int argc, char **argv)
+{
+	struct expect *e;
+	unsigned int i, len;
+	bool invert = false;
+
+	if (argc == 1) {
+		for (e = expect; e; e = e->next)
+			log_line(LOG_UI, "%s: %s\"%s\"",
+				 e->command,
+				 e->invert ? "! " : "",
+				 e->pattern);
+		return true;
+	}
+
+	if (argv[1] && streq(argv[1], "!")) {
+		invert = true;
+		argv++;
+		argc--;
+	}
+
+	if (argc < 3)
+		return false;
+
+	if (!tui_is_command(argv[1])) {
+		log_line(LOG_ALWAYS, "expect: %s is not a command\n",
+			 argv[1]);
+		return false;
+	}
+
+	e = talloc(NULL, struct expect);
+	e->matched = 0;
+	e->invert = invert;
+	e->next = expect;
+
+	e->command = talloc_strdup(e, argv[1]);
+
+	for (len = 0, i = 2; i < argc; i++)
+		len += strlen(argv[i])+1;
+	e->pattern = talloc_array(e, char, len + 1);
+
+	e->pattern[0] = '\0';
+	for (i = 2; i < argc; i++) {
+		if (i == 2)
+			sprintf(e->pattern+strlen(e->pattern), "%s", argv[i]);
+		else
+			sprintf(e->pattern+strlen(e->pattern), " %s", argv[i]);
+	}
+	expect = e;
+	return true;
+}
+
+static void expect_help(int argc, char **argv)
+{
+#include "generated-expect-help:expect"
+/*** XML Help:
+    <section id="c:expect">
+     <title><command>expect</command></title>
+     <para>Catch logging information for automated testing</para>
+     <cmdsynopsis>
+      <command>expect</command>
+     </cmdsynopsis>
+     <cmdsynopsis>
+      <command>expect</command>
+      <arg choice="opt">!</arg>
+      <arg choice="req"><replaceable>command</replaceable></arg>
+      <arg choice="req"><replaceable>pattern</replaceable></arg>
+     </cmdsynopsis>
+     <para><command>expect</command> will set up a set of patterns to expect in
+      logging messages for a particular command. If that command finishes
+      without matching the specified pattern, the simulator will exit with a
+      non-zero return value.  After the command is run, all expectations on that
+      command are deleted.</para>
+     <para><command>expect</command> with no arguments will print out the
+      current list of expectations, as a command and a pattern.</para>
+     <para><command>expect <replaceable>command pattern</replaceable></command>
+      will expect the specified pattern to occur the next time
+      <replaceable>command</replaceable> is invoked. If an '!' appears before
+      the command, then the expectation is negated - if the pattern appears in
+      the output, then the simulator will exit with an error
+     </para>
+     <para>The pattern itself is similar to a simple shell wildcard,
+      except whitespace is loosely matched.  The * character will
+      match any a string of any length.</para>
+     </section>
+*/
+}
+
+static void init(void)
+{
+	tui_register_command("expect", expect_cmd, expect_help);
+	tui_register_pre_post_hook(expect_pre_command, expect_post_command);
+}
+
+init_call(init);
diff --git a/ctdb/libctdb/test/expect.h b/ctdb/libctdb/test/expect.h
new file mode 100644
index 0000000000..d42ebdcc9c
--- /dev/null
+++ b/ctdb/libctdb/test/expect.h
@@ -0,0 +1,35 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Jeremy Kerr & Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef __HAVE_EXPECT_H
+#define __HAVE_EXPECT_H
+
+/* Expect interface */
+void expect_before_command(const char *command);
+bool expect_log_hook(const char *line);
+void expect_after_command(void);
+
+/* Are there any expect commands unresolved? */
+bool expects_remaining(void);
+
+#endif /* __HAVE_EXPECT_H */
diff --git a/ctdb/libctdb/test/failtest.c b/ctdb/libctdb/test/failtest.c
new file mode 100644
index 0000000000..2560e3e5eb
--- /dev/null
+++ b/ctdb/libctdb/test/failtest.c
@@ -0,0 +1,298 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2004 Jeremy Kerr & Rusty Russell
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+#include "utils.h"
+#include "tui.h"
+#include "log.h"
+#include "failtest.h"
+#include "talloc.h"
+#include "dlinklist.h"
+#include <stdlib.h>
+#include <stdbool.h>
+#include <getopt.h>
+#include <err.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <errno.h>
+
+static unsigned int fails = 0, excessive_fails = 2;
+unsigned int failpoints = 0;
+static const char *failtest_no_report = NULL;
+bool failtest = true;
+
+struct fail_decision {
+	struct fail_decision *next, *prev;
+	char *location;
+	unsigned int tui_line;
+	bool failed;
+};
+
+static struct fail_decision *decisions;
+
+/* Failure path to follow (initialized by --failpath). */
+static const char *failpath = NULL, *orig_failpath;
+
+/*** XML Argument:
+    <section id="a:failpath">
+     <title><option>--failpath
+      <replaceable>path</replaceable></option></title>
+     <subtitle>Replay a failure path</subtitle>
+     <para>Given a failure path, (from <option>--failtest</option>), this will
+      replay the sequence of sucesses/failures, allowing debugging.  The input
+      should be the same as the original which caused the failure.
+      </para>
+
+     <para>This testing can be slow, but allows for testing of failure
+      paths which would otherwise be very difficult to test
+     automatically.</para>
+    </section>
+*/
+static void cmdline_failpath(struct option *opt)
+{
+	extern char *optarg;
+	if (!optarg)
+		errx(1, "failtest option requires an argument");
+	orig_failpath = failpath = optarg;
+}
+cmdline_opt("failpath", 1, 0, cmdline_failpath);
+
+/*** XML Argument:
+    <section id="a:failtest-no-report">
+     <title><option>--failtest-no-report
+      <replaceable>function</replaceable></option></title>
+     <subtitle>Exclude a function from excessive failure reports</subtitle>
+
+     <para>Sometimes code deliberately ignores the failures of a
+     certain function.  This suppresses complaints about that for any
+     functions containing this name.</para> </section>
+*/
+static void cmdline_failtest_no_report(struct option *opt)
+{
+	extern char *optarg;
+	if (!optarg)
+		errx(1, "failtest-no-report option requires an argument");
+	failtest_no_report = optarg;
+}
+cmdline_opt("failtest-no-report", 1, 0, cmdline_failtest_no_report);
+
+/* Separate function to make .gdbinit easier */
+static bool failpath_fail(void)
+{
+	return true;
+}
+
+static bool do_failpath(const char *func)
+{
+	if (*failpath == '[') {
+		failpath++;
+		if (strncmp(failpath, func, strlen(func)) != 0
+		    || failpath[strlen(func)] != ']')
+			errx(1, "Failpath expected %.*s not %s\n",
+			     strcspn(failpath, "]"), failpath, func);
+		failpath += strlen(func) + 1;
+	}
+
+	if (*failpath == ':') {
+		unsigned long line;
+		char *after;
+		failpath++;
+		line = strtoul(failpath, &after, 10);
+		if (*after != ':')
+			errx(1, "Bad failure path line number %s\n",
+			     failpath);
+		if (line != tui_linenum)
+			errx(1, "Unexpected line number %lu vs %u\n",
+			     line, tui_linenum);
+		failpath = after+1;
+	}
+
+	switch ((failpath++)[0]) {
+	case 'F':
+	case 'f':
+		return failpath_fail();
+	case 'S':
+	case 's':
+		return false;
+	case 0:
+		failpath = NULL;
+		return false;
+	default:
+		errx(1, "Failpath '%c' failed to path",
+		     failpath[-1]);
+	}
+}
+
+static char *failpath_string_for_line(struct fail_decision *dec)
+{
+	char *ret = NULL;
+	struct fail_decision *i;
+
+	for (i = decisions; i; i = i->next) {
+		if (i->tui_line != dec->tui_line)
+			continue;
+		ret = talloc_asprintf_append(ret, "[%s]%c",
+					     i->location,
+					     i->failed ? 'F' : 'S');
+	}
+	return ret;
+}
+
+static char *failpath_string(void)
+{
+	char *ret = NULL;
+	struct fail_decision *i;
+
+	for (i = decisions; i; i = i->next)
+		ret = talloc_asprintf_append(ret, "[%s]:%i:%c",
+					     i->location, i->tui_line,
+					     i->failed ? 'F' : 'S');
+	return ret;
+}
+
+static void warn_failure(void)
+{
+	char *warning;
+	struct fail_decision *i;
+	int last_warning = 0;
+
+	log_line(LOG_ALWAYS, "WARNING: test ignores failures at %s",
+		 failpath_string());
+
+	for (i = decisions; i; i = i->next) {
+		if (i->failed && i->tui_line > last_warning) {
+			warning = failpath_string_for_line(i);
+			log_line(LOG_ALWAYS, " Line %i: %s",
+				 i->tui_line, warning);
+			talloc_free(warning);
+			last_warning = i->tui_line;
+		}
+	}
+}
+
+bool am_parent(void)
+{
+	struct fail_decision *i;
+
+	for (i = decisions; i; i = i->next) {
+		if (i->failed)
+			return false;
+	}
+	return true;
+}
+
+/* Should I fail at this point?  Once only: it would be too expensive
+ * to fail at every possible call. */
+bool should_i_fail_once(const char *location)
+{
+	char *p;
+	struct fail_decision *i;
+
+	if (failpath) {
+		p = strstr(orig_failpath ?: "", location);
+		if (p && p <= failpath
+		    && p[-1] == '[' && p[strlen(location)] == ']')
+			return false;
+
+		return do_failpath(location);
+	}
+
+	for (i = decisions; i; i = i->next)
+		if (streq(location, i->location))
+			return false;
+
+	if (should_i_fail(location)) {
+		excessive_fails++;
+		return true;
+	}
+	return false;
+}
+
+/* Should I fail at this point? */
+bool should_i_fail(const char *func)
+{
+	pid_t child;
+	int status;
+	struct fail_decision *dec;
+
+	if (failpath)
+		return do_failpath(func);
+
+	failpoints++;
+	if (!failtest)
+		return false;
+
+	/* If a testcase ignores a spuriously-inserted failure, it's
+	 * not specific enough, and we risk doing 2^n tests! */
+	if (fails > excessive_fails) {
+		static bool warned = false;
+		if (!warned++)
+			warn_failure();
+	}
+
+	dec = talloc(NULL, struct fail_decision);
+	dec->location = talloc_strdup(dec, func);
+	dec->tui_line = tui_linenum;
+
+	DLIST_ADD_END(decisions, dec, struct fail_decision);
+
+	fflush(stdout);
+	child = fork();
+	if (child == -1)
+		err(1, "fork failed for failtest!");
+
+	/* The child actually fails.  The script will screw up at this
+	 * point, but should not crash. */
+	if (child == 0) {
+		dec->failed = true;
+		if (!failtest_no_report || !strstr(func, failtest_no_report))
+			fails++;
+		return true;
+	}
+
+	dec->failed = false;
+	while (waitpid(child, &status, 0) < 0) {
+		if (errno != EINTR)
+			err(1, "failtest waitpid failed for child %i",
+			    (int)child);
+	}
+
+	/* If child succeeded, or mere script failure, continue. */
+	if (WIFEXITED(status) && (WEXITSTATUS(status) == EXIT_SUCCESS
+				  || WEXITSTATUS(status) == EXIT_SCRIPTFAIL))
+		return false;
+
+	/* Report unless child already reported it. */
+	if (!WIFEXITED(status) || WEXITSTATUS(status) != EXIT_SILENT) {
+		/* Reproduce child's path */
+		dec->failed = true;
+		log_line(LOG_ALWAYS, "Child %s %i on failure path: %s",
+			 WIFEXITED(status) ? "exited" : "signalled",
+			 WIFEXITED(status) ? WEXITSTATUS(status)
+			 : WTERMSIG(status), failpath_string());
+	}
+	exit(EXIT_SILENT);
+}
+
+void dump_failinfo(void)
+{
+	log_line(LOG_VERBOSE, "Hit %u failpoints: %s",
+		 failpoints, failpath_string());
+}
diff --git a/ctdb/libctdb/test/failtest.h b/ctdb/libctdb/test/failtest.h
new file mode 100644
index 0000000000..c361f84c9b
--- /dev/null
+++ b/ctdb/libctdb/test/failtest.h
@@ -0,0 +1,15 @@
+#ifndef FAILTEST_H
+#define FAILTEST_H
+#include <stdbool.h>
+
+bool should_i_fail_once(const char *location);
+bool should_i_fail(const char *func);
+
+bool failtest;
+
+/* Parent never has artificial failures. */
+bool am_parent(void);
+
+void dump_failinfo(void);
+
+#endif /* FAILTEST_H */
diff --git a/ctdb/libctdb/test/log.c b/ctdb/libctdb/test/log.c
new file mode 100644
index 0000000000..a4a9be62b8
--- /dev/null
+++ b/ctdb/libctdb/test/log.c
@@ -0,0 +1,231 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Jeremy Kerr & Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+#include "log.h"
+#include "tui.h"
+#include "utils.h"
+#include "expect.h"
+#include <string.h>
+#include <talloc.h>
+
+static struct {
+	enum log_type	type;
+	char *		name;
+} log_names[] = {
+	{ LOG_WRITE,	"write" },
+	{ LOG_READ,	"read" },
+	{ LOG_LIB,	"lib" },
+	{ LOG_VERBOSE,	"verbose" },
+};
+
+static FILE *logstream;
+static int typemask = ~LOG_VERBOSE;
+
+bool log_line(enum log_type type, const char *format, ...)
+{
+	va_list ap;
+	char *line;
+	bool ret;
+
+	va_start(ap, format);
+	line = talloc_vasprintf(NULL, format, ap);
+	va_end(ap);
+
+	if (!type || (type & typemask))
+		fprintf(logstream ?: stderr, "%s\n", line);
+
+	ret = expect_log_hook(line);
+	talloc_free(line);
+	return ret;
+}
+
+static void log_partial_v(enum log_type type,
+			  char *buf,
+			  unsigned bufsize,
+			  const char *format,
+			  va_list ap)
+{
+	char *ptr;
+	int len = strlen(buf);
+
+	/* write to the end of buffer */
+	if (vsnprintf(buf + len, bufsize - len - 1, format, ap)
+			> bufsize - len - 1)
+		log_line(LOG_ALWAYS, "log_line_partial buffer is full!");
+
+	ptr = buf;
+
+	/* print each bit that ends in a newline */
+	for (len = strcspn(ptr, "\n"); *(ptr + len);
+			ptr += len, len = strcspn(ptr, "\n")) {
+		log_line(type, "%.*s", len++, ptr);
+	}
+
+	/* if we've printed, copy any remaining (non-newlined)
+	   parts (including the \0) to the front of buf */
+	memmove(buf, ptr, strlen(ptr) + 1);
+}
+
+void log_partial(enum log_type type, char *buf, unsigned bufsize,
+		 const char *format, ...)
+{
+	va_list ap;
+
+	va_start(ap, format);
+	log_partial_v(type, buf, bufsize, format, ap);
+	va_end(ap);
+}
+
+static inline int parsetype(const char *type)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(log_names); i++)
+		if (streq(log_names[i].name, type))
+			return log_names[i].type;
+
+	return 0;
+}
+
+static bool log_admin(int argc, char **argv)
+{
+	int i;
+	int newtypemask = 0;
+
+	if (argc == 1) {
+		log_line(LOG_UI, "current log types:", typemask);
+
+		for (i = 0; i < ARRAY_SIZE(log_names); i++) {
+			if (typemask & log_names[i].type)
+				log_line(LOG_UI, "\t%s", log_names[i].name);
+		}
+		return true;
+	}
+
+	if (argc == 2) {
+		log_line(LOG_ALWAYS, "Expected =, + or - then args");
+		return false;
+	}
+
+	for (i = 2; i < argc; i++) {
+		int type;
+
+		if (!(type = parsetype(argv[i]))) {
+			log_line(LOG_ALWAYS, "no such type %s", argv[i]);
+			return false;
+		}
+		newtypemask |= type;
+	}
+
+	switch (*argv[1]) {
+	case '=':
+		typemask = newtypemask;
+		break;
+	case '-':
+		typemask &= ~newtypemask;
+		break;
+	case '+':
+		typemask |= newtypemask;
+		break;
+	default:
+		log_line(LOG_ALWAYS, "unknown modifer: %c", *argv[1]);
+		return false;
+	}
+
+	return true;
+}
+
+static void log_admin_help(int agc, char **argv)
+{
+#include "generated-log-help:log"
+/*** XML Help:
+    <section id="c:log">
+     <title><command>log</command></title>
+     <para>Manage logging settings</para>
+     <cmdsynopsis>
+      <command>log</command>
+      <group choice="opt">
+       <arg choice="plain">=</arg>
+       <arg choice="plain">+</arg>
+       <arg choice="plain">-</arg>
+      </group>
+      <arg choice="req"><replaceable>type, ...</replaceable></arg>
+     </cmdsynopsis>
+     <para>Each log message is classified into one of the following
+     types:</para>
+      <varlistentry>
+       <term>UI</term>
+       <listitem>
+        <para>Normal response from command lines.</para>
+       </listitem>
+      </varlistentry>
+      <varlistentry>
+       <term>LIB</term>
+       <listitem>
+        <para>Logging output from libctdb</para>
+       </listitem>
+      </varlistentry>
+     <variablelist>
+      <varlistentry>
+       <term>READ</term>
+       <listitem>
+        <para>Messages from ctdbd</para>
+       </listitem>
+      </varlistentry>
+      <varlistentry>
+       <term>WRITE</term>
+       <listitem>
+        <para>Messages to ctdbd</para>
+       </listitem>
+      </varlistentry>
+      <varlistentry>
+       <term>VERBOSE</term>
+       <listitem>
+        <para>Verbose debug output</para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+
+     <para>The <command>log</command> command allows you to select
+      which messages are displayed. By default, all messages except
+      debug will be shown.</para>
+
+     <para>Without any arguments, the current logged types are listed.</para>
+
+     <para>With +, - or = character, those types will be added,
+      removed or set as the current types of messages to be logged
+      (repectively).</para>
+
+      <para>Messages generated as a result of user input are always
+      logged.  </para> </section>
+*/
+}
+
+static void log_init(void)
+{
+	logstream = stdout;
+	if (tui_quiet)
+		typemask = 0;
+	tui_register_command("log", log_admin, log_admin_help);
+}
+
+init_call(log_init);
diff --git a/ctdb/libctdb/test/log.h b/ctdb/libctdb/test/log.h
new file mode 100644
index 0000000000..d19d6b2f5c
--- /dev/null
+++ b/ctdb/libctdb/test/log.h
@@ -0,0 +1,48 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Jeremy Kerr & Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef __HAVE_LOG_H
+#define __HAVE_LOG_H
+#include <stdbool.h>
+
+enum log_type {
+	LOG_ALWAYS	= 0x00,
+	/* Packets that are written to ctdbd. */
+	LOG_WRITE	= 0x02,
+	/* Packets that are read from ctdbd. */
+	LOG_READ	= 0x04,
+	/* Logging from libctdb. */
+	LOG_LIB		= 0x08,
+	/* Logging from normal operations. */
+	LOG_UI		= 0x10,
+	/* Verbose debugging. */
+	LOG_VERBOSE	= 0x20,
+};
+
+/* Adds a \n for convenient logging.  Returns true if it was expected. */
+bool log_line(enum log_type type, const char *format, ...);
+/* Builds up buffer and prints out line at a time. */
+void log_partial(enum log_type type, char *buf, unsigned bufsize,
+		 const char *format, ...);
+
+#endif /* __HAVE_LOG_H */
diff --git a/ctdb/libctdb/test/tests/attachdb1.txt b/ctdb/libctdb/test/tests/attachdb1.txt
new file mode 100644
index 0000000000..d258376258
--- /dev/null
+++ b/ctdb/libctdb/test/tests/attachdb1.txt
@@ -0,0 +1,8 @@
+connect
+# This is just a sanity check
+expect attachdb attached: 1
+attachdb test.tdb false
+expect attachdb attached: 2
+attachdb test2.tdb false
+detachdb 2
+detachdb 1
diff --git a/ctdb/libctdb/test/tests/connect1.txt b/ctdb/libctdb/test/tests/connect1.txt
new file mode 100644
index 0000000000..d2bfeca45c
--- /dev/null
+++ b/ctdb/libctdb/test/tests/connect1.txt
@@ -0,0 +1,3 @@
+# Simple connect, disconnect
+connect
+disconnect
diff --git a/ctdb/libctdb/test/tests/connect2.txt b/ctdb/libctdb/test/tests/connect2.txt
new file mode 100644
index 0000000000..d165a2cd85
--- /dev/null
+++ b/ctdb/libctdb/test/tests/connect2.txt
@@ -0,0 +1,3 @@
+# Simple connect with explicit path, then disconnect
+connect /tmp/ctdb.socket
+disconnect
diff --git a/ctdb/libctdb/test/tools/create-links b/ctdb/libctdb/test/tools/create-links
new file mode 100755
index 0000000000..265efc17ee
--- /dev/null
+++ b/ctdb/libctdb/test/tools/create-links
@@ -0,0 +1,39 @@
+#!/bin/sh
+# Create docbook-xml links
+
+# most things will be under this path.
+DOCB=/usr/share/sgml/docbook
+
+# potential location for xhtml/docbook.xsl
+XSLDIRS="$DOCB/xsl-stylesheets* $DOCB/stylesheet/xsl/nwalsh"
+
+# potential location for docbookx.dtd
+DTDDIRS="$DOCB/xml-dtd* $DOCB/dtd/xml/*"
+
+# look for a file (arg 1) in a set of dirs (arg 2). If it exists, create a link
+# (arg 3), in the current directory to the dir.
+condlink() {
+	file=$1
+	dirs=$2
+	link=$3
+
+	for d in $dirs
+	do
+		if [ -f $d/$file ]
+		then
+			dir=$d
+			break
+		fi
+	done
+
+	if [ -z "$dir" ]
+	then
+		echo Docbook support not found.  See README.  Faking it. >&2
+		exit 1
+	else
+		ln -sfn "$dir" "$link"
+	fi
+}
+
+condlink "xhtml/docbook.xsl" "$XSLDIRS" "link-xhtml"
+condlink "docbookx.dtd" "$DTDDIRS" "link-dtd"
diff --git a/ctdb/libctdb/test/tools/extract-help b/ctdb/libctdb/test/tools/extract-help
new file mode 100755
index 0000000000..4468956977
--- /dev/null
+++ b/ctdb/libctdb/test/tools/extract-help
@@ -0,0 +1,10 @@
+#! /bin/sh
+# Extract XML help from .c file.
+
+FILE=$1
+LINE=`expr $2 + 1`
+
+END=`tail -n +$LINE $1 | fgrep -n '*/' | cut -d: -f1 | head -n +1`
+END=`expr $END - 1`
+
+tail -n +$LINE $1 | head -n $END
diff --git a/ctdb/libctdb/test/tools/gen-help b/ctdb/libctdb/test/tools/gen-help
new file mode 100755
index 0000000000..b8839ae1a0
--- /dev/null
+++ b/ctdb/libctdb/test/tools/gen-help
@@ -0,0 +1,52 @@
+#! /bin/bash
+
+# We could have multiple occurances.  Create all of them.
+FILE=$1
+
+TMPF=`mktemp /tmp/gen-help.XXXXXX`
+trap "rm -f $TMPF*" EXIT
+cmdsed='s,.*<command>[ 	]*\([^ 	]*\)[ 	]*</command>.*,\1,p'
+
+STARTLINE=1
+for LINE in `fgrep -n '/*** XML Help:' < $FILE | cut -d: -f1`; do
+    if [ -L tools/link-dtd ]; then
+	echo '<?xml version="1.0"?>' > $TMPF
+	echo '<!DOCTYPE article PUBLIC "-//OASIS//DTD Docbook XML V4.1.2//EN"' \
+	    >> $TMPF
+	echo '"'`pwd`'/tools/link-dtd/docbookx.dtd">' >> $TMPF
+	echo '<article><section>' >> $TMPF
+	tools/extract-help $FILE $LINE >> $TMPF
+	echo '</section></article>' >> $TMPF
+
+	tr '\n' ' ' < $TMPF | sed -e 's/[[:space:]]\{2,\}/ /g' |
+        xsltproc tools/text.xsl - | fold -w80 -s > $TMPF.txt
+
+	COMMAND=`sed -n "$cmdsed" < $TMPF | head -n +1`
+	COMMAND_FILE=generated-`basename $FILE .c`-help:$COMMAND
+	#echo Creating $COMMAND_FILE
+
+        # Output description, in quotes.
+	echo 'log_line(LOG_ALWAYS,' > $COMMAND_FILE
+
+	TXTSTART=`grep -n '^  1\.1\.' $TMPF.txt | cut -d: -f1`
+	tail -n +`expr $TXTSTART + 2` $TMPF.txt | while read -r TXTLINE; do
+	    echo "$TXTLINE" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' \
+		-e 's/$/\\n"/' >> $COMMAND_FILE
+	done
+	echo ');' >> $COMMAND_FILE
+    else
+	tools/extract-help $FILE $LINE > $TMPF
+
+	COMMAND=`sed -n "$cmdsed" < $TMPF | head -n +1`
+	COMMAND_FILE=generated-`basename $FILE .c`-help:$COMMAND
+	echo Faking up $COMMAND_FILE
+
+	echo 'log_line(LOG_ALWAYS,' > $COMMAND_FILE
+	sed 's/<arg [^>]*>/   /;s/<[^>]*>//g' < $TMPF |
+	    sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' -e 's/$/\\n"/' \
+		>> $COMMAND_FILE
+	echo ');' >> $COMMAND_FILE
+    fi
+
+    STARTLINE=$LINE
+done
diff --git a/ctdb/libctdb/test/tools/gen-usage b/ctdb/libctdb/test/tools/gen-usage
new file mode 100755
index 0000000000..6fad0e1e4b
--- /dev/null
+++ b/ctdb/libctdb/test/tools/gen-usage
@@ -0,0 +1,56 @@
+#! /bin/sh
+
+# read a list of C files with inline XML usage given on the command line, and
+# write a usage function on stdout
+
+tmpf=`mktemp /tmp/gen-help.XXXXXX`
+trap "rm -f $tmpf*" EXIT
+quote() {
+    sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/^/"/' -e 's/$/\\n"/'
+}
+
+cat <<EOF
+#include <stdio.h>
+#include "utils.h"
+
+void print_usage(void)
+{
+	fprintf(stderr,
+"Usage: ctdb-test [options]\n"
+"Options available:\n"
+"\n"
+EOF
+
+for file in "$@"
+do
+    for line in `fgrep -n '/*** XML Argument:' < $file | cut -d: -f1`;
+    do
+	if [ -L tools/link-dtd ]; then
+
+	    cat > $tmpf <<EOF
+<?xml version="1.0"?>
+<!DOCTYPE article PUBLIC "-//OASIS//DTD Docbook XML V4.1.2//EN"
+"`pwd`/tools/link-dtd/docbookx.dtd">
+<article>
+EOF
+	    tools/extract-help $file $line >> $tmpf
+	    echo '</article>' >> $tmpf
+
+	    tr '\n' ' ' < $tmpf | sed -e 's/[[:space:]]\{2,\}/ /g' |
+		xsltproc tools/usage.xsl - | fold -w80 -s > $tmpf.txt
+
+	    quote < $tmpf.txt
+	else
+	    # if we don't have docbook, just strip out the tags and grab
+	    # the first two lines
+	    tools/extract-help $file $line > $tmpf
+	    sed 's/<arg [^>]*>/   /;s/<[^>]*>//g;' < $tmpf | head -3 | quote
+	fi
+
+    done
+done
+
+cat <<EOF
+);
+}
+EOF
diff --git a/ctdb/libctdb/test/tools/text.xsl b/ctdb/libctdb/test/tools/text.xsl
new file mode 100644
index 0000000000..dcbc64e89c
--- /dev/null
+++ b/ctdb/libctdb/test/tools/text.xsl
@@ -0,0 +1,111 @@
+<?xml version="1.0"?>
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                version="1.0">
+ <xsl:output method="text"/>
+ <xsl:strip-space elements="*"/>
+
+ <xsl:template match="/">
+  <xsl:apply-templates select="article"/>
+ </xsl:template>
+
+ <xsl:template match="article">
+  <xsl:apply-templates select="section"/><xsl:text>
+</xsl:text>
+ </xsl:template>
+
+ <xsl:template match="section">
+  <xsl:apply-templates select="title|para|cmdsynopsis|section"/>
+ </xsl:template>
+
+ <xsl:template match="title">
+  <xsl:apply-templates/><xsl:text>
+</xsl:text>
+ </xsl:template>
+
+ <xsl:template match="subtitle">
+  <xsl:apply-templates/><xsl:text>
+</xsl:text>
+ </xsl:template>
+
+ <xsl:template match="command|filename|varname|computeroutput|constant">
+  <xsl:apply-templates/>
+ </xsl:template>
+
+ <xsl:template match="option">
+  <xsl:apply-templates/>
+ </xsl:template>
+
+ <xsl:template match="screen">
+  <xsl:text>
+</xsl:text><xsl:apply-templates/><xsl:text>
+</xsl:text>
+ </xsl:template>
+
+
+ <xsl:template match="arg">
+  <xsl:choose>
+   <xsl:when test="@choice='opt'">
+    <xsl:text> [</xsl:text><xsl:apply-templates/><xsl:text>]</xsl:text>
+   </xsl:when>
+   <xsl:otherwise>
+    <xsl:text> </xsl:text><xsl:apply-templates/>
+   </xsl:otherwise>
+  </xsl:choose>
+ </xsl:template>
+
+ <xsl:template match="para">
+  <xsl:text>
+</xsl:text>
+<xsl:apply-templates/><xsl:text>
+</xsl:text>
+</xsl:template>
+
+ <xsl:template match="cmdsynopsis">
+  <xsl:text>
+</xsl:text>
+  <xsl:apply-templates select="command|sbr|arg"/><xsl:text>
+</xsl:text>
+ </xsl:template>
+
+ <xsl:template match="synopfragmentref">
+  <xsl:variable name="target" select="id(@linkend)"/>
+  <xsl:apply-templates select="$target"/>
+ </xsl:template>
+
+ <xsl:template match="synopfragment">
+  <xsl:apply-templates/>
+ </xsl:template>
+
+ <xsl:template match="group">
+  <xsl:text>{ </xsl:text><xsl:for-each select="arg">
+   <xsl:apply-templates/>
+   <xsl:if test="position() != last()"><xsl:text> | </xsl:text></xsl:if>
+  </xsl:for-each><xsl:text> }</xsl:text>
+ </xsl:template>
+
+
+ <xsl:template match="replaceable">{<xsl:apply-templates/>}</xsl:template>
+
+
+ <xsl:template match="sbr">
+  <xsl:text>
+</xsl:text>
+</xsl:template>
+
+ <xsl:template match="text()"><xsl:value-of select="."/></xsl:template>
+
+ <xsl:template match="node()">
+  <xsl:message terminate="yes">Unknown node <xsl:value-of select="name()"/>
+</xsl:message>
+ </xsl:template>
+
+ <xsl:template match="simplelist">
+  <xsl:for-each select="member">
+   <xsl:apply-templates/>
+   <xsl:if test="position() != last()"><xsl:text>, </xsl:text></xsl:if>
+  </xsl:for-each>
+ </xsl:template>
+
+
+</xsl:stylesheet>
diff --git a/ctdb/libctdb/test/tools/usage.xsl b/ctdb/libctdb/test/tools/usage.xsl
new file mode 100644
index 0000000000..1a6150c2b3
--- /dev/null
+++ b/ctdb/libctdb/test/tools/usage.xsl
@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+                version="1.0">
+ <xsl:output method="text"/>
+ <xsl:strip-space elements="*"/>
+ <xsl:include href="text.xsl"/>
+
+ <xsl:template match="section">
+  <xsl:apply-templates select="title"/>
+  <xsl:text>    </xsl:text><xsl:apply-templates select="subtitle"/>
+ </xsl:template>
+
+ <xsl:template match="para">
+  <xsl:apply-templates/>
+ </xsl:template>
+
+</xsl:stylesheet>
diff --git a/ctdb/libctdb/test/tui.c b/ctdb/libctdb/test/tui.c
new file mode 100644
index 0000000000..d411e6f0b4
--- /dev/null
+++ b/ctdb/libctdb/test/tui.c
@@ -0,0 +1,459 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Jeremy Kerr & Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#include "tui.h"
+#include "log.h"
+#include "ctdb-test.h"
+#include "utils.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <errno.h>
+#include <err.h>
+#include <assert.h>
+#include <readline/readline.h>
+#include <readline/history.h>
+#include <talloc.h>
+#include <dlinklist.h>
+
+int tui_echo_commands;
+int tui_abort_on_fail;
+int tui_quiet;
+int tui_linenum = 1;
+char *extension_path;
+static bool stop;
+
+struct command {
+	struct command *next, *prev;
+	char	name[TUI_MAX_CMD_LEN+1];
+	bool	(*handler)(int, char **);
+	void	(*helpfn) (int, char **);
+};
+
+struct pre_post_hook {
+	struct pre_post_hook *next, *prev;
+	void	(*pre)(const char *);
+	bool	(*post)(const char *);
+};
+
+static struct command *commands;
+static struct pre_post_hook *pre_post_hooks;
+
+static bool tui_exit(int argc, char **argv)
+{
+	stop = true;
+	return true;
+}
+
+static bool tui_argtest(int argc, char **argv)
+{
+	int i;
+
+	for (i = 0; i < argc; i++)
+		log_line(LOG_UI, "argv[%d]: \"%s\"", i, argv[i]);
+
+	return true;
+}
+
+static inline struct command *find_command(const char *name)
+{
+	struct command *cmd;
+	for (cmd = commands; cmd; cmd = cmd->next)
+		if (!strcmp(name, cmd->name))
+			return cmd;
+
+	return NULL;
+}
+
+bool tui_is_command(const char *name)
+{
+	return find_command(name) != NULL;
+}
+
+static void do_pre_commands(const char *cmd)
+{
+	struct pre_post_hook *i;
+	for (i = pre_post_hooks; i; i = i->next)
+		if (i->pre)
+			i->pre(cmd);
+}
+
+static bool do_post_commands(const char *cmd)
+{
+	struct pre_post_hook *i;
+	bool ret = true;
+
+	for (i = pre_post_hooks; i; i = i->next)
+		if (i->post && !i->post(cmd))
+			ret = false;
+	return ret;
+}
+
+static bool tui_help(int argc, char **argv)
+{
+	struct command *cmd;
+
+	if (argc == 1) {
+		log_line(LOG_UI, "CTDB tester\n"
+		"help is available on the folowing commands:");
+		for (cmd = commands; cmd; cmd = cmd->next) {
+			if (cmd->helpfn)
+				log_line(LOG_UI, "\t%s", cmd->name);
+		}
+	} else {
+		if (!(cmd = find_command(argv[1]))) {
+			log_line(LOG_ALWAYS, "No such command '%s'", argv[1]);
+			return false;
+		}
+		if (!cmd->helpfn) {
+			log_line(LOG_UI, "No help for the '%s' function",
+				argv[1]);
+			return false;
+		}
+		cmd->helpfn(argc-1, argv+1);
+	}
+	return true;
+
+
+}
+
+static void tui_help_help(int argc, char **argv)
+{
+#include "generated-tui-help:help"
+/*** XML Help:
+    <section id="c:help">
+     <title><command>help</command></title>
+     <para>Displays general help, or help for a specified command</para>
+     <cmdsynopsis>
+      <command>help</command>
+      <arg choice="opt">command</arg>
+     </cmdsynopsis>
+     <para>With no arguments, <command>help</command> will show general system
+      help, and list the available commands. If an argument is specified, then
+      <command>help</command> will show help for that command, if
+      available.</para>
+    </section>
+*/
+}
+
+static void tui_exit_help(int argc, char **argv)
+{
+#include "generated-tui-help:exit"
+/*** XML Help:
+    <section id="c:exit">
+     <title><command>exit</command>,
+     <command>quit</command></title>
+     <para>Exit the simulator</para>
+     <cmdsynopsis>
+      <command>exit</command>
+     </cmdsynopsis>
+     <cmdsynopsis>
+      <command>quit</command>
+     </cmdsynopsis>
+
+     <para>The <command>exit</command> and <command>quit</command>
+      commands are synonomous.  They both exit the simulator.
+     </para>
+    </section>
+ */
+}
+
+void script_fail(const char *fmt, ...)
+{
+	char *str;
+	va_list arglist;
+
+	log_line(LOG_ALWAYS, "Script failed at line %i: ", tui_linenum);
+
+	va_start(arglist, fmt);
+	str = talloc_vasprintf(NULL, fmt, arglist);
+	va_end(arglist);
+
+	log_line(LOG_ALWAYS, "%s", str);
+	talloc_free(str);
+
+	check_allocations();
+	exit(EXIT_SCRIPTFAIL);
+}
+
+bool tui_do_command(int argc, char *argv[], bool abort)
+{
+	struct command *cmd;
+	bool ret = true;
+
+	if ((cmd = find_command(argv[0]))) {
+		do_pre_commands(cmd->name);
+		if (!cmd->handler(argc, argv)) {
+			/* Abort on UNEXPECTED failure. */
+			if (!log_line(LOG_UI, "%s: command failed", argv[0])
+			    && abort)
+				script_fail("%s failed", argv[0]);
+			ret = false;
+		}
+		if (!do_post_commands(cmd->name))
+			ret = false;
+		return ret;
+	}
+
+	if (abort)
+		script_fail("%s not found", argv[0]);
+
+	log_line(LOG_ALWAYS, "%s: command not found", argv[0]);
+	return false;
+}
+
+/**
+ * backslash-escape a binary data block into a newly allocated
+ * string
+ *
+ * @param src a pointer to the data block
+ * @param src_len the length of the data block
+ * @return NULL if out of memory, or a pointer to the allocated escaped
+ *    string, which is terminated with a '\0' character
+ */
+static char *escape(const char *src, size_t src_len)
+{
+	static const char hexbuf[]= "0123456789abcdef";
+	char *dest, *p;
+	size_t i;
+
+	/* src_len * 4 is always safe, it's the longest escape
+	   sequence for all characters */
+	dest = talloc_array(src, char, src_len * 4 + 1);
+	p = dest;
+
+	for (i = 0; i < src_len; i++) {
+		if (src[i] == '\n') {
+			*p++ = '\\';
+			*p++ = 'n';
+		} else if (src[i] == '\r') {
+			*p++ = '\\';
+			*p++ = 'r';
+		} else if (src[i] == '\0') {
+			*p++ = '\\';
+			*p++ = '0';
+		} else if (src[i] == '\t') {
+			*p++ = '\\';
+			*p++ = 't';
+		} else if (src[i] == '\\') {
+			*p++ = '\\';
+			*p++ = '\\';
+		} else if (src[i] & 0x80 || (src[i] & 0xe0) == 0) {
+			*p++ = '\\';
+			*p++ = 'x';
+			*p++ = hexbuf[(src[i] >> 4) & 0xf];
+			*p++ = hexbuf[src[i] & 0xf];
+		} else
+			*p++ = src[i];
+	}
+
+	*p++ = 0;
+	return dest;
+}
+
+/* Process `command`: update off to point to tail backquote */
+static char *backquote(char *line, unsigned int *off)
+{
+	char *end, *cmdstr, *str;
+	FILE *cmdfile;
+	size_t used, len, i;
+	int status;
+
+	/* Skip first backquote, look for next one. */
+	(*off)++;
+	end = strchr(line + *off, '`');
+	if (!end)
+		script_fail("no matching \"`\" found");
+
+	len = end - (line + *off);
+	cmdstr = talloc_asprintf(line, "PATH=%s; %.*s",
+				 extension_path, (int)len, line + *off);
+	cmdfile = popen(cmdstr, "r");
+	if (!cmdfile)
+		script_fail("failed to popen '%s': %s\n",
+			    cmdstr, strerror(errno));
+
+       /* Jump to backquote. */
+       *off += len;
+
+	/* Read command output. */
+	used = 0;
+	len = 1024;
+	str = talloc_array(line, char, len);
+
+	while ((i = fread(str + used, 1, len - used, cmdfile)) != 0) {
+		used += i;
+		if (used == len) {
+			if (len > 1024*1024)
+				script_fail("command '%s' output too long\n",
+					    cmdstr);
+			len *= 2;
+			str = talloc_realloc(line, str, char, len);
+		}
+	}
+	status = pclose(cmdfile);
+	if (status == -1 || !WIFEXITED(status) || WEXITSTATUS(status) != 0)
+		script_fail("command '%s' failed\n", cmdstr);
+
+	return escape(str, used);
+}
+
+static char *append_char(char **argv, unsigned int argc, char c)
+{
+	if (!argv[argc])
+		return talloc_asprintf(argv, "%c", c);
+	return talloc_asprintf_append(argv[argc], "%c", c);
+}
+
+static char *append_string(char **argv, unsigned int argc, const char *str)
+{
+	if (!argv[argc])
+		return talloc_asprintf(argv, "%s", str);
+	return talloc_asprintf_append(argv[argc], "%s", str);
+}
+
+static void process_line(char *line, unsigned int off)
+{
+	unsigned int argc, i;
+	char **argv;
+
+	if (tui_echo_commands)
+		printf("%u:%s\n", tui_linenum, line + off);
+
+	/* Talloc argv off line so commands can use it for auto-cleanup. */
+	argv = talloc_zero_array(line, char *, TUI_MAX_ARGS+1);
+	argc = 0;
+	for (i = off; line[i]; i++) {
+		if (isspace(line[i])) {
+			/* If anything in this arg, move to next. */
+			if (argv[argc])
+				argc++;
+		} else if (line[i] == '`') {
+			char *inside = backquote(line, &i);
+			argv[argc] = append_string(argv, argc, inside);
+		} else {
+			/* If it is a comment, stop before we process `` */
+			if (!argv[0] && line[i] == '#')
+				goto out;
+
+			argv[argc] = append_char(argv, argc, line[i]);
+		}
+	}
+
+	if (argv[0]) {
+		if (argv[argc])
+			argv[++argc] = NULL;
+		tui_do_command(argc, argv, tui_abort_on_fail);
+	}
+
+out:
+	tui_linenum++;
+	return;
+}
+
+static void readline_process_line(char *line)
+{
+	char *talloc_line;
+	if (!line) {
+		stop = true;
+		return;
+	}
+
+	add_history(line);
+
+	/* Readline isn't talloc-aware, so copy string: functions can
+	 * hang temporary variables off this. */
+	talloc_line = talloc_strdup(NULL, line);
+	process_line(talloc_line, 0);
+	talloc_free(talloc_line);
+}
+
+static void run_whole_file(int fd)
+{
+	char *file, *p;
+	size_t size, len;
+
+	file = grab_fd(fd, &size);
+	if (!file)
+		err(1, "Grabbing file");
+
+	for (p = file; p < file + size; p += len+1) {
+		len = strcspn(p, "\n");
+		p[len] = '\0';
+		process_line(file, p - file);
+	}
+}
+
+void tui_run(int fd)
+{
+	tui_register_command("exit", tui_exit, tui_exit_help);
+	tui_register_command("quit", tui_exit, tui_exit_help);
+	tui_register_command("q", tui_exit, tui_exit_help);
+	tui_register_command("test", tui_argtest, NULL);
+	tui_register_command("help", tui_help, tui_help_help);
+
+	if (fd == STDIN_FILENO) {
+		stop = false;
+		rl_callback_handler_install(tui_quiet ? "" : "> ",
+					    readline_process_line);
+		while (!stop)
+			rl_callback_read_char();
+		rl_callback_handler_remove();
+		if (!tui_quiet)
+			printf("\n");
+	} else
+		run_whole_file(fd);
+}
+
+int tui_register_pre_post_hook(void (*pre)(const char *),
+			       bool (*post)(const char *))
+{
+	struct pre_post_hook *h;
+
+	h = talloc(NULL, struct pre_post_hook);
+	h->pre = pre;
+	h->post = post;
+	DLIST_ADD(pre_post_hooks, h);
+	return 0;
+}
+
+int tui_register_command(const char *command,
+			 bool (*handler)(int, char **),
+			 void (*helpfn)(int, char **))
+{
+	struct command *cmd;
+
+	assert(strlen(command) < TUI_MAX_CMD_LEN);
+
+	cmd = talloc(NULL, struct command);
+	strncpy(cmd->name, command, TUI_MAX_CMD_LEN);
+	cmd->handler = handler;
+	cmd->helpfn  = helpfn;
+
+	DLIST_ADD(commands, cmd);
+
+	return 0;
+}
diff --git a/ctdb/libctdb/test/tui.h b/ctdb/libctdb/test/tui.h
new file mode 100644
index 0000000000..3861599edb
--- /dev/null
+++ b/ctdb/libctdb/test/tui.h
@@ -0,0 +1,54 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Jeremy Kerr & Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef __HAVE_TUI_H
+#define __HAVE_TUI_H
+
+#include <stdbool.h>
+
+#define TUI_MAX_CMD_LEN		1024
+#define TUI_MAX_ARGS		128
+
+int tui_register_command(const char *command,
+			 bool (*handler)(int argc, char **argv),
+			 void (*helpfn)(int argc, char **argv));
+
+int tui_register_pre_post_hook(void (*pre)(const char *),
+			       bool (*post)(const char *));
+
+void tui_run(int fd);
+
+bool tui_do_command(int argc, char *argv[], bool abort);
+
+/* Is this a valid command?  Sanity check for expect. */
+bool tui_is_command(const char *name);
+
+/* A script test failed (a command failed with -e, or an expect failed). */
+void script_fail(const char *fmt, ...) __attribute__((noreturn));
+
+extern int tui_echo_commands;
+extern int tui_abort_on_fail;
+extern int tui_quiet;
+extern int tui_linenum;
+
+#endif /* __HAVE_TUI_H */
diff --git a/ctdb/libctdb/test/utils.h b/ctdb/libctdb/test/utils.h
new file mode 100644
index 0000000000..eb84913589
--- /dev/null
+++ b/ctdb/libctdb/test/utils.h
@@ -0,0 +1,87 @@
+/*
+
+This file is taken from nfsim (http://ozlabs.org/~jk/projects/nfsim/)
+
+Copyright (c) 2003,2004 Rusty Russell
+
+This file is part of nfsim.
+
+nfsim is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+nfsim is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with nfsim; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+*/
+
+#ifndef _UTILS_H
+#define _UTILS_H
+#include <stdbool.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <getopt.h>
+
+/* Is A == B ? */
+#define streq(a,b) (strcmp((a),(b)) == 0)
+
+/* Does A start with B ? */
+#define strstarts(a,b) (strncmp((a),(b),strlen(b)) == 0)
+
+/* Does A end in B ? */
+static inline bool strends(const char *a, const char *b)
+{
+	if (strlen(a) < strlen(b))
+		return false;
+
+	return streq(a + strlen(a) - strlen(b), b);
+}
+
+#define ARRAY_SIZE(arr) (sizeof(arr)/sizeof(arr[0]))
+
+/* Paste two tokens together. */
+#define ___cat(a,b) a ## b
+#define __cat(a,b) ___cat(a,b)
+
+/* Try to give a unique identifier: this comes close, iff used as static. */
+#define __unique_id(stem) __cat(__cat(__uniq,stem),__LINE__)
+
+enum exitcodes {
+	/* EXIT_SUCCESS, EXIT_FAILURE is in stdlib.h */
+	EXIT_SCRIPTFAIL = EXIT_FAILURE + 1,
+	EXIT_SILENT,
+};
+
+/* init code */
+typedef void (*initcall_t)(void);
+#define init_call(fn) \
+	static initcall_t __initcall_##fn \
+	__attribute__((__unused__)) \
+	__attribute__((__section__("init_call"))) = &fn
+
+/* distributed command line options */
+struct cmdline_option
+{
+       struct option opt;
+       void (*parse)(struct option *opt);
+}  __attribute__((aligned(64)));	/* align it to 64 for 64bit arch,
+ * <rusty> LaF0rge: space is cheap.  A comment might be nice. */
+
+#define cmdline_opt(_name, _has_arg, _c, _fn)                                \
+       static struct cmdline_option __cat(__cmdlnopt_,__unique_id(_fn))      \
+       __attribute__((__unused__))                                           \
+       __attribute__((__section__("cmdline")))                               \
+       = { .opt = { .name = _name, .has_arg = _has_arg, .val = _c },         \
+	   .parse = _fn }
+
+/* In generated-usage.c */
+void print_usage(void);
+
+#endif /* _UTILS_H */