merge local copy of tdb from samba4 tdb

(This used to be ctdb commit d4619ce98ce44acaebeb6ae9c516a7917bf4e27f)

15 files changed, 763 insertions, 361 deletions

diff --git a/ctdb/lib/tdb/autogen.sh b/ctdb/lib/tdb/autogen.sh
index bf84eeee19..1691689e10 100755
--- a/ctdb/lib/tdb/autogen.sh
+++ b/ctdb/lib/tdb/autogen.sh
@@ -3,7 +3,7 @@
 rm -rf autom4te.cache
 rm -f configure config.h.in
 
-IPATHS="-I libreplace -I lib/replace -I ../libreplace -I ../replace"
+IPATHS="-I libreplace -I lib/replace -I ../libreplace -I ../replace -I ../lib/replace"
 autoconf $IPATHS || exit 1
 autoheader $IPATHS || exit 1
 
diff --git a/ctdb/lib/tdb/common/dump.c b/ctdb/lib/tdb/common/dump.c
index 577f23aac6..3f5c2c87f5 100644
--- a/ctdb/lib/tdb/common/dump.c
+++ b/ctdb/lib/tdb/common/dump.c
@@ -28,7 +28,8 @@
 
 #include "tdb_private.h"
 
-static tdb_off_t tdb_dump_record(struct tdb_context *tdb, tdb_off_t offset)
+static tdb_off_t tdb_dump_record(struct tdb_context *tdb, int hash,
+				 tdb_off_t offset)
 {
 	struct list_struct rec;
 	tdb_off_t tailer_ofs, tailer;
@@ -39,8 +40,10 @@ static tdb_off_t tdb_dump_record(struct tdb_context *tdb, tdb_off_t offset)
 		return 0;
 	}
 
-	printf(" rec: offset=0x%08x next=0x%08x rec_len=%d key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
-	       offset, rec.next, rec.rec_len, rec.key_len, rec.data_len, rec.full_hash, rec.magic);
+	printf(" rec: hash=%d offset=0x%08x next=0x%08x rec_len=%d "
+	       "key_len=%d data_len=%d full_hash=0x%x magic=0x%x\n",
+	       hash, offset, rec.next, rec.rec_len, rec.key_len, rec.data_len,
+	       rec.full_hash, rec.magic);
 
 	tailer_ofs = offset + sizeof(rec) + rec.rec_len - sizeof(tdb_off_t);
 
@@ -72,7 +75,7 @@ static int tdb_dump_chain(struct tdb_context *tdb, int i)
 		printf("hash=%d\n", i);
 
 	while (rec_ptr) {
-		rec_ptr = tdb_dump_record(tdb, rec_ptr);
+		rec_ptr = tdb_dump_record(tdb, i, rec_ptr);
 	}
 
 	return tdb_unlock(tdb, i, F_WRLCK);
diff --git a/ctdb/lib/tdb/common/freelistcheck.c b/ctdb/lib/tdb/common/freelistcheck.c
index 3f79a016b8..63d2dbadc2 100644
--- a/ctdb/lib/tdb/common/freelistcheck.c
+++ b/ctdb/lib/tdb/common/freelistcheck.c
@@ -39,7 +39,7 @@ static int seen_insert(struct tdb_context *mem_tdb, tdb_off_t rec_ptr)
 	TDB_DATA key, data;
 
 	memset(&data, '\0', sizeof(data));
-	key.dptr = (char *)&rec_ptr;
+	key.dptr = (unsigned char *)&rec_ptr;
 	key.dsize = sizeof(rec_ptr);
 	return tdb_store(mem_tdb, key, data, TDB_INSERT);
 }
diff --git a/ctdb/lib/tdb/common/io.c b/ctdb/lib/tdb/common/io.c
index 5d2c5c8e2e..cd06dbb1e3 100644
--- a/ctdb/lib/tdb/common/io.c
+++ b/ctdb/lib/tdb/common/io.c
@@ -124,8 +124,10 @@ static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf,
 		if (ret != (ssize_t)len) {
 			/* Ensure ecode is set for log fn. */
 			tdb->ecode = TDB_ERR_IO;
-			TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %d len=%d ret=%d (%s) map_size=%d\n",
-				 off, len, ret, strerror(errno), (int)tdb->map_size));
+			TDB_LOG((tdb, TDB_DEBUG_FATAL,"tdb_read failed at %d "
+				 "len=%d ret=%d (%s) map_size=%d\n",
+				 (int)off, (int)len, (int)ret, strerror(errno),
+				 (int)tdb->map_size));
 			return TDB_ERRCODE(TDB_ERR_IO, -1);
 		}
 	}
@@ -339,7 +341,7 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len
 		len = 1;
 	}
 
-	if (!(buf = malloc(len))) {
+	if (!(buf = (unsigned char *)malloc(len))) {
 		/* Ensure ecode is set for log fn. */
 		tdb->ecode = TDB_ERR_OOM;
 		TDB_LOG((tdb, TDB_DEBUG_ERROR,"tdb_alloc_read malloc failed len=%d (%s)\n",
@@ -353,6 +355,40 @@ unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len
 	return buf;
 }
 
+/* Give a piece of tdb data to a parser */
+
+int tdb_parse_data(struct tdb_context *tdb, TDB_DATA key,
+		   tdb_off_t offset, tdb_len_t len,
+		   int (*parser)(TDB_DATA key, TDB_DATA data,
+				 void *private_data),
+		   void *private_data)
+{
+	TDB_DATA data;
+	int result;
+
+	data.dsize = len;
+
+	if ((tdb->transaction == NULL) && (tdb->map_ptr != NULL)) {
+		/*
+		 * Optimize by avoiding the malloc/memcpy/free, point the
+		 * parser directly at the mmap area.
+		 */
+		if (tdb->methods->tdb_oob(tdb, offset+len, 0) != 0) {
+			return -1;
+		}
+		data.dptr = offset + (unsigned char *)tdb->map_ptr;
+		return parser(key, data, private_data);
+	}
+
+	if (!(data.dptr = tdb_alloc_read(tdb, offset, len))) {
+		return -1;
+	}
+
+	result = parser(key, data, private_data);
+	free(data.dptr);
+	return result;
+}
+
 /* read/write a record */
 int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec)
 {
diff --git a/ctdb/lib/tdb/common/lock.c b/ctdb/lib/tdb/common/lock.c
index a5bff2d0b3..8a964371d3 100644
--- a/ctdb/lib/tdb/common/lock.c
+++ b/ctdb/lib/tdb/common/lock.c
@@ -107,6 +107,9 @@ int tdb_brlock_upgrade(struct tdb_context *tdb, tdb_off_t offset, size_t len)
 /* lock a list in the database. list -1 is the alloc list */
 int tdb_lock(struct tdb_context *tdb, int list, int ltype)
 {
+	struct tdb_lock_type *new_lck;
+	int i;
+
 	/* a global lock allows us to avoid per chain locks */
 	if (tdb->global_lock.count && 
 	    (ltype == tdb->global_lock.ltype || ltype == F_RDLCK)) {
@@ -125,18 +128,50 @@ int tdb_lock(struct tdb_context *tdb, int list, int ltype)
 	if (tdb->flags & TDB_NOLOCK)
 		return 0;
 
+	for (i=0; i<tdb->num_lockrecs; i++) {
+		if (tdb->lockrecs[i].list == list) {
+			if (tdb->lockrecs[i].count == 0) {
+				/*
+				 * Can't happen, see tdb_unlock(). It should
+				 * be an assert.
+				 */
+				TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_lock: "
+					 "lck->count == 0 for list %d", list));
+			}
+			/*
+			 * Just increment the in-memory struct, posix locks
+			 * don't stack.
+			 */
+			tdb->lockrecs[i].count++;
+			return 0;
+		}
+	}
+
+	new_lck = (struct tdb_lock_type *)realloc(
+		tdb->lockrecs,
+		sizeof(*tdb->lockrecs) * (tdb->num_lockrecs+1));
+	if (new_lck == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+	tdb->lockrecs = new_lck;
+
 	/* Since fcntl locks don't nest, we do a lock for the first one,
 	   and simply bump the count for future ones */
-	if (tdb->locked[list+1].count == 0) {
-		if (tdb->methods->tdb_brlock(tdb,FREELIST_TOP+4*list,ltype,F_SETLKW, 0, 1)) {
-			TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_lock failed on list %d ltype=%d (%s)\n", 
-				 list, ltype, strerror(errno)));
-			return -1;
-		}
-		tdb->locked[list+1].ltype = ltype;
-		tdb->num_locks++;
+	if (tdb->methods->tdb_brlock(tdb,FREELIST_TOP+4*list,ltype,F_SETLKW,
+				     0, 1)) {
+		TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_lock failed on list %d "
+			 "ltype=%d (%s)\n",  list, ltype, strerror(errno)));
+		return -1;
 	}
-	tdb->locked[list+1].count++;
+
+	tdb->num_locks++;
+
+	tdb->lockrecs[tdb->num_lockrecs].list = list;
+	tdb->lockrecs[tdb->num_lockrecs].count = 1;
+	tdb->lockrecs[tdb->num_lockrecs].ltype = ltype;
+	tdb->num_lockrecs += 1;
+
 	return 0;
 }
 
@@ -146,6 +181,8 @@ int tdb_lock(struct tdb_context *tdb, int list, int ltype)
 int tdb_unlock(struct tdb_context *tdb, int list, int ltype)
 {
 	int ret = -1;
+	int i;
+	struct tdb_lock_type *lck = NULL;
 
 	/* a global lock allows us to avoid per chain locks */
 	if (tdb->global_lock.count && 
@@ -166,19 +203,52 @@ int tdb_unlock(struct tdb_context *tdb, int list, int ltype)
 		return ret;
 	}
 
-	if (tdb->locked[list+1].count==0) {
+	for (i=0; i<tdb->num_lockrecs; i++) {
+		if (tdb->lockrecs[i].list == list) {
+			lck = &tdb->lockrecs[i];
+			break;
+		}
+	}
+
+	if ((lck == NULL) || (lck->count == 0)) {
 		TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_unlock: count is 0\n"));
-		return ret;
+		return -1;
+	}
+
+	if (lck->count > 1) {
+		lck->count--;
+		return 0;
 	}
 
-	if (tdb->locked[list+1].count == 1) {
-		/* Down to last nested lock: unlock underneath */
-		ret = tdb->methods->tdb_brlock(tdb, FREELIST_TOP+4*list, F_UNLCK, F_SETLKW, 0, 1);
-		tdb->num_locks--;
-	} else {
-		ret = 0;
+	/*
+	 * This lock has count==1 left, so we need to unlock it in the
+	 * kernel. We don't bother with decrementing the in-memory array
+	 * element, we're about to overwrite it with the last array element
+	 * anyway.
+	 */
+
+	ret = tdb->methods->tdb_brlock(tdb, FREELIST_TOP+4*list, F_UNLCK,
+				       F_SETLKW, 0, 1);
+	tdb->num_locks--;
+
+	/*
+	 * Shrink the array by overwriting the element just unlocked with the
+	 * last array element.
+	 */
+
+	if (tdb->num_lockrecs > 1) {
+		*lck = tdb->lockrecs[tdb->num_lockrecs-1];
+	}
+	tdb->num_lockrecs -= 1;
+
+	/*
+	 * We don't bother with realloc when the array shrinks, but if we have
+	 * a completely idle tdb we should get rid of the locked array.
+	 */
+
+	if (tdb->num_lockrecs == 0) {
+		SAFE_FREE(tdb->lockrecs);
 	}
-	tdb->locked[list+1].count--;
 
 	if (ret)
 		TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_unlock: An error occurred unlocking!\n")); 
diff --git a/ctdb/lib/tdb/common/open.c b/ctdb/lib/tdb/common/open.c
index e1f21aa856..c7fd3f6656 100644
--- a/ctdb/lib/tdb/common/open.c
+++ b/ctdb/lib/tdb/common/open.c
@@ -263,15 +263,7 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 	tdb->map_size = st.st_size;
 	tdb->device = st.st_dev;
 	tdb->inode = st.st_ino;
-	tdb->locked = (struct tdb_lock_type *)calloc(tdb->header.hash_size+1,
-						     sizeof(tdb->locked[0]));
-	if (!tdb->locked) {
-		TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_open_ex: "
-			 "failed to allocate lock structure for %s\n",
-			 name));
-		errno = ENOMEM;
-		goto fail;
-	}
+	tdb->max_dead_records = 0;
 	tdb_mmap(tdb);
 	if (locked) {
 		if (tdb->methods->tdb_brlock(tdb, ACTIVE_LOCK, F_UNLCK, F_SETLK, 0, 1) == -1) {
@@ -324,13 +316,21 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 	if (tdb->fd != -1)
 		if (close(tdb->fd) != 0)
 			TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_open_ex: failed to close tdb->fd on error!\n"));
-	SAFE_FREE(tdb->locked);
 	SAFE_FREE(tdb);
 	errno = save_errno;
 	return NULL;
 	}
 }
 
+/*
+ * Set the maximum number of dead records per hash chain
+ */
+
+void tdb_set_max_dead(struct tdb_context *tdb, int max_dead)
+{
+	tdb->max_dead_records = max_dead;
+}
+
 /**
  * Close a database.
  *
@@ -354,7 +354,7 @@ int tdb_close(struct tdb_context *tdb)
 	SAFE_FREE(tdb->name);
 	if (tdb->fd != -1)
 		ret = close(tdb->fd);
-	SAFE_FREE(tdb->locked);
+	SAFE_FREE(tdb->lockrecs);
 
 	/* Remove from contexts list */
 	for (i = &tdbs; *i; i = &(*i)->next) {
@@ -372,9 +372,9 @@ int tdb_close(struct tdb_context *tdb)
 
 /* register a loging function */
 void tdb_set_logging_function(struct tdb_context *tdb,
-                              const struct tdb_logging_context *log)
+                              const struct tdb_logging_context *log_ctx)
 {
-        tdb->log = *log;
+        tdb->log = *log_ctx;
 }
 
 void *tdb_get_logging_private(struct tdb_context *tdb)
diff --git a/ctdb/lib/tdb/common/tdb.c b/ctdb/lib/tdb/common/tdb.c
index 5810f46d56..25103d826e 100644
--- a/ctdb/lib/tdb/common/tdb.c
+++ b/ctdb/lib/tdb/common/tdb.c
@@ -56,6 +56,10 @@ static void tdb_increment_seqnum(struct tdb_context *tdb)
 	tdb_brlock(tdb, TDB_SEQNUM_OFS, F_UNLCK, F_SETLKW, 1, 1);
 }
 
+static int tdb_key_compare(TDB_DATA key, TDB_DATA data, void *private_data)
+{
+	return memcmp(data.dptr, key.dptr, data.dsize);
+}
 
 /* Returns 0 on fail.  On success, return offset of record, and fills
    in rec */
@@ -73,19 +77,12 @@ static tdb_off_t tdb_find(struct tdb_context *tdb, TDB_DATA key, u32 hash,
 		if (tdb_rec_read(tdb, rec_ptr, r) == -1)
 			return 0;
 
-		if (!TDB_DEAD(r) && hash==r->full_hash && key.dsize==r->key_len) {
-			unsigned char *k;
-			/* a very likely hit - read the key */
-			k = tdb_alloc_read(tdb, rec_ptr + sizeof(*r), 
-					   r->key_len);
-			if (!k)
-				return 0;
-
-			if (memcmp(key.dptr, k, key.dsize) == 0) {
-				SAFE_FREE(k);
-				return rec_ptr;
-			}
-			SAFE_FREE(k);
+		if (!TDB_DEAD(r) && hash==r->full_hash
+		    && key.dsize==r->key_len
+		    && tdb_parse_data(tdb, key, rec_ptr + sizeof(*r),
+				      r->key_len, tdb_key_compare,
+				      NULL) == 0) {
+			return rec_ptr;
 		}
 		rec_ptr = r->next;
 	}
@@ -163,6 +160,47 @@ TDB_DATA tdb_fetch(struct tdb_context *tdb, TDB_DATA key)
 	return ret;
 }
 
+/*
+ * Find an entry in the database and hand the record's data to a parsing
+ * function. The parsing function is executed under the chain read lock, so it
+ * should be fast and should not block on other syscalls.
+ *
+ * DONT CALL OTHER TDB CALLS FROM THE PARSER, THIS MIGHT LEAD TO SEGFAULTS.
+ *
+ * For mmapped tdb's that do not have a transaction open it points the parsing
+ * function directly at the mmap area, it avoids the malloc/memcpy in this
+ * case. If a transaction is open or no mmap is available, it has to do
+ * malloc/read/parse/free.
+ *
+ * This is interesting for all readers of potentially large data structures in
+ * the tdb records, ldb indexes being one example.
+ */
+
+int tdb_parse_record(struct tdb_context *tdb, TDB_DATA key,
+		     int (*parser)(TDB_DATA key, TDB_DATA data,
+				   void *private_data),
+		     void *private_data)
+{
+	tdb_off_t rec_ptr;
+	struct list_struct rec;
+	int ret;
+	u32 hash;
+
+	/* find which hash bucket it is in */
+	hash = tdb->hash_fn(&key);
+
+	if (!(rec_ptr = tdb_find_lock_hash(tdb,key,hash,F_RDLCK,&rec))) {
+		return TDB_ERRCODE(TDB_ERR_NOEXIST, 0);
+	}
+
+	ret = tdb_parse_data(tdb, key, rec_ptr + sizeof(rec) + rec.key_len,
+			     rec.data_len, parser, private_data);
+
+	tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
+
+	return ret;
+}
+
 /* check if an entry in the database exists 
 
    note that 1 is returned if the key is found and 0 is returned if not found
@@ -220,6 +258,66 @@ int tdb_do_delete(struct tdb_context *tdb, tdb_off_t rec_ptr, struct list_struct
 	return 0;
 }
 
+static int tdb_count_dead(struct tdb_context *tdb, u32 hash)
+{
+	int res = 0;
+	tdb_off_t rec_ptr;
+	struct list_struct rec;
+	
+	/* read in the hash top */
+	if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1)
+		return 0;
+
+	while (rec_ptr) {
+		if (tdb_rec_read(tdb, rec_ptr, &rec) == -1)
+			return 0;
+
+		if (rec.magic == TDB_DEAD_MAGIC) {
+			res += 1;
+		}
+		rec_ptr = rec.next;
+	}
+	return res;
+}
+
+/*
+ * Purge all DEAD records from a hash chain
+ */
+static int tdb_purge_dead(struct tdb_context *tdb, u32 hash)
+{
+	int res = -1;
+	struct list_struct rec;
+	tdb_off_t rec_ptr;
+
+	if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
+		return -1;
+	}
+	
+	/* read in the hash top */
+	if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1)
+		goto fail;
+
+	while (rec_ptr) {
+		tdb_off_t next;
+
+		if (tdb_rec_read(tdb, rec_ptr, &rec) == -1) {
+			goto fail;
+		}
+
+		next = rec.next;
+
+		if (rec.magic == TDB_DEAD_MAGIC
+		    && tdb_do_delete(tdb, rec_ptr, &rec) == -1) {
+			goto fail;
+		}
+		rec_ptr = next;
+	}
+	res = 0;
+ fail:
+	tdb_unlock(tdb, -1, F_WRLCK);
+	return res;
+}
+
 /* delete an entry in the database given a key */
 static int tdb_delete_hash(struct tdb_context *tdb, TDB_DATA key, u32 hash)
 {
@@ -227,9 +325,42 @@ static int tdb_delete_hash(struct tdb_context *tdb, TDB_DATA key, u32 hash)
 	struct list_struct rec;
 	int ret;
 
-	if (!(rec_ptr = tdb_find_lock_hash(tdb, key, hash, F_WRLCK, &rec)))
-		return -1;
-	ret = tdb_do_delete(tdb, rec_ptr, &rec);
+	if (tdb->max_dead_records != 0) {
+
+		/*
+		 * Allow for some dead records per hash chain, mainly for
+		 * tdb's with a very high create/delete rate like locking.tdb.
+		 */
+
+		if (tdb_lock(tdb, BUCKET(hash), F_WRLCK) == -1)
+			return -1;
+
+		if (tdb_count_dead(tdb, hash) >= tdb->max_dead_records) {
+			/*
+			 * Don't let the per-chain freelist grow too large,
+			 * delete all existing dead records
+			 */
+			tdb_purge_dead(tdb, hash);
+		}
+
+		if (!(rec_ptr = tdb_find(tdb, key, hash, &rec))) {
+			tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
+			return -1;
+		}
+
+		/*
+		 * Just mark the record as dead.
+		 */
+		rec.magic = TDB_DEAD_MAGIC;
+		ret = tdb_rec_write(tdb, rec_ptr, &rec);
+	}
+	else {
+		if (!(rec_ptr = tdb_find_lock_hash(tdb, key, hash, F_WRLCK,
+						   &rec)))
+			return -1;
+
+		ret = tdb_do_delete(tdb, rec_ptr, &rec);
+	}
 
 	if (ret == 0) {
 		tdb_increment_seqnum(tdb);
@@ -246,6 +377,35 @@ int tdb_delete(struct tdb_context *tdb, TDB_DATA key)
 	return tdb_delete_hash(tdb, key, hash);
 }
 
+/*
+ * See if we have a dead record around with enough space
+ */
+static tdb_off_t tdb_find_dead(struct tdb_context *tdb, u32 hash,
+			       struct list_struct *r, tdb_len_t length)
+{
+	tdb_off_t rec_ptr;
+	
+	/* read in the hash top */
+	if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec_ptr) == -1)
+		return 0;
+
+	/* keep looking until we find the right record */
+	while (rec_ptr) {
+		if (tdb_rec_read(tdb, rec_ptr, r) == -1)
+			return 0;
+
+		if (TDB_DEAD(r) && r->rec_len >= length) {
+			/*
+			 * First fit for simple coding, TODO: change to best
+			 * fit
+			 */
+			return rec_ptr;
+		}
+		rec_ptr = r->next;
+	}
+	return 0;
+}
+
 /* store an element in the database, replacing any existing element
    with the same key 
 
@@ -257,7 +417,7 @@ int tdb_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, int flag)
 	u32 hash;
 	tdb_off_t rec_ptr;
 	char *p = NULL;
-	int ret = 0;
+	int ret = -1;
 
 	if (tdb->read_only || tdb->traverse_read) {
 		tdb->ecode = TDB_ERR_RDONLY;
@@ -277,8 +437,9 @@ int tdb_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, int flag)
 		}
 	} else {
 		/* first try in-place update, on modify or replace. */
-		if (tdb_update_hash(tdb, key, hash, dbuf) == 0)
-			goto out;
+		if (tdb_update_hash(tdb, key, hash, dbuf) == 0) {
+			goto done;
+		}
 		if (tdb->ecode == TDB_ERR_NOEXIST &&
 		    flag == TDB_MODIFY) {
 			/* if the record doesn't exist and we are in TDB_MODIFY mode then
@@ -307,9 +468,56 @@ int tdb_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, int flag)
 	if (dbuf.dsize)
 		memcpy(p+key.dsize, dbuf.dptr, dbuf.dsize);
 
+	if (tdb->max_dead_records != 0) {
+		/*
+		 * Allow for some dead records per hash chain, look if we can
+		 * find one that can hold the new record. We need enough space
+		 * for key, data and tailer. If we find one, we don't have to
+		 * consult the central freelist.
+		 */
+		rec_ptr = tdb_find_dead(
+			tdb, hash, &rec,
+			key.dsize + dbuf.dsize + sizeof(tdb_off_t));
+
+		if (rec_ptr != 0) {
+			rec.key_len = key.dsize;
+			rec.data_len = dbuf.dsize;
+			rec.full_hash = hash;
+			rec.magic = TDB_MAGIC;
+			if (tdb_rec_write(tdb, rec_ptr, &rec) == -1
+			    || tdb->methods->tdb_write(
+				    tdb, rec_ptr + sizeof(rec),
+				    p, key.dsize + dbuf.dsize) == -1) {
+				goto fail;
+			}
+			goto done;
+		}
+	}
+
+	/*
+	 * We have to allocate some space from the freelist, so this means we
+	 * have to lock it. Use the chance to purge all the DEAD records from
+	 * the hash chain under the freelist lock.
+	 */
+
+	if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
+		goto fail;
+	}
+
+	if ((tdb->max_dead_records != 0)
+	    && (tdb_purge_dead(tdb, hash) == -1)) {
+		tdb_unlock(tdb, -1, F_WRLCK);
+		goto fail;
+	}
+
 	/* we have to allocate some space */
-	if (!(rec_ptr = tdb_allocate(tdb, key.dsize + dbuf.dsize, &rec)))
+	rec_ptr = tdb_allocate(tdb, key.dsize + dbuf.dsize, &rec);
+
+	tdb_unlock(tdb, -1, F_WRLCK);
+
+	if (rec_ptr == 0) {
 		goto fail;
+	}
 
 	/* Read hash top into next ptr */
 	if (tdb_ofs_read(tdb, TDB_HASH_TOP(hash), &rec.next) == -1)
@@ -328,15 +536,16 @@ int tdb_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, int flag)
 		goto fail;
 	}
 
-	tdb_increment_seqnum(tdb);
+ done:
+	ret = 0;
+ fail:
+	if (ret == 0) {
+		tdb_increment_seqnum(tdb);
+	}
 
- out:
 	SAFE_FREE(p); 
 	tdb_unlock(tdb, BUCKET(hash), F_WRLCK);
 	return ret;
-fail:
-	ret = -1;
-	goto out;
 }
 
 
@@ -355,9 +564,10 @@ int tdb_append(struct tdb_context *tdb, TDB_DATA key, TDB_DATA new_dbuf)
 	dbuf = tdb_fetch(tdb, key);
 
 	if (dbuf.dptr == NULL) {
-		dbuf.dptr = malloc(new_dbuf.dsize);
+		dbuf.dptr = (unsigned char *)malloc(new_dbuf.dsize);
 	} else {
-		dbuf.dptr = realloc(dbuf.dptr, dbuf.dsize + new_dbuf.dsize);
+		dbuf.dptr = (unsigned char *)realloc(dbuf.dptr,
+						     dbuf.dsize + new_dbuf.dsize);
 	}
 
 	if (dbuf.dptr == NULL) {
diff --git a/ctdb/lib/tdb/common/tdb_private.h b/ctdb/lib/tdb/common/tdb_private.h
index a3495e42aa..9d39de0200 100644
--- a/ctdb/lib/tdb/common/tdb_private.h
+++ b/ctdb/lib/tdb/common/tdb_private.h
@@ -123,6 +123,7 @@ struct tdb_header {
 };
 
 struct tdb_lock_type {
+	int list;
 	u32 count;
 	u32 ltype;
 };
@@ -152,7 +153,8 @@ struct tdb_context {
 	int read_only; /* opened read-only */
 	int traverse_read; /* read-only traversal */
 	struct tdb_lock_type global_lock;
-	struct tdb_lock_type *locked; /* array of chain locks */
+	int num_lockrecs;
+	struct tdb_lock_type *lockrecs; /* only real locks, all with count>0 */
 	enum TDB_ERROR ecode; /* error code for last tdb error */
 	struct tdb_header header; /* a cached copy of the header */
 	u32 flags; /* the flags passed to tdb_open */
@@ -167,6 +169,7 @@ struct tdb_context {
 	const struct tdb_methods *methods;
 	struct tdb_transaction *transaction;
 	int page_size;
+	int max_dead_records;
 };
 
 
@@ -194,9 +197,16 @@ int tdb_rec_read(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *
 int tdb_rec_write(struct tdb_context *tdb, tdb_off_t offset, struct list_struct *rec);
 int tdb_do_delete(struct tdb_context *tdb, tdb_off_t rec_ptr, struct list_struct *rec);
 unsigned char *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len);
+int tdb_parse_data(struct tdb_context *tdb, TDB_DATA key,
+		   tdb_off_t offset, tdb_len_t len,
+		   int (*parser)(TDB_DATA key, TDB_DATA data,
+				 void *private_data),
+		   void *private_data);
 tdb_off_t tdb_find_lock_hash(struct tdb_context *tdb, TDB_DATA key, u32 hash, int locktype,
 			   struct list_struct *rec);
 void tdb_io_init(struct tdb_context *tdb);
 int tdb_expand(struct tdb_context *tdb, tdb_off_t size);
+int rec_free_read(struct tdb_context *tdb, tdb_off_t off,
+		  struct list_struct *rec);
 
 
diff --git a/ctdb/lib/tdb/common/transaction.c b/ctdb/lib/tdb/common/transaction.c
index 7dff9a95e3..0a609af521 100644
--- a/ctdb/lib/tdb/common/transaction.c
+++ b/ctdb/lib/tdb/common/transaction.c
@@ -39,7 +39,7 @@
     by the header. This removes the need for extra journal files as
     used by some other databases
 
-  - dymacially allocated the transaction recover record, re-using it
+  - dynamically allocated the transaction recover record, re-using it
     for subsequent transactions. If a larger record is needed then
     tdb_free() the old record to place it on the normal tdb freelist
     before allocating the new record
@@ -88,6 +88,12 @@
 
 */
 
+struct tdb_transaction_el {
+	struct tdb_transaction_el *next, *prev;
+	tdb_off_t offset;
+	tdb_len_t length;
+	unsigned char *data;
+};
 
 /*
   hold the context of any current transaction
@@ -105,12 +111,7 @@ struct tdb_transaction {
 	   ordered, with first element at the front of the list. It
 	   needs to be doubly linked as the read/write traversals need
 	   to be backwards, while the commit needs to be forwards */
-	struct tdb_transaction_el {
-		struct tdb_transaction_el *next, *prev;
-		tdb_off_t offset;
-		tdb_len_t length;
-		unsigned char *data;
-	} *elements, *elements_last;
+	struct tdb_transaction_el *elements, *elements_last;
 
 	/* non-zero when an internal transaction error has
 	   occurred. All write operations will then fail until the
@@ -357,8 +358,8 @@ static int transaction_expand_file(struct tdb_context *tdb, tdb_off_t size,
 /*
   brlock during a transaction - ignore them
 */
-int transaction_brlock(struct tdb_context *tdb, tdb_off_t offset, 
-		       int rw_type, int lck_type, int probe, size_t len)
+static int transaction_brlock(struct tdb_context *tdb, tdb_off_t offset, 
+			      int rw_type, int lck_type, int probe, size_t len)
 {
 	return 0;
 }
@@ -516,14 +517,14 @@ int tdb_transaction_cancel(struct tdb_context *tdb)
 
 	/* remove any locks created during the transaction */
 	if (tdb->num_locks != 0) {
-		int h;
-		for (h=0;h<tdb->header.hash_size+1;h++) {
-			if (tdb->locked[h].count != 0) {
-				tdb_brlock(tdb,FREELIST_TOP+4*h,F_UNLCK,F_SETLKW, 0, 1);
-				tdb->locked[h].count = 0;
-			}
+		int i;
+		for (i=0;i<tdb->num_lockrecs;i++) {
+			tdb_brlock(tdb,FREELIST_TOP+4*tdb->lockrecs[i].list,
+				   F_UNLCK,F_SETLKW, 0, 1);
 		}
 		tdb->num_locks = 0;
+		tdb->num_lockrecs = 0;
+		SAFE_FREE(tdb->lockrecs);
 	}
 
 	/* restore the normal io methods */
diff --git a/ctdb/lib/tdb/config.mk b/ctdb/lib/tdb/config.mk
index 0162b78381..24e9de744c 100644
--- a/ctdb/lib/tdb/config.mk
+++ b/ctdb/lib/tdb/config.mk
@@ -18,7 +18,6 @@ PUBLIC_HEADERS = include/tdb.h
 # Start BINARY tdbtool
 [BINARY::tdbtool]
 INSTALLDIR = BINDIR
-ENABLE = NO
 OBJ_FILES= \
 		tools/tdbtool.o
 PRIVATE_DEPENDENCIES = \
diff --git a/ctdb/lib/tdb/include/config.h.in b/ctdb/lib/tdb/include/config.h.in
index 68a0b60bf6..420cb32c4b 100644
--- a/ctdb/lib/tdb/include/config.h.in
+++ b/ctdb/lib/tdb/include/config.h.in
@@ -77,9 +77,6 @@
 /* Define to 1 if you have the `endnetgrent' function. */
 #undef HAVE_ENDNETGRENT
 
-/* Define to 1 if you have the `epoll_create' function. */
-#undef HAVE_EPOLL_CREATE
-
 /* Whether errno() is available */
 #undef HAVE_ERRNO_DECL
 
@@ -236,6 +233,9 @@
 /* Define to 1 if you have the `seteuid' function. */
 #undef HAVE_SETEUID
 
+/* Define to 1 if you have the <setjmp.h> header file. */
+#undef HAVE_SETJMP_H
+
 /* Define to 1 if you have the `setlinebuf' function. */
 #undef HAVE_SETLINEBUF
 
@@ -358,9 +358,6 @@
    */
 #undef HAVE_SYS_DIR_H
 
-/* Define to 1 if you have the <sys/epoll.h> header file. */
-#undef HAVE_SYS_EPOLL_H
-
 /* Define to 1 if you have the <sys/fcntl.h> header file. */
 #undef HAVE_SYS_FCNTL_H
 
@@ -452,6 +449,9 @@
 /* Define to 1 if you have the <unistd.h> header file. */
 #undef HAVE_UNISTD_H
 
+/* Define to 1 if you have the `unsetenv' function. */
+#undef HAVE_UNSETENV
+
 /* Define to 1 if you have the `usleep' function. */
 #undef HAVE_USLEEP
 
diff --git a/ctdb/lib/tdb/include/tdb.h b/ctdb/lib/tdb/include/tdb.h
index cbcaf9023c..dafe2a130e 100644
--- a/ctdb/lib/tdb/include/tdb.h
+++ b/ctdb/lib/tdb/include/tdb.h
@@ -94,13 +94,18 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
 			 int open_flags, mode_t mode,
 			 const struct tdb_logging_context *log_ctx,
 			 tdb_hash_func hash_fn);
+void tdb_set_max_dead(struct tdb_context *tdb, int max_dead);
 
 int tdb_reopen(struct tdb_context *tdb);
 int tdb_reopen_all(int parent_longlived);
-void tdb_set_logging_function(struct tdb_context *tdb, const struct tdb_logging_context *log);
+void tdb_set_logging_function(struct tdb_context *tdb, const struct tdb_logging_context *log_ctx);
 enum TDB_ERROR tdb_error(struct tdb_context *tdb);
 const char *tdb_errorstr(struct tdb_context *tdb);
 TDB_DATA tdb_fetch(struct tdb_context *tdb, TDB_DATA key);
+int tdb_parse_record(struct tdb_context *tdb, TDB_DATA key,
+		     int (*parser)(TDB_DATA key, TDB_DATA data,
+				   void *private_data),
+		     void *private_data);
 int tdb_delete(struct tdb_context *tdb, TDB_DATA key);
 int tdb_store(struct tdb_context *tdb, TDB_DATA key, TDB_DATA dbuf, int flag);
 int tdb_append(struct tdb_context *tdb, TDB_DATA key, TDB_DATA new_dbuf);
diff --git a/ctdb/lib/tdb/tdb.pc b/ctdb/lib/tdb/tdb.pc
index 3fde368922..e2984a6112 100644
--- a/ctdb/lib/tdb/tdb.pc
+++ b/ctdb/lib/tdb/tdb.pc
@@ -5,6 +5,7 @@ includedir=${prefix}/include
 
 Name: tdb
 Description: Trivial Database Library
+Requires.private: 
 Version: 0.0.1
 Libs: -L${libdir} -ltdb
 Libs.private: -lreplace 
diff --git a/ctdb/lib/tdb/tools/tdbtest.c b/ctdb/lib/tdb/tools/tdbtest.c
index 1564a42bc4..416bc50a5b 100644
--- a/ctdb/lib/tdb/tools/tdbtest.c
+++ b/ctdb/lib/tdb/tools/tdbtest.c
@@ -214,7 +214,7 @@ static void merge_test(void)
 	key.dptr = keys[3];
 	tdb_delete(db, key);
 }
-	
+
  int main(int argc, const char *argv[])
 {
 	int i, seed=0;
diff --git a/ctdb/lib/tdb/tools/tdbtool.c b/ctdb/lib/tdb/tools/tdbtool.c
index de0ecdd7ca..cf801d5dc5 100644
--- a/ctdb/lib/tdb/tools/tdbtool.c
+++ b/ctdb/lib/tdb/tools/tdbtool.c
@@ -21,50 +21,82 @@
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
 
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <string.h>
-#include <fcntl.h>
-#include <time.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <ctype.h>
-#include <signal.h>
-#include <stdarg.h>
+#include "replace.h"
+#include "system/locale.h"
+#include "system/time.h"
+#include "system/filesys.h"
 #include "tdb.h"
 
-/* a tdb tool for manipulating a tdb database */
-
-#define FSTRING_LEN 256
-typedef char fstring[FSTRING_LEN];
-
-typedef struct connections_key {
-	pid_t pid;
-	int cnum;
-	fstring name;
-} connections_key;
+static int do_command(void);
+const char *cmdname;
+char *arg1, *arg2;
+size_t arg1len, arg2len;
+int bIterate = 0;
+char *line;
+TDB_DATA iterate_kbuf;
+char cmdline[1024];
+
+enum commands {
+	CMD_CREATE_TDB,
+	CMD_OPEN_TDB,
+	CMD_ERASE,
+	CMD_DUMP,
+	CMD_INSERT,
+	CMD_MOVE,
+	CMD_STORE,
+	CMD_SHOW,
+	CMD_KEYS,
+	CMD_HEXKEYS,
+	CMD_DELETE,
+	CMD_LIST_HASH_FREE,
+	CMD_LIST_FREE,
+	CMD_INFO,
+	CMD_FIRST,
+	CMD_NEXT,
+	CMD_SYSTEM,
+	CMD_QUIT,
+	CMD_HELP
+};
+
+typedef struct {
+	const char *name;
+	enum commands cmd;
+} COMMAND_TABLE;
+
+COMMAND_TABLE cmd_table[] = {
+	{"create",	CMD_CREATE_TDB},
+	{"open",	CMD_OPEN_TDB},
+	{"erase",	CMD_ERASE},
+	{"dump",	CMD_DUMP},
+	{"insert",	CMD_INSERT},
+	{"move",	CMD_MOVE},
+	{"store",	CMD_STORE},
+	{"show",	CMD_SHOW},
+	{"keys",	CMD_KEYS},
+	{"hexkeys",	CMD_HEXKEYS},
+	{"delete",	CMD_DELETE},
+	{"list",	CMD_LIST_HASH_FREE},
+	{"free",	CMD_LIST_FREE},
+	{"info",	CMD_INFO},
+	{"first",	CMD_FIRST},
+	{"1",		CMD_FIRST},
+	{"next",	CMD_NEXT},
+	{"n",		CMD_NEXT},
+	{"quit",	CMD_QUIT},
+	{"q",		CMD_QUIT},
+	{"!",		CMD_SYSTEM},
+	{NULL,		CMD_HELP}
+};
 
-typedef struct connections_data {
-	int magic;
-	pid_t pid;
-	int cnum;
-	uid_t uid;
-	gid_t gid;
-	char name[24];
-	char addr[24];
-	char machine[128];
-	time_t start;
-} connections_data;
+/* a tdb tool for manipulating a tdb database */
 
-static struct tdb_context *tdb;
+static TDB_CONTEXT *tdb;
 
-static int print_rec(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
+static int print_rec(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
+static int print_key(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
+static int print_hexkey(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state);
 
-static void print_asc(unsigned char *buf,int len)
+static void print_asc(const char *buf,int len)
 {
 	int i;
 
@@ -78,20 +110,7 @@ static void print_asc(unsigned char *buf,int len)
 		printf("%c",isprint(buf[i])?buf[i]:'.');
 }
 
-#ifdef PRINTF_ATTRIBUTE
-static void tdb_log(struct tdb_context *t, enum tdb_debug_level level, const char *format, ...) PRINTF_ATTRIBUTE(3,4);
-#endif
-static void tdb_log(struct tdb_context *t, enum tdb_debug_level level, const char *format, ...)
-{
-	va_list ap;
-    
-	va_start(ap, format);
-	vfprintf(stdout, format, ap);
-	va_end(ap);
-	fflush(stdout);
-}
-
-static void print_data(unsigned char *buf,int len)
+static void print_data(const char *buf,int len)
 {
 	int i=0;
 	if (len<=0) return;
@@ -131,6 +150,9 @@ static void help(void)
 "  open      dbname     : open an existing database\n"
 "  erase                : erase the database\n"
 "  dump                 : dump the database as strings\n"
+"  keys                 : dump the database keys as strings\n"
+"  hexkeys              : dump the database keys as hex values\n"
+"  info                 : print summary info about the database\n"
 "  insert    key  data  : insert a record\n"
 "  move      key  file  : move a record to a destination tdb\n"
 "  store     key  data  : store a record (replace)\n"
@@ -138,6 +160,7 @@ static void help(void)
 "  delete    key        : delete a record by key\n"
 "  list                 : print the database hash table and freelist\n"
 "  free                 : print the database freelist\n"
+"  ! command            : execute system command\n"             
 "  1 | first            : print the first record\n"
 "  n | next             : print the next record\n"
 "  q | quit             : terminate\n"
@@ -150,110 +173,62 @@ static void terror(const char *why)
 	printf("%s\n", why);
 }
 
-static char *get_token(int startover)
+static void create_tdb(const char *tdbname)
 {
-	static char tmp[1024];
-	static char *cont = NULL;
-	char *insert, *start;
-	char *k = strtok(NULL, " ");
-
-	if (!k)
-	  return NULL;
-
-	if (startover)
-	  start = tmp;
-	else
-	  start = cont;
-
-	strcpy(start, k);
-	insert = start + strlen(start) - 1;
-	while (*insert == '\\') {
-	  *insert++ = ' ';
-	  k = strtok(NULL, " ");
-	  if (!k)
-	    break;
-	  strcpy(insert, k);
-	  insert = start + strlen(start) - 1;
-	}
-
-	/* Get ready for next call */
-	cont = start + strlen(start) + 1;
-	return start;
-}
-
-static void create_tdb(void)
-{
-	char *tok = get_token(1);
-
-	struct tdb_logging_context log_ctx;
-	log_ctx.log_fn = tdb_log;
-
-	if (!tok) {
-		help();
-		return;
-	}
 	if (tdb) tdb_close(tdb);
-	tdb = tdb_open_ex(tok, 0, TDB_CLEAR_IF_FIRST,
-			  O_RDWR | O_CREAT | O_TRUNC, 0600, &log_ctx, NULL);
+	tdb = tdb_open(tdbname, 0, TDB_CLEAR_IF_FIRST,
+		       O_RDWR | O_CREAT | O_TRUNC, 0600);
 	if (!tdb) {
-		printf("Could not create %s: %s\n", tok, strerror(errno));
+		printf("Could not create %s: %s\n", tdbname, strerror(errno));
 	}
 }
 
-static void open_tdb(void)
+static void open_tdb(const char *tdbname)
 {
-	struct tdb_logging_context log_ctx;
-	char *tok = get_token(1);
-
-	log_ctx.log_fn = tdb_log;
-
-	if (!tok) {
-		help();
-		return;
-	}
 	if (tdb) tdb_close(tdb);
-	tdb = tdb_open_ex(tok, 0, 0, O_RDWR, 0600, &log_ctx, NULL);
+	tdb = tdb_open(tdbname, 0, 0, O_RDWR, 0600);
 	if (!tdb) {
-		printf("Could not open %s: %s\n", tok, strerror(errno));
+		printf("Could not open %s: %s\n", tdbname, strerror(errno));
 	}
 }
 
-static void insert_tdb(void)
+static void insert_tdb(char *keyname, size_t keylen, char* data, size_t datalen)
 {
-	char *k = get_token(1);
-	char *d = get_token(0);
 	TDB_DATA key, dbuf;
 
-	if (!k || !d) {
-		help();
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror("need key");
 		return;
 	}
 
-	key.dptr = (unsigned char *)k;
-	key.dsize = strlen(k)+1;
-	dbuf.dptr = (unsigned char *)d;
-	dbuf.dsize = strlen(d)+1;
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+	dbuf.dptr = (unsigned char *)data;
+	dbuf.dsize = datalen;
 
 	if (tdb_store(tdb, key, dbuf, TDB_INSERT) == -1) {
 		terror("insert failed");
 	}
 }
 
-static void store_tdb(void)
+static void store_tdb(char *keyname, size_t keylen, char* data, size_t datalen)
 {
-	char *k = get_token(1);
-	char *d = get_token(0);
 	TDB_DATA key, dbuf;
 
-	if (!k || !d) {
-		help();
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror("need key");
+		return;
+	}
+
+	if ((data == NULL) || (datalen == 0)) {
+		terror("need data");
 		return;
 	}
 
-	key.dptr = (unsigned char *)k;
-	key.dsize = strlen(k)+1;
-	dbuf.dptr = (unsigned char *)d;
-	dbuf.dsize = strlen(d)+1;
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
+	dbuf.dptr = (unsigned char *)data;
+	dbuf.dsize = datalen;
 
 	printf("Storing key:\n");
 	print_rec(tdb, key, dbuf, NULL);
@@ -263,32 +238,24 @@ static void store_tdb(void)
 	}
 }
 
-static void show_tdb(void)
+static void show_tdb(char *keyname, size_t keylen)
 {
-	char *k = get_token(1);
 	TDB_DATA key, dbuf;
 
-	if (!k) {
-		help();
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror("need key");
 		return;
 	}
 
-	key.dptr = (unsigned char *)k;
-	key.dsize = strlen(k)+1;
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
 
 	dbuf = tdb_fetch(tdb, key);
 	if (!dbuf.dptr) {
-		/* maybe it is non-NULL terminated key? */
-		key.dsize = strlen(k); 
-		dbuf = tdb_fetch(tdb, key);
-		
-		if ( !dbuf.dptr ) {
-			terror("fetch failed");
-			return;
-		}
+	    terror("fetch failed");
+	    return;
 	}
 	
-	/* printf("%s : %*.*s\n", k, (int)dbuf.dsize, (int)dbuf.dsize, dbuf.dptr); */
 	print_rec(tdb, key, dbuf, NULL);
 	
 	free( dbuf.dptr );
@@ -296,62 +263,50 @@ static void show_tdb(void)
 	return;
 }
 
-static void delete_tdb(void)
+static void delete_tdb(char *keyname, size_t keylen)
 {
-	char *k = get_token(1);
 	TDB_DATA key;
 
-	if (!k) {
-		help();
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror("need key");
 		return;
 	}
 
-	key.dptr = (unsigned char *)k;
-	key.dsize = strlen(k)+1;
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
 
 	if (tdb_delete(tdb, key) != 0) {
 		terror("delete failed");
 	}
 }
 
-static void move_rec(void)
+static void move_rec(char *keyname, size_t keylen, char* tdbname)
 {
-	char *k = get_token(1);
-	char *file = get_token(0);	
 	TDB_DATA key, dbuf;
-	struct tdb_context *dst_tdb;
-
-	struct tdb_logging_context log_ctx;
-	log_ctx.log_fn = tdb_log;
+	TDB_CONTEXT *dst_tdb;
 
-	if (!k) {
-		help();
+	if ((keyname == NULL) || (keylen == 0)) {
+		terror("need key");
 		return;
 	}
-	
-	if ( !file ) {
+
+	if ( !tdbname ) {
 		terror("need destination tdb name");
 		return;
 	}
 
-	key.dptr = (unsigned char *)k;
-	key.dsize = strlen(k)+1;
+	key.dptr = (unsigned char *)keyname;
+	key.dsize = keylen;
 
 	dbuf = tdb_fetch(tdb, key);
 	if (!dbuf.dptr) {
-		/* maybe it is non-NULL terminated key? */
-		key.dsize = strlen(k); 
-		dbuf = tdb_fetch(tdb, key);
-		
-		if ( !dbuf.dptr ) {
-			terror("fetch failed");
-			return;
-		}
+		terror("fetch failed");
+		return;
 	}
 	
 	print_rec(tdb, key, dbuf, NULL);
 	
-	dst_tdb = tdb_open_ex(file, 0, 0, O_RDWR, 0600, &log_ctx, NULL);
+	dst_tdb = tdb_open(tdbname, 0, 0, O_RDWR, 0600);
 	if ( !dst_tdb ) {
 		terror("unable to open destination tdb");
 		return;
@@ -368,25 +323,34 @@ static void move_rec(void)
 	return;
 }
 
-static int print_rec(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+static int print_rec(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
 {
-	printf("\nkey %d bytes\n", key.dsize);
-	print_asc(key.dptr, key.dsize);
-	printf("\ndata %d bytes\n", dbuf.dsize);
-	print_data(dbuf.dptr, dbuf.dsize);
+	printf("\nkey %d bytes\n", (int)key.dsize);
+	print_asc((const char *)key.dptr, key.dsize);
+	printf("\ndata %d bytes\n", (int)dbuf.dsize);
+	print_data((const char *)dbuf.dptr, dbuf.dsize);
 	return 0;
 }
 
-static int print_key(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+static int print_key(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
 {
-	print_asc(key.dptr, key.dsize);
+	printf("key %d bytes: ", (int)key.dsize);
+	print_asc((const char *)key.dptr, key.dsize);
+	printf("\n");
+	return 0;
+}
+
+static int print_hexkey(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+{
+	printf("key %d bytes\n", (int)key.dsize);
+	print_data((const char *)key.dptr, key.dsize);
 	printf("\n");
 	return 0;
 }
 
 static int total_bytes;
 
-static int traverse_fn(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
+static int traverse_fn(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *state)
 {
 	total_bytes += dbuf.dsize;
 	return 0;
@@ -396,7 +360,7 @@ static void info_tdb(void)
 {
 	int count;
 	total_bytes = 0;
-	if ((count = tdb_traverse(tdb, traverse_fn, NULL) == -1))
+	if ((count = tdb_traverse(tdb, traverse_fn, NULL)) == -1)
 		printf("Error = %s\n", tdb_errorstr(tdb));
 	else
 		printf("%d records totalling %d bytes\n", count, total_bytes);
@@ -404,23 +368,23 @@ static void info_tdb(void)
 
 static char *tdb_getline(const char *prompt)
 {
-	static char line[1024];
+	static char thisline[1024];
 	char *p;
 	fputs(prompt, stdout);
-	line[0] = 0;
-	p = fgets(line, sizeof(line)-1, stdin);
+	thisline[0] = 0;
+	p = fgets(thisline, sizeof(thisline)-1, stdin);
 	if (p) p = strchr(p, '\n');
 	if (p) *p = 0;
-	return p?line:NULL;
+	return p?thisline:NULL;
 }
 
-static int do_delete_fn(struct tdb_context *the_tdb, TDB_DATA key, TDB_DATA dbuf,
+static int do_delete_fn(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf,
                      void *state)
 {
     return tdb_delete(the_tdb, key);
 }
 
-static void first_record(struct tdb_context *the_tdb, TDB_DATA *pkey)
+static void first_record(TDB_CONTEXT *the_tdb, TDB_DATA *pkey)
 {
 	TDB_DATA dbuf;
 	*pkey = tdb_firstkey(the_tdb);
@@ -428,12 +392,11 @@ static void first_record(struct tdb_context *the_tdb, TDB_DATA *pkey)
 	dbuf = tdb_fetch(the_tdb, *pkey);
 	if (!dbuf.dptr) terror("fetch failed");
 	else {
-		/* printf("%s : %*.*s\n", k, (int)dbuf.dsize, (int)dbuf.dsize, dbuf.dptr); */
 		print_rec(the_tdb, *pkey, dbuf, NULL);
 	}
 }
 
-static void next_record(struct tdb_context *the_tdb, TDB_DATA *pkey)
+static void next_record(TDB_CONTEXT *the_tdb, TDB_DATA *pkey)
 {
 	TDB_DATA dbuf;
 	*pkey = tdb_nextkey(the_tdb, *pkey);
@@ -442,98 +405,202 @@ static void next_record(struct tdb_context *the_tdb, TDB_DATA *pkey)
 	if (!dbuf.dptr) 
 		terror("fetch failed");
 	else
-		/* printf("%s : %*.*s\n", k, (int)dbuf.dsize, (int)dbuf.dsize, dbuf.dptr); */
 		print_rec(the_tdb, *pkey, dbuf, NULL);
 }
 
+static int do_command(void)
+{
+	COMMAND_TABLE *ctp = cmd_table;
+	enum commands mycmd = CMD_HELP;
+	int cmd_len;
+
+	if (cmdname && strlen(cmdname) == 0) {
+	    mycmd = CMD_NEXT;
+	} else {
+	    while (ctp->name) {
+		cmd_len = strlen(ctp->name);
+		if (strncmp(ctp->name,cmdname,cmd_len) == 0) {
+			mycmd = ctp->cmd;
+			break;
+		}
+		ctp++;
+	    }
+	}
+
+	switch (mycmd) {
+	case CMD_CREATE_TDB:
+            bIterate = 0;
+            create_tdb(arg1);
+	    return 0;
+	case CMD_OPEN_TDB:
+            bIterate = 0;
+            open_tdb(arg1);
+            return 0;
+	case CMD_SYSTEM:
+	    /* Shell command */
+	    system(arg1);
+	    return 0;
+	case CMD_QUIT:
+	    return 1;
+	default:
+	    /* all the rest require a open database */
+	    if (!tdb) {
+		bIterate = 0;
+		terror("database not open");
+		help();
+		return 0;
+	    }
+	    switch (mycmd) {
+	    case CMD_ERASE:
+		bIterate = 0;
+		tdb_traverse(tdb, do_delete_fn, NULL);
+		return 0;
+	    case CMD_DUMP:
+		bIterate = 0;
+		tdb_traverse(tdb, print_rec, NULL);
+		return 0;
+	    case CMD_INSERT:
+		bIterate = 0;
+		insert_tdb(arg1, arg1len,arg2,arg2len);
+		return 0;
+	    case CMD_MOVE:
+		bIterate = 0;
+		move_rec(arg1,arg1len,arg2);
+		return 0;
+	    case CMD_STORE:
+		bIterate = 0;
+		store_tdb(arg1,arg1len,arg2,arg2len);
+		return 0;
+	    case CMD_SHOW:
+		bIterate = 0;
+		show_tdb(arg1, arg1len);
+		return 0;
+	    case CMD_KEYS:
+		tdb_traverse(tdb, print_key, NULL);
+		return 0;
+	    case CMD_HEXKEYS:
+		tdb_traverse(tdb, print_hexkey, NULL);
+		return 0;
+	    case CMD_DELETE:
+		bIterate = 0;
+		delete_tdb(arg1,arg1len);
+		return 0;
+	    case CMD_LIST_HASH_FREE:
+		tdb_dump_all(tdb);
+		return 0;
+	    case CMD_LIST_FREE:
+		tdb_printfreelist(tdb);
+		return 0;
+	    case CMD_INFO:
+		info_tdb();
+		return 0;
+	    case CMD_FIRST:
+		bIterate = 1;
+		first_record(tdb, &iterate_kbuf);
+		return 0;
+	    case CMD_NEXT:
+	       if (bIterate)
+		  next_record(tdb, &iterate_kbuf);
+		return 0;
+	    case CMD_HELP:
+		help();
+		return 0;
+            case CMD_CREATE_TDB:
+            case CMD_OPEN_TDB:
+            case CMD_SYSTEM:
+            case CMD_QUIT:
+                /*
+                 * unhandled commands.  cases included here to avoid compiler
+                 * warnings.
+                 */
+                return 0;
+	    }
+	}
+
+	return 0;
+}
+
+static char *convert_string(char *instring, size_t *sizep)
+{
+    size_t length = 0;
+    char *outp, *inp;
+    char temp[3];
+    
+
+    outp = inp = instring;
+
+    while (*inp) {
+	if (*inp == '\\') {
+	    inp++;
+	    if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+		temp[0] = *inp++;
+		temp[1] = '\0';
+		if (*inp && strchr("0123456789abcdefABCDEF",(int)*inp)) {
+		    temp[1] = *inp++;
+		    temp[2] = '\0';
+		}
+		*outp++ = (char)strtol((const char *)temp,NULL,16);
+	    } else {
+		*outp++ = *inp++;
+	    }
+	} else {
+	    *outp++ = *inp++;
+	}
+	length++;
+    }
+    *sizep = length;
+    return instring;
+}
+
 int main(int argc, char *argv[])
 {
-    int bIterate = 0;
-    char *line;
-    char *tok;
-	TDB_DATA iterate_kbuf;
+    cmdname = "";
+    arg1 = NULL;
+    arg1len = 0;
+    arg2 = NULL;
+    arg2len = 0;
 
     if (argv[1]) {
-	static char tmp[1024];
-        sprintf(tmp, "open %s", argv[1]);
-        tok=strtok(tmp," ");
-        open_tdb();
+	cmdname = "open";
+	arg1 = argv[1];
+        do_command();
+	cmdname =  "";
+	arg1 = NULL;
     }
 
-    while ((line = tdb_getline("tdb> "))) {
-
-        /* Shell command */
-        
-        if (line[0] == '!') {
-            system(line + 1);
-            continue;
-        }
-        
-        if ((tok = strtok(line," "))==NULL) {
-           if (bIterate)
-              next_record(tdb, &iterate_kbuf);
-           continue;
-        }
-        if (strcmp(tok,"create") == 0) {
-            bIterate = 0;
-            create_tdb();
-            continue;
-        } else if (strcmp(tok,"open") == 0) {
-            open_tdb();
-            continue;
-        } else if ((strcmp(tok, "q") == 0) ||
-                   (strcmp(tok, "quit") == 0)) {
-            break;
-	}
-            
-        /* all the rest require a open database */
-        if (!tdb) {
-            bIterate = 0;
-            terror("database not open");
-            help();
-            continue;
-        }
-            
-        if (strcmp(tok,"insert") == 0) {
-            bIterate = 0;
-            insert_tdb();
-        } else if (strcmp(tok,"store") == 0) {
-            bIterate = 0;
-            store_tdb();
-        } else if (strcmp(tok,"show") == 0) {
-            bIterate = 0;
-            show_tdb();
-        } else if (strcmp(tok,"erase") == 0) {
-            bIterate = 0;
-            tdb_traverse(tdb, do_delete_fn, NULL);
-        } else if (strcmp(tok,"delete") == 0) {
-            bIterate = 0;
-            delete_tdb();
-        } else if (strcmp(tok,"dump") == 0) {
-            bIterate = 0;
-            tdb_traverse(tdb, print_rec, NULL);
-        } else if (strcmp(tok,"move") == 0) {
-            bIterate = 0;
-            move_rec();
-        } else if (strcmp(tok,"list") == 0) {
-            tdb_dump_all(tdb);
-        } else if (strcmp(tok, "free") == 0) {
-            tdb_printfreelist(tdb);
-        } else if (strcmp(tok,"info") == 0) {
-            info_tdb();
-        } else if ( (strcmp(tok, "1") == 0) ||
-                    (strcmp(tok, "first") == 0)) {
-            bIterate = 1;
-            first_record(tdb, &iterate_kbuf);
-        } else if ((strcmp(tok, "n") == 0) ||
-                   (strcmp(tok, "next") == 0)) {
-            next_record(tdb, &iterate_kbuf);
-        } else if ((strcmp(tok, "keys") == 0)) {
-                bIterate = 0;
-                tdb_traverse(tdb, print_key, NULL);
-        } else {
-            help();
-        }
+    switch (argc) {
+	case 1:
+	case 2:
+	    /* Interactive mode */
+	    while ((cmdname = tdb_getline("tdb> "))) {
+		arg2 = arg1 = NULL;
+		if ((arg1 = strchr((const char *)cmdname,' ')) != NULL) {
+		    arg1++;
+		    arg2 = arg1;
+		    while (*arg2) {
+			if (*arg2 == ' ') {
+			    *arg2++ = '\0';
+			    break;
+			}
+			if ((*arg2++ == '\\') && (*arg2 == ' ')) {
+			    arg2++;
+			}
+		    }
+		}
+		if (arg1) arg1 = convert_string(arg1,&arg1len);
+		if (arg2) arg2 = convert_string(arg2,&arg2len);
+		if (do_command()) break;
+	    }
+	    break;
+	case 5:
+	    arg2 = convert_string(argv[4],&arg2len);
+	case 4:
+	    arg1 = convert_string(argv[3],&arg1len);
+	case 3:
+	    cmdname = argv[2];
+	default:
+	    do_command();
+	    break;
     }
 
     if (tdb) tdb_close(tdb);