Merge remote branch 'ddiss/master_pmda_and_client_timeouts'

(This used to be ctdb commit 7bebfc7bad8f36e54003b8e25372fdaf54836e21)

25 files changed, 1054 insertions, 46 deletions

diff --git a/ctdb/Makefile.in b/ctdb/Makefile.in
index df1d010b87..01483d0b77 100755
--- a/ctdb/Makefile.in
+++ b/ctdb/Makefile.in
@@ -29,6 +29,10 @@ POPT_LIBS = @POPT_LIBS@
 POPT_CFLAGS = @POPT_CFLAGS@
 POPT_OBJ = @POPT_OBJ@
 
+PMDA_LIBS = -lpcp -lpcp_pmda
+PMDA_INSTALL = @CTDB_PMDA_INSTALL@
+PMDA_DEST_DIR = /var/lib/pcp/pmdas
+
 CFLAGS=-g -I$(srcdir)/include -Iinclude -Ilib -Ilib/util -I$(srcdir) \
        -I@tallocdir@ -I@tdbdir@/include -I@libreplacedir@ \
 	-DVARDIR=\"$(localstatedir)\" -DETCDIR=\"$(etcdir)\" \
@@ -73,7 +77,8 @@ TEST_BINS=tests/bin/ctdb_bench tests/bin/ctdb_fetch tests/bin/ctdb_fetch_one \
 	tests/bin/ctdb_takeover_tests
 	@INFINIBAND_BINS@
 
-BINS = bin/ctdb @CTDB_SCSI_IO@ bin/smnotify bin/ping_pong bin/ltdbtool
+BINS = bin/ctdb @CTDB_SCSI_IO@ bin/smnotify bin/ping_pong bin/ltdbtool @CTDB_PMDA@
+
 SBINS = bin/ctdbd
 
 DIRS = lib bin tests/bin
@@ -150,6 +155,9 @@ bin/ping_pong: utils/ping_pong/ping_pong.o
 	@echo Linking $@
 	@$(CC) $(CFLAGS) -o $@ utils/ping_pong/ping_pong.o
 
+bin/pmdactdb: $(CTDB_CLIENT_OBJ) utils/pmda/pmda_ctdb.o
+	@echo Linking $@
+	@$(CC) $(CFLAGS) -o $@ utils/pmda/pmda_ctdb.o $(CTDB_CLIENT_OBJ) $(LIB_FLAGS) $(PMDA_LIBS)
 
 tests/bin/rb_test: $(CTDB_CLIENT_OBJ) tests/src/rb_test.o 
 	@echo Linking $@
@@ -224,7 +232,7 @@ distclean: clean
 	rm -f config.log config.status config.cache config.h
 	rm -f Makefile
 
-install: all
+install: all $(PMDA_INSTALL)
 	mkdir -p $(DESTDIR)$(libdir)/pkgconfig
 	mkdir -p $(DESTDIR)$(bindir)
 	mkdir -p $(DESTDIR)$(sbindir)
@@ -277,6 +285,12 @@ install: all
 	if [ ! -f $(DESTDIR)$(etcdir)/ctdb/notify.sh ];then ${INSTALLCMD} -m 755 config/notify.sh $(DESTDIR)$(etcdir)/ctdb; fi
 	if [ ! -f $(DESTDIR)$(etcdir)/ctdb/ctdb-crash-cleanup.sh ];then ${INSTALLCMD} -m 755 config/ctdb-crash-cleanup.sh $(DESTDIR)$(etcdir)/ctdb; fi
 
+install_pmda:
+	$(INSTALLCMD) -m 755 -d $(PMDA_DEST_DIR)
+	$(INSTALLCMD) -m 755 pmda/Install pmda/Remove $(PMDA_DEST_DIR)
+	$(INSTALLCMD) -m 644 pmda/pmns pmda/domain.h pmda/help pmda/README $(PMDA_DEST_DIR)
+	$(INSTALLCMD) -m 755 bin/pmdactdb $(PMDA_DEST_DIR)
+
 test: all
 	tests/run_tests.sh
 
diff --git a/ctdb/client/ctdb_client.c b/ctdb/client/ctdb_client.c
index 55c32735d9..0828e989d2 100644
--- a/ctdb/client/ctdb_client.c
+++ b/ctdb/client/ctdb_client.c
@@ -186,7 +186,7 @@ static void ctdb_client_reply_control(struct ctdb_context *ctdb, struct ctdb_req
 /*
   this is called in the client, when data comes in from the daemon
  */
-static void ctdb_client_read_cb(uint8_t *data, size_t cnt, void *args)
+void ctdb_client_read_cb(uint8_t *data, size_t cnt, void *args)
 {
 	struct ctdb_context *ctdb = talloc_get_type(args, struct ctdb_context);
 	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
@@ -1696,7 +1696,11 @@ static int ctdb_fetch_func(struct ctdb_call_info *call)
 /*
   attach to a specific database - client call
 */
-struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name, bool persistent, uint32_t tdb_flags)
+struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb,
+				    struct timeval timeout,
+				    const char *name,
+				    bool persistent,
+				    uint32_t tdb_flags)
 {
 	struct ctdb_db_context *ctdb_db;
 	TDB_DATA data;
@@ -1731,7 +1735,7 @@ struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name,
 	ctdb_db->db_id = *(uint32_t *)data.dptr;
 	talloc_free(data.dptr);
 
-	ret = ctdb_ctrl_getdbpath(ctdb, timeval_current_ofs(2, 0), CTDB_CURRENT_NODE, ctdb_db->db_id, ctdb_db, &ctdb_db->db_path);
+	ret = ctdb_ctrl_getdbpath(ctdb, timeout, CTDB_CURRENT_NODE, ctdb_db->db_id, ctdb_db, &ctdb_db->db_path);
 	if (ret != 0) {
 		DEBUG(DEBUG_ERR,("Failed to get dbpath for database '%s'\n", name));
 		talloc_free(ctdb_db);
diff --git a/ctdb/common/cmdline.c b/ctdb/common/cmdline.c
index 145a13a81b..5c00763ed7 100644
--- a/ctdb/common/cmdline.c
+++ b/ctdb/common/cmdline.c
@@ -111,7 +111,8 @@ struct ctdb_context *ctdb_cmdline_init(struct event_context *ev)
 /*
   startup a client only ctdb context
  */
-struct ctdb_context *ctdb_cmdline_client(struct event_context *ev)
+struct ctdb_context *ctdb_cmdline_client(struct tevent_context *ev,
+					 struct timeval req_timeout)
 {
 	struct ctdb_context *ctdb;
 	char *socket_name;
@@ -152,7 +153,7 @@ struct ctdb_context *ctdb_cmdline_client(struct event_context *ev)
 	}
 
 	/* get our pnn */
-	ctdb->pnn = ctdb_ctrl_getpnn(ctdb, timeval_current_ofs(3, 0), CTDB_CURRENT_NODE);
+	ctdb->pnn = ctdb_ctrl_getpnn(ctdb, req_timeout, CTDB_CURRENT_NODE);
 	if (ctdb->pnn == (uint32_t)-1) {
 		DEBUG(DEBUG_CRIT,(__location__ " Failed to get ctdb pnn\n"));
 		talloc_free(ctdb);
diff --git a/ctdb/configure.ac b/ctdb/configure.ac
index cc3a1143da..adcc5dadad 100644
--- a/ctdb/configure.ac
+++ b/ctdb/configure.ac
@@ -64,6 +64,15 @@ m4_include(lib/util/signal.m4)
 m4_include(lib/util/fault.m4)
 
 AC_CHECK_HEADERS(sched.h)
+AC_CHECK_HEADERS(pcp/pmapi.h pcp/impl.h pcp/pmda.h, [], [],
+[[#ifdef HAVE_PCP_PMAPI_H
+# include <pcp/pmapi.h>
+#endif
+#ifdef HAVE_PCP_IMPL_H
+# include <pcp/impl.h>
+#endif
+]])
+
 AC_CHECK_FUNCS(sched_setscheduler)
 AC_CHECK_FUNCS(mlockall)
 
@@ -77,9 +86,20 @@ if test x"$ctdb_cv_HAVE_SOCK_SIN_LEN" = x"yes"; then
     AC_DEFINE(HAVE_SOCK_SIN_LEN,1,[Whether the sockaddr_in struct has a sin_len property])
 fi
 
+if test x"$ac_cv_header_pcp_pmda_h" = x"yes"; then
+    CTDB_PMDA=bin/pmdactdb
+    CTDB_PMDA_INSTALL=install_pmda
+else
+    CTDB_PMDA=
+    CTDB_PMDA_INSTALL=
+fi
+
+
 AC_SUBST(EXTRA_OBJ)
 AC_SUBST(CTDB_SYSTEM_OBJ)
 AC_SUBST(CTDB_SCSI_IO)
 AC_SUBST(CTDB_PCAP_LDFLAGS)
+AC_SUBST(CTDB_PMDA)
+AC_SUBST(CTDB_PMDA_INSTALL)
 
 AC_OUTPUT(Makefile ctdb.pc)
diff --git a/ctdb/include/ctdb_client.h b/ctdb/include/ctdb_client.h
index 3dc115f0cc..95b907378a 100644
--- a/ctdb/include/ctdb_client.h
+++ b/ctdb/include/ctdb_client.h
@@ -105,7 +105,12 @@ int ctdb_start_daemon(struct ctdb_context *ctdb, bool do_fork, bool use_syslog,
 /*
   attach to a ctdb database
 */
-struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb, const char *name, bool persistent, uint32_t tdb_flags);
+struct ctdb_db_context *ctdb_attach(struct ctdb_context *ctdb,
+				    struct timeval timeout,
+				    const char *name,
+				    bool persistent,
+				    uint32_t tdb_flags);
+
 
 /*
   find an attached ctdb_db handle given a name
@@ -188,7 +193,8 @@ int ctdb_register_message_handler(struct ctdb_context *ctdb,
 struct ctdb_db_context *find_ctdb_db(struct ctdb_context *ctdb, uint32_t id);
 
 
-struct ctdb_context *ctdb_cmdline_client(struct tevent_context *ev);
+struct ctdb_context *ctdb_cmdline_client(struct tevent_context *ev,
+					 struct timeval req_timeout);
 
 struct ctdb_statistics;
 int ctdb_ctrl_statistics(struct ctdb_context *ctdb, uint32_t destnode, struct ctdb_statistics *status);
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index b74cbcf0a2..edf1d88dd8 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -798,6 +798,7 @@ int ctdb_call_local(struct ctdb_db_context *ctdb_db, struct ctdb_call *call,
 void ctdb_recv_raw_pkt(void *p, uint8_t *data, uint32_t length);
 
 int ctdb_socket_connect(struct ctdb_context *ctdb);
+void ctdb_client_read_cb(uint8_t *data, size_t cnt, void *args);
 
 #define CTDB_BAD_REQID ((uint32_t)-1)
 uint32_t ctdb_reqid_new(struct ctdb_context *ctdb, void *state);
diff --git a/ctdb/server/ctdb_recoverd.c b/ctdb/server/ctdb_recoverd.c
index 2db9109532..0fcfd3370d 100644
--- a/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@ -841,7 +841,7 @@ static void vacuum_fetch_handler(struct ctdb_context *ctdb, uint64_t srvid,
 	}
 
 	/* attach to it */
-	ctdb_db = ctdb_attach(ctdb, name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, CONTROL_TIMEOUT(), name, persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to attach to database '%s'\n", name));
 		talloc_free(tmp_ctx);
diff --git a/ctdb/tests/src/ctdb_bench.c b/ctdb/tests/src/ctdb_bench.c
index 6d812237ec..8e0e0d7956 100644
--- a/ctdb/tests/src/ctdb_bench.c
+++ b/ctdb/tests/src/ctdb_bench.c
@@ -222,10 +222,11 @@ int main(int argc, const char *argv[])
 	ev = event_context_init(NULL);
 
 	/* initialise ctdb */
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
+	ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb",
+			      false, 0);
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
diff --git a/ctdb/tests/src/ctdb_fetch.c b/ctdb/tests/src/ctdb_fetch.c
index a8f38eeea3..f0311c8948 100644
--- a/ctdb/tests/src/ctdb_fetch.c
+++ b/ctdb/tests/src/ctdb_fetch.c
@@ -207,7 +207,7 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 
 	if (ctdb == NULL) {
 		printf("failed to connect to ctdb daemon.\n");
@@ -218,7 +218,8 @@ int main(int argc, const char *argv[])
 				 &cluster_ready);
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
+	ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb",
+			      false, 0);
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
diff --git a/ctdb/tests/src/ctdb_fetch_one.c b/ctdb/tests/src/ctdb_fetch_one.c
index 7348aa6986..042e605f96 100644
--- a/ctdb/tests/src/ctdb_fetch_one.c
+++ b/ctdb/tests/src/ctdb_fetch_one.c
@@ -114,7 +114,7 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 
 	if (ctdb == NULL) {
 		printf("failed to connect to ctdb deamon.\n");
@@ -122,7 +122,8 @@ int main(int argc, const char *argv[])
 	}
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
+	ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb",
+			      false, 0);
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
diff --git a/ctdb/tests/src/ctdb_persistent.c b/ctdb/tests/src/ctdb_persistent.c
index 6074b6a1e3..98c100f4ed 100644
--- a/ctdb/tests/src/ctdb_persistent.c
+++ b/ctdb/tests/src/ctdb_persistent.c
@@ -220,7 +220,7 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 	if (ctdb == NULL) {
 		printf("Could not attach to daemon\n");
 		return 1;
@@ -228,9 +228,11 @@ int main(int argc, const char *argv[])
 
 	/* attach to a specific database */
 	if (unsafe_writes == 1) {
-		ctdb_db = ctdb_attach(ctdb, "persistent.tdb", true, TDB_NOSYNC);
+		ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0),
+				      "persistent.tdb", true, TDB_NOSYNC);
 	} else {
-		ctdb_db = ctdb_attach(ctdb, "persistent.tdb", true, 0);
+		ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0),
+				      "persistent.tdb", true, 0);
 	}
 
 	if (!ctdb_db) {
diff --git a/ctdb/tests/src/ctdb_randrec.c b/ctdb/tests/src/ctdb_randrec.c
index 5b7eef83d2..dbaa4ba0fc 100644
--- a/ctdb/tests/src/ctdb_randrec.c
+++ b/ctdb/tests/src/ctdb_randrec.c
@@ -181,7 +181,7 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 
 	if (ctdb == NULL) {
 		printf("failed to connect to daemon\n");
@@ -189,7 +189,8 @@ int main(int argc, const char *argv[])
 	}
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
+	ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb",
+			      false, 0);
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
diff --git a/ctdb/tests/src/ctdb_store.c b/ctdb/tests/src/ctdb_store.c
index 82669b7b6d..c657f7850e 100644
--- a/ctdb/tests/src/ctdb_store.c
+++ b/ctdb/tests/src/ctdb_store.c
@@ -136,7 +136,7 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 
 	if (ctdb == NULL) {
 		printf("failed to connect to ctdb daemon.\n");
@@ -144,7 +144,7 @@ int main(int argc, const char *argv[])
 	}
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, "test.tdb", false, 0);
+	ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), "test.tdb", false, 0);
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
diff --git a/ctdb/tests/src/ctdb_transaction.c b/ctdb/tests/src/ctdb_transaction.c
index 39e40dd16b..6508874717 100644
--- a/ctdb/tests/src/ctdb_transaction.c
+++ b/ctdb/tests/src/ctdb_transaction.c
@@ -252,7 +252,7 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 	if (ctdb == NULL) {
 		DEBUG(DEBUG_ERR, ("Could not attach to daemon\n"));
 		return 1;
@@ -260,9 +260,11 @@ int main(int argc, const char *argv[])
 
 	/* attach to a specific database */
 	if (unsafe_writes == 1) {
-		ctdb_db = ctdb_attach(ctdb, "transaction.tdb", true, TDB_NOSYNC);
+		ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0),
+				      "transaction.tdb", true, TDB_NOSYNC);
 	} else {
-		ctdb_db = ctdb_attach(ctdb, "transaction.tdb", true, 0);
+		ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0),
+				      "transaction.tdb", true, 0);
 	}
 
 	if (!ctdb_db) {
diff --git a/ctdb/tests/src/ctdb_traverse.c b/ctdb/tests/src/ctdb_traverse.c
index c4510e43c9..218e7f777d 100644
--- a/ctdb/tests/src/ctdb_traverse.c
+++ b/ctdb/tests/src/ctdb_traverse.c
@@ -89,10 +89,10 @@ int main(int argc, const char *argv[])
 
 	ev = event_context_init(NULL);
 
-	ctdb = ctdb_cmdline_client(ev);
+	ctdb = ctdb_cmdline_client(ev, timeval_current_ofs(3, 0));
 
 	/* attach to a specific database */
-	ctdb_db = ctdb_attach(ctdb, dbname, false, 0);
+	ctdb_db = ctdb_attach(ctdb, timeval_current_ofs(2, 0), dbname, false, 0);
 	if (!ctdb_db) {
 		printf("ctdb_attach failed - %s\n", ctdb_errstr(ctdb));
 		exit(1);
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index 8528de70ee..a8474cf9b8 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -624,9 +624,10 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
 		return ret;
 	}
 
-	if(options.machinereadable){
-		printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped:Inactive:PartiallyOnline:\n");
-		for(i=0;i<nodemap->num;i++){
+	if (options.machinereadable) {
+		printf(":Node:IP:Disconnected:Banned:Disabled:Unhealthy:Stopped"
+		       ":Inactive:PartiallyOnline:ThisNode:\n");
+		for (i=0;i<nodemap->num;i++) {
 			int partially_online = 0;
 			int j;
 
@@ -650,7 +651,7 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
 					talloc_free(ifaces);
 				}
 			}
-			printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:\n", nodemap->nodes[i].pnn,
+			printf(":%d:%s:%d:%d:%d:%d:%d:%d:%d:%d:\n", nodemap->nodes[i].pnn,
 				ctdb_addr_to_str(&nodemap->nodes[i].addr),
 			       !!(nodemap->nodes[i].flags&NODE_FLAGS_DISCONNECTED),
 			       !!(nodemap->nodes[i].flags&NODE_FLAGS_BANNED),
@@ -658,7 +659,8 @@ static int control_status(struct ctdb_context *ctdb, int argc, const char **argv
 			       !!(nodemap->nodes[i].flags&NODE_FLAGS_UNHEALTHY),
 			       !!(nodemap->nodes[i].flags&NODE_FLAGS_STOPPED),
 			       !!(nodemap->nodes[i].flags&NODE_FLAGS_INACTIVE),
-			       partially_online);
+			       partially_online,
+			       (nodemap->nodes[i].pnn == mypnn));
 		}
 		return 0;
 	}
@@ -2983,7 +2985,7 @@ static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, db_name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
 
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
@@ -3029,7 +3031,7 @@ static int control_readkey(struct ctdb_context *ctdb, int argc, const char **arg
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, db_name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
 
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
@@ -3078,7 +3080,7 @@ static int control_writekey(struct ctdb_context *ctdb, int argc, const char **ar
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, db_name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
 
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
@@ -3142,7 +3144,7 @@ static int control_pfetch(struct ctdb_context *ctdb, int argc, const char **argv
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, db_name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
 
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
@@ -3305,8 +3307,7 @@ static int control_pstore(struct ctdb_context *ctdb, int argc, const char **argv
 
 	db_name = argv[0];
 
-	ctdb_db = ctdb_attach(ctdb, db_name, true, 0);
-
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, true, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
 		talloc_free(tmp_ctx);
@@ -3950,7 +3951,7 @@ static int control_attach(struct ctdb_context *ctdb, int argc, const char **argv
 		persistent = true;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, db_name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), db_name, persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", db_name));
 		return -1;
@@ -4158,7 +4159,7 @@ static int control_backupdb(struct ctdb_context *ctdb, int argc, const char **ar
 				     allow_unhealthy));
 	}
 
-	ctdb_db = ctdb_attach(ctdb, argv[0], dbmap->dbs[i].persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), argv[0], dbmap->dbs[i].persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", argv[0]));
 		talloc_free(tmp_ctx);
@@ -4305,7 +4306,7 @@ static int control_restoredb(struct ctdb_context *ctdb, int argc, const char **a
 		dbname, tbuf);
 
 
-	ctdb_db = ctdb_attach(ctdb, dbname, dbhdr.persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), dbname, dbhdr.persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,("Unable to attach to database '%s'\n", dbname));
 		talloc_free(tmp_ctx);
@@ -4557,7 +4558,7 @@ static int control_wipedb(struct ctdb_context *ctdb, int argc,
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, argv[0], dbmap->dbs[i].persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), argv[0], dbmap->dbs[i].persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR, ("Unable to attach to database '%s'\n",
 				  argv[0]));
@@ -5134,7 +5135,7 @@ int main(int argc, const char *argv[])
 			}
 
 			/* initialise ctdb */
-			ctdb = ctdb_cmdline_client(ev);
+			ctdb = ctdb_cmdline_client(ev, TIMELIMIT());
 
 			if (ctdb_commands[i].without_daemon == false) {
 				const char *socket_name;
diff --git a/ctdb/tools/ctdb_vacuum.c b/ctdb/tools/ctdb_vacuum.c
index bd75a69e25..e5ed9515fd 100644
--- a/ctdb/tools/ctdb_vacuum.c
+++ b/ctdb/tools/ctdb_vacuum.c
@@ -260,7 +260,7 @@ static int ctdb_vacuum_db(struct ctdb_context *ctdb, uint32_t db_id, struct ctdb
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), name, persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to attach to database '%s'\n", name));
 		talloc_free(vdata);
@@ -581,7 +581,7 @@ static int ctdb_repack_db(struct ctdb_context *ctdb, uint32_t db_id,
 		return -1;
 	}
 
-	ctdb_db = ctdb_attach(ctdb, name, persistent, 0);
+	ctdb_db = ctdb_attach(ctdb, TIMELIMIT(), name, persistent, 0);
 	if (ctdb_db == NULL) {
 		DEBUG(DEBUG_ERR,(__location__ " Failed to attach to database '%s'\n", name));
 		return -1;
diff --git a/ctdb/utils/pmda/Install b/ctdb/utils/pmda/Install
new file mode 100644
index 0000000000..a56a63506a
--- /dev/null
+++ b/ctdb/utils/pmda/Install
@@ -0,0 +1,36 @@
+#! /bin/sh
+#
+# Copyright (c) 1997 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+# Install the ctdb PMDA and/or PMNS
+#
+
+. $PCP_DIR/etc/pcp.env
+. $PCP_SHARE_DIR/lib/pmdaproc.sh
+
+iam=ctdb
+pmda_interface=2
+
+# runs as daemon and only supports pipe IPC
+daemon_opt=true
+dso_opt=false
+pipe_opt=true
+socket_opt=false
+
+pmdaSetup
+pmdaInstall
+exit 0
diff --git a/ctdb/utils/pmda/README b/ctdb/utils/pmda/README
new file mode 100644
index 0000000000..f8dbbbccac
--- /dev/null
+++ b/ctdb/utils/pmda/README
@@ -0,0 +1,84 @@
+CTDB PMDA
+===========
+
+This PMDA extracts metrics from the locally running ctdbd daemon for
+export to PMCD.
+
+Note:
+	This PMDA may be remade from source and hence requires IDO (or
+	more specifically a C compiler) to be installed.
+
+	Uses of make(1) may fail (without removing or clobbering files)
+	if the C compiler cannot be found.  This is most likely to
+	happen when running the PMDA ./Install script.
+
+	The only remedial action is to install the C compiler, or
+	hand-craft changes to the Makefile.
+
+Metrics
+=======
+
+The file ./help contains descriptions for all of the metrics exported
+by this PMDA.
+
+Once the PMDA has been installed, the following command will list all
+the available metrics and their explanatory "help" text:
+
+	$ pminfo -fT ctdb
+
+Installation
+============
+
+ +  # cd $PCP_PMDAS_DIR/ctdb
+
+ +  Check that there is no clash in the Performance Metrics Domain
+    defined in ./domain.h and the other PMDAs currently in use (see
+    $PCP_PMCDCONF_PATH).  If there is, edit ./domain.h to choose another
+    domain number.
+
+ +  Then simply use
+
+	# ./Install
+
+    and choose both the "collector" and "monitor" installation
+    configuration options.
+
+    You will be prompted to choose either a daemon implementation
+    or a DSO implementation of the PMDA, and in the case of the daemon
+    variant to select an IPC method -- everything else is automated
+
+De-installation
+===============
+
+ +  Simply use
+
+	# cd $PCP_PMDAS_DIR/ctdb
+	# ./Remove
+
+Troubleshooting
+===============
+
+ +  After installing or restarting the agent, the PMCD log file
+    ($PCP_LOG_DIR/pmcd/pmcd.log) and the PMDA log file
+    ($PCP_LOG_DIR/pmcd/pmda_ctdb.log) should be checked for any warnings
+    or errors.
+
+
+Adding a New Metric
+===================
+
+This section walks through the development task of adding a new metric to the
+CTDB PMDA.
+
+  + Define the metric in the pmns file with a unique metric id. See the pmns(4)
+    man page for details.
+
+  + Add a description of the metric to the help file.
+
+  + Taking note of the previously assigned metric id, add a new entry to the
+    metrictab structure in pmda_ctdb.c. See the pmdaInit(3) man page for
+    details.
+
+  + Ensure the counter is already a member of the ctdb_statistics structure.
+    Finally, add code to pmda_ctdb_fetch_cb() to handle fetch requests for the
+    newly defined metric.
diff --git a/ctdb/utils/pmda/Remove b/ctdb/utils/pmda/Remove
new file mode 100644
index 0000000000..7d1c509e50
--- /dev/null
+++ b/ctdb/utils/pmda/Remove
@@ -0,0 +1,29 @@
+#! /bin/sh
+#
+# Copyright (c) 1997 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+# Remove the ctdb PMDA
+#
+
+. $PCP_DIR/etc/pcp.env
+. $PCP_SHARE_DIR/lib/pmdaproc.sh
+
+iam=ctdb
+
+pmdaSetup
+pmdaRemove
+exit 0
diff --git a/ctdb/utils/pmda/domain.h b/ctdb/utils/pmda/domain.h
new file mode 100644
index 0000000000..d9b316f66c
--- /dev/null
+++ b/ctdb/utils/pmda/domain.h
@@ -0,0 +1,19 @@
+/* domain.h
+ *
+ * Copyright (c) 2004-2009 Silicon Graphics, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#define CTDB 155
diff --git a/ctdb/utils/pmda/help b/ctdb/utils/pmda/help
new file mode 100644
index 0000000000..0e9984eab3
--- /dev/null
+++ b/ctdb/utils/pmda/help
@@ -0,0 +1,106 @@
+#
+# Copyright (c) 2000-2004 Silicon Graphics, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+# ctdb PMDA help file in the ASCII format
+#
+# lines beginning with a # are ignored
+# lines beginning @ introduce a new entry of the form
+#  @ metric_name oneline-text
+#  help test goes
+#  here over multiple lines
+#  ...
+#
+# the metric_name is decoded against the default PMNS -- as a special case,
+# a name of the form NNN.MM (for numeric NNN and MM) is interpreted as an
+# instance domain identification, and the text describes the instance domain
+#
+# blank lines before the @ line are ignored
+#
+
+@ ctdb.num_clients number of clients connected to ctdbd
+
+@ ctdb.frozen whether any databases are frozen
+
+@ ctdb.recovering whether recovery is active
+
+@ ctdb.client_packets_sent number of packets sent to all clients
+
+@ ctdb.client_packets_recv number of packets received from all clients
+
+@ ctdb.node_packets_sent number of packets sent to other nodes
+
+@ ctdb.node_packets_recv number of packets received from other nodes
+
+@ ctdb.keepalive_packets_sent number of keepalive packets sent to other nodes
+
+@ ctdb.keepalive_packets_recv number of keepalive packets received from other nodes
+
+@ ctdb.node.req_call number of node CTDB_REQ_CALL packets handled
+
+@ ctdb.node.reply_call number of node CTDB_REPLY_CALL packets handled
+
+@ ctdb.node.req_dmaster number of node CTDB_REQ_DMASTER packets handled
+
+@ ctdb.node.reply_dmaster number of node CTDB_REPLY_DMASTER packets handled
+
+@ ctdb.node.reply_error number of node CTDB_REPLY_ERROR packets handled
+
+@ ctdb.node.req_message number of node CTDB_REQ_MESSAGE packets handled
+
+@ ctdb.node.req_control number of node CTDB_REQ_CONTROL packets handled
+
+@ ctdb.node.reply_control number of node CTDB_REPLY_CONTROL packets handled
+
+@ ctdb.client.req_call number of client CTDB_REQ_CALL packets handled
+
+@ ctdb.client.req_message number of client CTDB_REQ_MESSAGE packets handled
+
+@ ctdb.client.req_control number of client CTDB_REQ_CONTROL packets handled
+
+@ ctdb.timeouts.call (counter not implemented) number of call timeouts
+
+@ ctdb.timeouts.control number of node control message request timeouts awaiting reply
+
+@ ctdb.timeouts.traverse number of database traversal timeouts
+
+@ ctdb.total_calls total number of client ctdb request calls received
+
+@ ctdb.pending_calls total number of client ctdb request calls in progress
+
+@ ctdb.lockwait_calls number of tdb chainlock lockwait calls
+
+@ ctdb.pending_lockwait_calls number of lockwait calls waiting for a lock
+
+@ ctdb.childwrite_calls number of childwrite calls
+
+@ ctdb.pending_childwrite_calls number of childwrite calls in progress
+
+@ ctdb.memory_used total size of the ctdbd null talloc pool
+
+@ ctdb.max_hop_count maximum hops performed by a CTDB_REQ_CALL packet
+
+@ ctdb.max_reclock_ctdbd maximum recovery lock latency during setrecmode
+
+@ ctdb.max_reclock_recd maximum recovery lock latency as reported by the recovery process
+
+@ ctdb.max_call_latency maximum time spent handling a client request call
+
+@ ctdb.max_lockwait_latency maximum time spent waiting for a tdb chainlock
+
+@ ctdb.max_childwrite_latency maximum time spent performing a childwrite
+
+@ ctdb.num_recoveries number of recoveries finished
diff --git a/ctdb/utils/pmda/pmda_ctdb.c b/ctdb/utils/pmda/pmda_ctdb.c
new file mode 100644
index 0000000000..927fea5805
--- /dev/null
+++ b/ctdb/utils/pmda/pmda_ctdb.c
@@ -0,0 +1,596 @@
+/*
+ * CTDB Performance Metrics Domain Agent (PMDA) for Performance Co-Pilot (PCP)
+ *
+ * Copyright (c) 1995,2004 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright (c) 2011 David Disseldorp
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+
+#include <pcp/pmapi.h>
+#include <pcp/impl.h>
+#include <pcp/pmda.h>
+#include "../../include/includes.h"
+#include "../../lib/tevent/tevent.h"
+#include "../../include/ctdb.h"
+#include "../../include/ctdb_private.h"
+#include "../../include/ctdb_protocol.h"
+#include "domain.h"
+
+/*
+ * CTDB PMDA
+ *
+ * This PMDA connects to the locally running ctdbd daemon and pulls
+ * statistics for export via PCP. The ctdbd Unix domain socket path can be
+ * specified with the CTDB_SOCKET environment variable, otherwise the default
+ * path is used.
+ */
+
+/*
+ * All metrics supported in this PMDA - one table entry for each.
+ * The 4th field specifies the serial number of the instance domain
+ * for the metric, and must be either PM_INDOM_NULL (denoting a
+ * metric that only ever has a single value), or the serial number
+ * of one of the instance domains declared in the instance domain table
+ * (i.e. in indomtab, above).
+ */
+static pmdaMetric metrictab[] = {
+	/* num_clients */
+	{ NULL, { PMDA_PMID(0,0), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* frozen */
+	{ NULL, { PMDA_PMID(1,2), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* recovering */
+	{ NULL, { PMDA_PMID(3,3), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* client_packets_sent */
+	{ NULL, { PMDA_PMID(4,4), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* client_packets_recv */
+	{ NULL, { PMDA_PMID(5,5), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* node_packets_sent */
+	{ NULL, { PMDA_PMID(6,6), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* node_packets_recv */
+	{ NULL, { PMDA_PMID(7,7), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* keepalive_packets_sent */
+	{ NULL, { PMDA_PMID(8,8), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* keepalive_packets_recv */
+	{ NULL, { PMDA_PMID(9,9), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_call */
+	{ NULL, { PMDA_PMID(10,10), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* reply_call */
+	{ NULL, { PMDA_PMID(10,11), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_dmaster */
+	{ NULL, { PMDA_PMID(10,12), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* reply_dmaster */
+	{ NULL, { PMDA_PMID(10,13), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* reply_error */
+	{ NULL, { PMDA_PMID(10,14), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_message */
+	{ NULL, { PMDA_PMID(10,15), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_control */
+	{ NULL, { PMDA_PMID(10,16), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* reply_control */
+	{ NULL, { PMDA_PMID(10,17), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_call */
+	{ NULL, { PMDA_PMID(11,18), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_message */
+	{ NULL, { PMDA_PMID(11,19), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* req_control */
+	{ NULL, { PMDA_PMID(11,20), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* call */
+	{ NULL, { PMDA_PMID(12,21), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,0) }, },
+	/* control */
+	{ NULL, { PMDA_PMID(12,22), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,0) }, },
+	/* traverse */
+	{ NULL, { PMDA_PMID(12,23), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,0) }, },
+	/* total_calls */
+	{ NULL, { PMDA_PMID(13,24), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* pending_calls */
+	{ NULL, { PMDA_PMID(14,25), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* lockwait_calls */
+	{ NULL, { PMDA_PMID(15,27), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* pending_lockwait_calls */
+	{ NULL, { PMDA_PMID(16,27), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* childwrite_calls */
+	{ NULL, { PMDA_PMID(17,28), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_COUNTER,
+		PMDA_PMUNITS(0,0,1,0,0,PM_COUNT_ONE) }, },
+	/* pending_childwrite_calls */
+	{ NULL, { PMDA_PMID(18,29), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* memory_used */
+	{ NULL, { PMDA_PMID(19,30), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(1,0,0,PM_SPACE_BYTE,0,0) }, },
+	/* max_hop_count */
+	{ NULL, { PMDA_PMID(20,31), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+	/* max_reclock_ctdbd */
+	{ NULL, { PMDA_PMID(21,32), PM_TYPE_DOUBLE, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,1,0,0,PM_TIME_SEC,0) }, },
+	/* max_reclock_recd */
+	{ NULL, { PMDA_PMID(22,33), PM_TYPE_DOUBLE, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,1,0,0,PM_TIME_SEC,0) }, },
+	/* max_call_latency */
+	{ NULL, { PMDA_PMID(23,34), PM_TYPE_DOUBLE, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,1,0,0,PM_TIME_SEC,0) }, },
+	/* max_lockwait_latency */
+	{ NULL, { PMDA_PMID(24,35), PM_TYPE_DOUBLE, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,1,0,0,PM_TIME_SEC,0) }, },
+	/* max_childwrite_latency */
+	{ NULL, { PMDA_PMID(25,36), PM_TYPE_DOUBLE, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,1,0,0,PM_TIME_SEC,0) }, },
+	/* num_recoveries */
+	{ NULL, { PMDA_PMID(26,37), PM_TYPE_U32, PM_INDOM_NULL, PM_SEM_INSTANT,
+		PMDA_PMUNITS(0,0,0,0,0,0) }, },
+};
+
+static struct event_context *ev;
+static struct ctdb_context *ctdb;
+static struct ctdb_statistics *stats;
+
+static void
+pmda_ctdb_q_read_cb(uint8_t *data, size_t cnt, void *args)
+{
+	if (cnt == 0) {
+		fprintf(stderr, "ctdbd unreachable\n");
+		/* cleanup on request timeout */
+		return;
+	}
+
+	ctdb_client_read_cb(data, cnt, args);
+}
+
+
+static int
+pmda_ctdb_daemon_connect(void)
+{
+	const char *socket_name;
+	int ret;
+	struct sockaddr_un addr;
+
+	ev = event_context_init(NULL);
+	if (ev == NULL) {
+		fprintf(stderr, "Failed to init event ctx\n");
+		return -1;
+	}
+
+	ctdb = ctdb_init(ev);
+	if (ctdb == NULL) {
+		fprintf(stderr, "Failed to init ctdb\n");
+		goto err_ev;
+	}
+
+	socket_name = getenv("CTDB_SOCKET");
+	if (socket_name == NULL) {
+		socket_name = CTDB_PATH;
+	}
+
+	ret = ctdb_set_socketname(ctdb, socket_name);
+	if (ret == -1) {
+		fprintf(stderr, "ctdb_set_socketname failed - %s\n",
+				ctdb_errstr(ctdb));
+		goto err_ctdb;
+	}
+
+	/*
+	 * ctdb_socket_connect() sets a default queue callback handler that
+	 * calls exit() if ctdbd is unavailable on recv, use our own wrapper to
+	 * work around this
+	 */
+
+	memset(&addr, 0, sizeof(addr));
+	addr.sun_family = AF_UNIX;
+	strncpy(addr.sun_path, ctdb->daemon.name, sizeof(addr.sun_path));
+
+	ctdb->daemon.sd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (ctdb->daemon.sd == -1) {
+		fprintf(stderr, "Failed to open client socket\n");
+		goto err_ctdb;
+	}
+
+	set_nonblocking(ctdb->daemon.sd);
+	set_close_on_exec(ctdb->daemon.sd);
+
+	if (connect(ctdb->daemon.sd, (struct sockaddr *)&addr, sizeof(addr)) == -1) {
+		fprintf(stderr, "Failed to connect to ctdb daemon via %s\n",
+			ctdb->daemon.name);
+		goto err_sd;
+	}
+
+	ctdb->daemon.queue = ctdb_queue_setup(ctdb, ctdb, ctdb->daemon.sd,
+					      CTDB_DS_ALIGNMENT,
+					      pmda_ctdb_q_read_cb, ctdb,
+					      "to-ctdbd");
+	if (ctdb->daemon.queue == NULL) {
+		fprintf(stderr, "Failed to setup queue\n");
+		goto err_sd;
+	}
+
+	ctdb->pnn = ctdb_ctrl_getpnn(ctdb, timeval_current_ofs(3, 0),
+				     CTDB_CURRENT_NODE);
+	if (ctdb->pnn == (uint32_t)-1) {
+		fprintf(stderr, "Failed to get ctdb pnn\n");
+		goto err_sd;
+	}
+
+	return 0;
+err_sd:
+	close(ctdb->daemon.sd);
+err_ctdb:
+	talloc_free(ctdb);
+err_ev:
+	talloc_free(ev);
+	ctdb = NULL;
+	return -1;
+}
+
+static void
+pmda_ctdb_daemon_disconnect(void)
+{
+	if (ctdb->methods) {
+		ctdb->methods->shutdown(ctdb);
+	}
+
+	if (ctdb->daemon.sd != -1) {
+		close(ctdb->daemon.sd);
+	}
+
+	talloc_free(ctdb);
+	talloc_free(ev);
+	ctdb = NULL;
+}
+
+static int
+fill_node(unsigned int item, pmAtomValue *atom)
+{
+	switch (item) {
+	case 10:
+		atom->ul = stats->node.req_call;
+		break;
+	case 11:
+		atom->ul = stats->node.reply_call;
+		break;
+	case 12:
+		atom->ul = stats->node.req_dmaster;
+		break;
+	case 13:
+		atom->ul = stats->node.reply_dmaster;
+		break;
+	case 14:
+		atom->ul = stats->node.reply_error;
+		break;
+	case 15:
+		atom->ul = stats->node.req_message;
+		break;
+	case 16:
+		atom->ul = stats->node.req_control;
+		break;
+	case 17:
+		atom->ul = stats->node.reply_control;
+		break;
+	default:
+		return PM_ERR_PMID;
+	}
+
+	return 0;
+}
+
+static int
+fill_client(unsigned int item, pmAtomValue *atom)
+{
+	switch (item) {
+	case 18:
+		atom->ul = stats->client.req_call;
+		break;
+	case 19:
+		atom->ul = stats->client.req_message;
+		break;
+	case 20:
+		atom->ul = stats->client.req_control;
+		break;
+	default:
+		return PM_ERR_PMID;
+	}
+
+	return 0;
+}
+
+static int
+fill_timeout(unsigned int item, pmAtomValue *atom)
+{
+	switch (item) {
+	case 21:
+		atom->ul = stats->timeouts.call;
+		break;
+	case 22:
+		atom->ul = stats->timeouts.control;
+		break;
+	case 23:
+		atom->ul = stats->timeouts.traverse;
+		break;
+	default:
+		return PM_ERR_PMID;
+	}
+
+	return 0;
+}
+
+/*
+ * callback provided to pmdaFetch
+ */
+static int
+pmda_ctdb_fetch_cb(pmdaMetric *mdesc, unsigned int inst, pmAtomValue *atom)
+{
+	int ret;
+	__pmID_int *id = (__pmID_int *)&(mdesc->m_desc.pmid);
+
+	if (inst != PM_IN_NULL) {
+		return PM_ERR_INST;
+	}
+
+	if (stats == NULL) {
+		fprintf(stderr, "stats not available\n");
+		ret = PM_ERR_VALUE;
+		goto err_out;
+	}
+
+
+	switch (id->cluster) {
+	case 0:
+		atom->ul = stats->num_clients;
+		break;
+	case 1:
+		atom->ul = stats->frozen;
+		break;
+	case 3:
+		atom->ul = stats->recovering;
+		break;
+	case 4:
+		atom->ul = stats->client_packets_sent;
+		break;
+	case 5:
+		atom->ul = stats->client_packets_recv;
+		break;
+	case 6:
+		atom->ul = stats->node_packets_sent;
+		break;
+	case 7:
+		atom->ul = stats->node_packets_recv;
+		break;
+	case 8:
+		atom->ul = stats->keepalive_packets_sent;
+		break;
+	case 9:
+		atom->ul = stats->keepalive_packets_recv;
+		break;
+	case 10:
+		ret = fill_node(id->item, atom);
+		if (ret) {
+			goto err_out;
+		}
+		break;
+	case 11:
+		ret = fill_client(id->item, atom);
+		if (ret) {
+			goto err_out;
+		}
+		break;
+	case 12:
+		ret = fill_timeout(id->item, atom);
+		if (ret) {
+			goto err_out;
+		}
+		break;
+	case 13:
+		atom->ul = stats->total_calls;
+		break;
+	case 14:
+		atom->ul = stats->pending_calls;
+		break;
+	case 15:
+		atom->ul = stats->lockwait_calls;
+		break;
+	case 16:
+		atom->ul = stats->pending_lockwait_calls;
+		break;
+	case 17:
+		atom->ul = stats->childwrite_calls;
+		break;
+	case 18:
+		atom->ul = stats->pending_childwrite_calls;
+		break;
+	case 19:
+		atom->ul = stats->memory_used;
+		break;
+	case 20:
+		atom->ul = stats->max_hop_count;
+		break;
+	case 21:
+		atom->d = stats->reclock.ctdbd.max;
+		break;
+	case 22:
+		atom->d = stats->reclock.recd.max;
+		break;
+	case 23:
+		atom->d = stats->call_latency.max;
+		break;
+	case 24:
+		atom->d = stats->lockwait_latency.max;
+		break;
+	case 25:
+		atom->d = stats->childwrite_latency.max;
+		break;
+	case 26:
+		atom->d = stats->num_recoveries;
+		break;
+	default:
+		return PM_ERR_PMID;
+	}
+
+	ret = 0;
+err_out:
+	return ret;
+}
+
+/*
+ * This routine is called once for each pmFetch(3) operation, so is a
+ * good place to do once-per-fetch functions, such as value caching or
+ * instance domain evaluation.
+ */
+static int
+pmda_ctdb_fetch(int numpmid, pmID pmidlist[], pmResult **resp, pmdaExt *pmda)
+{
+	int ret;
+	TDB_DATA data;
+	int32_t res;
+	struct timeval ctdb_timeout;
+
+	if (ctdb == NULL) {
+		fprintf(stderr, "attempting reconnect to ctdbd\n");
+		ret = pmda_ctdb_daemon_connect();
+		if (ret < 0) {
+			fprintf(stderr, "reconnect failed\n");
+			return PM_ERR_VALUE;
+		}
+	}
+
+	ctdb_timeout = timeval_current_ofs(1, 0);
+	ret = ctdb_control(ctdb, ctdb->pnn, 0,
+			   CTDB_CONTROL_STATISTICS, 0, tdb_null,
+			   ctdb, &data, &res, &ctdb_timeout, NULL);
+
+	if (ret != 0 || res != 0) {
+		fprintf(stderr, "ctdb control for statistics failed, reconnecting\n");
+		pmda_ctdb_daemon_disconnect();
+		ret = PM_ERR_VALUE;
+		goto err_out;
+	}
+
+	stats = (struct ctdb_statistics *)data.dptr;
+
+	if (data.dsize != sizeof(struct ctdb_statistics)) {
+		fprintf(stderr, "incorrect statistics size %zu - not %zu\n",
+			data.dsize, sizeof(struct ctdb_statistics));
+		ret = PM_ERR_VALUE;
+		goto err_stats;
+	}
+
+	ret = pmdaFetch(numpmid, pmidlist, resp, pmda);
+
+err_stats:
+	talloc_free(stats);
+err_out:
+	return ret;
+}
+
+/*
+ * Initialise the agent
+ */
+void
+pmda_ctdb_init(pmdaInterface *dp)
+{
+	if (dp->status != 0) {
+		return;
+	}
+
+	dp->version.two.fetch = pmda_ctdb_fetch;
+	pmdaSetFetchCallBack(dp, pmda_ctdb_fetch_cb);
+
+	pmdaInit(dp, NULL, 0, metrictab,
+		 (sizeof(metrictab) / sizeof(metrictab[0])));
+}
+
+static char *
+helpfile(void)
+{
+	static char buf[MAXPATHLEN];
+
+	if (!buf[0]) {
+		snprintf(buf, sizeof(buf), "%s/ctdb/help",
+			 pmGetConfig("PCP_PMDAS_DIR"));
+	}
+	return buf;
+}
+
+static void
+usage(void)
+{
+	fprintf(stderr, "Usage: %s [options]\n\n", pmProgname);
+	fputs("Options:\n"
+	  "  -d domain        use domain (numeric) for metrics domain of PMDA\n"
+	  "  -l logfile       write log into logfile rather than using default log name\n"
+	  "\nExactly one of the following options may appear:\n"
+	  "  -i port          expect PMCD to connect on given inet port (number or name)\n"
+	  "  -p               expect PMCD to supply stdin/stdout (pipe)\n"
+	  "  -u socket        expect PMCD to connect on given unix domain socket\n",
+	  stderr);
+	exit(1);
+}
+
+/*
+ * Set up the agent if running as a daemon.
+ */
+int
+main(int argc, char **argv)
+{
+	int err = 0;
+	char log_file[] = "pmda_ctdb.log";
+	pmdaInterface dispatch;
+
+	__pmSetProgname(argv[0]);
+
+	pmdaDaemon(&dispatch, PMDA_INTERFACE_2, pmProgname, CTDB,
+		   log_file, helpfile());
+
+	if (pmdaGetOpt(argc, argv, "d:i:l:pu:?", &dispatch, &err) != EOF) {
+		err++;
+	}
+
+	if (err) {
+		usage();
+	}
+
+	pmdaOpenLog(&dispatch);
+	pmda_ctdb_init(&dispatch);
+	pmdaConnect(&dispatch);
+	pmdaMain(&dispatch);
+
+	exit(0);
+}
+
diff --git a/ctdb/utils/pmda/pmns b/ctdb/utils/pmda/pmns
new file mode 100644
index 0000000000..38509defb9
--- /dev/null
+++ b/ctdb/utils/pmda/pmns
@@ -0,0 +1,73 @@
+/*
+ * Metrics for CTDB PMDA
+ *
+ * Copyright (c) 2000-2004 Silicon Graphics, Inc.  All Rights Reserved.
+ * Copyright (c) 2011 David Disseldorp
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+
+ctdb {
+	num_clients		CTDB:0:0
+	frozen			CTDB:1:2
+	recovering		CTDB:3:3
+	client_packets_sent	CTDB:4:4
+	client_packets_recv	CTDB:5:5
+	node_packets_sent	CTDB:6:6
+	node_packets_recv	CTDB:7:7
+	keepalive_packets_sent	CTDB:8:8
+	keepalive_packets_recv	CTDB:9:9
+	node
+	client
+	timeouts
+	total_calls		CTDB:13:24
+	pending_calls		CTDB:14:25
+	lockwait_calls		CTDB:15:27
+	pending_lockwait_calls	CTDB:16:27
+	childwrite_calls	CTDB:17:28
+	pending_childwrite_calls CTDB:18:29
+	memory_used		CTDB:19:30
+	max_hop_count		CTDB:20:31
+	max_reclock_ctdbd	CTDB:21:32
+	max_reclock_recd	CTDB:22:33
+	max_call_latency	CTDB:23:34
+	max_lockwait_latency	CTDB:24:35
+	max_childwrite_latency	CTDB:25:36
+	num_recoveries		CTDB:26:37
+}
+
+ctdb.node {
+	req_call	CTDB:10:10
+	reply_call	CTDB:10:11
+	req_dmaster	CTDB:10:12
+	reply_dmaster	CTDB:10:13
+	reply_error	CTDB:10:14
+	req_message	CTDB:10:15
+	req_control	CTDB:10:16
+	reply_control	CTDB:10:17
+}
+
+ctdb.client {
+	req_call	CTDB:11:18
+	req_message	CTDB:11:19
+	req_control	CTDB:11:20
+}
+
+ctdb.timeouts {
+	call		CTDB:12:21
+	control		CTDB:12:22
+	traverse	CTDB:12:23
+}
+
diff --git a/ctdb/utils/pmda/root b/ctdb/utils/pmda/root
new file mode 100644
index 0000000000..ff036ed7fb
--- /dev/null
+++ b/ctdb/utils/pmda/root
@@ -0,0 +1,10 @@
+/*
+ * fake "root" for validating the local PMNS subtree
+ */
+
+#include <stdpmid>
+
+root { ctdb }
+
+#include "pmns"
+