ReadOnly: add per-database statistics to view how much delegations/revokes we have

(This used to be ctdb commit 751ed46197661eb841042ab6a02855a51dd0b17c)

11 files changed, 237 insertions, 0 deletions

diff --git a/ctdb/include/ctdb.h b/ctdb/include/ctdb.h
index 8dbdd47e1e..93224cbdaa 100644
--- a/ctdb/include/ctdb.h
+++ b/ctdb/include/ctdb.h
@@ -510,6 +510,39 @@ bool ctdb_getpnn_recv(struct ctdb_connection *ctdb,
 
 
 /**
+ * ctdb_getdbstat_send - read statistics for a db
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @db_id:    the database to collect the statistics from
+ * @callback: the callback when ctdb replies to our message (typesafe)
+ * @cbdata: the argument to callback()
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ */
+struct ctdb_request *
+ctdb_getdbstat_send(struct ctdb_connection *ctdb,
+		     uint32_t destnode,
+		     uint32_t db_id,
+		     ctdb_callback_t callback,
+		     void *cbdata);
+/**
+ * ctdb_getdbstat_recv - read an ctdb_getdbstat reply from ctdbd
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @req: the completed request.
+ * @stat: a pointer to the *stat to fill in
+ *
+ * This returns false if something went wrong, or otherwise fills in **stats
+ * stats must be freed later by calling ctdb_free_dbstat();
+ */
+bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
+			 struct ctdb_request *req,
+			 struct ctdb_db_statistics **stat);
+
+void ctdb_free_dbstat(struct ctdb_db_statistics *stat);
+
+/**
  * ctdb_check_message_handlers_send - check a list of message_handlers
  * if they are registered
  * message_handlers are registered on the daemon using the
@@ -876,6 +909,26 @@ bool ctdb_getpnn(struct ctdb_connection *ctdb,
 		 uint32_t *pnn);
 
 /**
+ * ctdb_getdbstat - read the db stat of a node (synchronous)
+ * @ctdb: the ctdb_connection from ctdb_connect.
+ * @destnode: the destination node (see below)
+ * @db_id:    the database to collect the statistics from
+ * @stat: a pointer to the *stat to fill in
+ *
+ * There are several special values for destnode, detailed in
+ * ctdb_protocol.h, particularly CTDB_CURRENT_NODE which means the
+ * local ctdbd.
+ *
+ * This returns false if something went wrong, or otherwise fills in **stat
+ * stat must be freed later by calling ctdb_free_dbstat();
+ */
+bool ctdb_getdbstat(struct ctdb_connection *ctdb,
+		    uint32_t destnode,
+		    uint32_t db_id,
+		    struct ctdb_db_statistics **stat);
+
+
+/**
  * ctdb_check_message_handlers - check a list of message_handlers (synchronous)
  * @ctdb: the ctdb_connection from ctdb_connect.
  * @destnode: the destination node (see below)
@@ -1092,6 +1145,10 @@ void ctdb_free_vnnmap(struct ctdb_vnn_map *vnnmap);
 	ctdb_getpnn_send((ctdb), (destnode),				\
 			 ctdb_sendcb((cb), (cbdata)), (cbdata))
 
+#define ctdb_getdbstat_send(ctdb, destnode, db_id, cb, cbdata)		\
+	ctdb_getdbstat_send((ctdb), (destnode), (db_id),		\
+			    ctdb_sendcb((cb), (cbdata)), (cbdata))
+
 #define ctdb_check_message_handlers_send(ctdb, destnode, num, mhs,	\
 			 cb, cbdata)					\
 	ctdb_check_message_handlers_send((ctdb), (destnode), (num), 	\
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index f58356fbe7..f420e6c341 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -300,6 +300,11 @@ struct ctdb_daemon_data {
 		ctdb->statistics_current.counter++;					\
 	}
 
+#define CTDB_INCREMENT_DB_STAT(ctdb_db, counter) \
+	{										\
+		ctdb_db->statistics.counter++;						\
+	}
+
 #define CTDB_DECREMENT_STAT(ctdb, counter) \
 	{										\
 		if (ctdb->statistics.counter > 0)					\
@@ -513,6 +518,8 @@ struct ctdb_db_context {
 	   so we can avoid sending duplicate fetch requests
 	*/
 	struct trbt_tree *deferred_fetch;
+
+	struct ctdb_db_statistics statistics;
 };
 
 
@@ -1440,4 +1447,8 @@ int ctdb_fetch_func(struct ctdb_call_info *call);
 
 int ctdb_fetch_with_header_func(struct ctdb_call_info *call);
 
+int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
+				uint32_t db_id,
+				TDB_DATA *outdata);
+
 #endif
diff --git a/ctdb/include/ctdb_protocol.h b/ctdb/include/ctdb_protocol.h
index 9cdf91c8d4..4cf27893ff 100644
--- a/ctdb/include/ctdb_protocol.h
+++ b/ctdb/include/ctdb_protocol.h
@@ -377,6 +377,7 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0,
 		    CTDB_CONTROL_SET_DB_READONLY	 = 129,
 		    CTDB_CONTROL_CHECK_SRVIDS		 = 130,
 		    CTDB_CONTROL_TRAVERSE_START_EXT	 = 131,
+		    CTDB_CONTROL_GET_DB_STATISTICS	 = 132,
 };
 
 /*
@@ -659,6 +660,14 @@ struct ctdb_statistics_wire {
 };
 
 /*
+ * db statistics
+ */
+struct ctdb_db_statistics {
+	uint32_t db_ro_delegations;
+	uint32_t db_ro_revokes;
+};
+
+/*
  * wire format for interface list
  */
 #ifdef IFNAMSIZ
diff --git a/ctdb/libctdb/control.c b/ctdb/libctdb/control.c
index ba22444ee5..8741ad9c0b 100644
--- a/ctdb/libctdb/control.c
+++ b/ctdb/libctdb/control.c
@@ -25,6 +25,7 @@
 #undef ctdb_getrecmaster_send
 #undef ctdb_getrecmode_send
 #undef ctdb_getpnn_send
+#undef ctdb_getdbstat_send
 #undef ctdb_check_message_handlers_send
 #undef ctdb_getnodemap_send
 #undef ctdb_getpublicips_send
@@ -112,6 +113,56 @@ struct ctdb_request *ctdb_getpnn_send(struct ctdb_connection *ctdb,
 					NULL, 0, callback, private_data);
 }
 
+bool ctdb_getdbstat_recv(struct ctdb_connection *ctdb,
+			 struct ctdb_request *req,
+			 struct ctdb_db_statistics **stat)
+{
+	struct ctdb_reply_control *reply;
+	struct ctdb_db_statistics *s;
+
+	reply = unpack_reply_control(req, CTDB_CONTROL_GET_DB_STATISTICS);
+	if (!reply) {
+		return false;
+	}
+	if (reply->status == -1) {
+		DEBUG(ctdb, LOG_ERR, "ctdb_getpnn_recv: status -1");
+		return false;
+	}
+	if (reply->datalen != sizeof(struct ctdb_db_statistics)) {
+		DEBUG(ctdb, LOG_ERR, "ctdb_getdbstat_recv: returned data is %d bytes but should be %d", reply->datalen, (int)sizeof(struct ctdb_db_statistics));
+		return false;
+	}
+
+	s = malloc(sizeof(struct ctdb_db_statistics));
+	if (!s) {
+		return false;
+	}
+	memcpy(s, reply->data, sizeof(struct ctdb_db_statistics));
+	*stat = s;
+
+	return true;
+}
+
+struct ctdb_request *ctdb_getdbstat_send(struct ctdb_connection *ctdb,
+				      uint32_t destnode,
+				      uint32_t db_id,
+				      ctdb_callback_t callback,
+				      void *private_data)
+{
+	uint32_t indata = db_id;
+
+	return new_ctdb_control_request(ctdb, CTDB_CONTROL_GET_DB_STATISTICS, destnode,
+					&indata, sizeof(indata), callback, private_data);
+}
+
+void ctdb_free_dbstat(struct ctdb_db_statistics *stat)
+{
+	if (stat == NULL) {
+		return;
+	}
+	free(stat);
+}
+
 bool ctdb_getnodemap_recv(struct ctdb_connection *ctdb,
 		      struct ctdb_request *req, struct ctdb_node_map **nodemap)
 {
diff --git a/ctdb/libctdb/sync.c b/ctdb/libctdb/sync.c
index f2b6957078..4b68cd1e32 100644
--- a/ctdb/libctdb/sync.c
+++ b/ctdb/libctdb/sync.c
@@ -136,6 +136,24 @@ bool ctdb_getpnn(struct ctdb_connection *ctdb,
 	return ret;
 }
 
+bool ctdb_getdbstat(struct ctdb_connection *ctdb,
+		    uint32_t destnode, uint32_t db_id,
+		    struct ctdb_db_statistics **stat)
+{
+	struct ctdb_request *req;
+	bool done = false;
+	bool ret = false;
+
+	req = synchronous(ctdb,
+			  ctdb_getdbstat_send(ctdb, destnode, db_id, set, &done),
+			  &done);
+	if (req != NULL) {
+		ret = ctdb_getdbstat_recv(ctdb, req, stat);
+		ctdb_request_free(req);
+	}
+	return ret;
+}
+
 bool ctdb_check_message_handlers(struct ctdb_connection *ctdb,
 		      uint32_t destnode, uint32_t num,
 		      uint64_t *mhs, uint8_t *result)
diff --git a/ctdb/server/ctdb_call.c b/ctdb/server/ctdb_call.c
index 8856f2ac28..9922233690 100644
--- a/ctdb/server/ctdb_call.c
+++ b/ctdb/server/ctdb_call.c
@@ -514,6 +514,7 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 	if (header.flags & CTDB_REC_RO_REVOKE_COMPLETE) {
 		header.flags &= ~(CTDB_REC_RO_HAVE_DELEGATIONS|CTDB_REC_RO_HAVE_READONLY|CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_REVOKE_COMPLETE);
 		CTDB_INCREMENT_STAT(ctdb, total_ro_revokes);
+		CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_revokes);
 		if (ctdb_ltdb_store(ctdb_db, call->key, &header, data) != 0) {
 			ctdb_fatal(ctdb, "Failed to write header with cleared REVOKE flag");
 		}
@@ -621,6 +622,7 @@ void ctdb_request_call(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 
 		ctdb_queue_packet(ctdb, &r->hdr);
 		CTDB_INCREMENT_STAT(ctdb, total_ro_delegations);
+		CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_delegations);
 
 		talloc_free(r);
 		return;
diff --git a/ctdb/server/ctdb_control.c b/ctdb/server/ctdb_control.c
index 060f2c5cf3..c73458825c 100644
--- a/ctdb/server/ctdb_control.c
+++ b/ctdb/server/ctdb_control.c
@@ -630,6 +630,10 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 		CHECK_CONTROL_DATA_SIZE(size);
 		return ctdb_control_schedule_for_deletion(ctdb, indata);
 	}
+	case CTDB_CONTROL_GET_DB_STATISTICS:
+		CHECK_CONTROL_DATA_SIZE(sizeof(uint32_t));
+		return ctdb_control_get_db_statistics(ctdb, *(uint32_t *)indata.dptr, outdata);
+
 	default:
 		DEBUG(DEBUG_CRIT,(__location__ " Unknown CTDB control opcode %u\n", opcode));
 		return -1;
diff --git a/ctdb/server/ctdb_daemon.c b/ctdb/server/ctdb_daemon.c
index bff91c5fa6..2436997864 100644
--- a/ctdb/server/ctdb_daemon.c
+++ b/ctdb/server/ctdb_daemon.c
@@ -691,6 +691,7 @@ static void daemon_request_call_from_client(struct ctdb_client *client,
 	if (header.flags & CTDB_REC_RO_REVOKE_COMPLETE) {
 		header.flags &= ~(CTDB_REC_RO_HAVE_DELEGATIONS|CTDB_REC_RO_HAVE_READONLY|CTDB_REC_RO_REVOKING_READONLY|CTDB_REC_RO_REVOKE_COMPLETE);
 		CTDB_INCREMENT_STAT(ctdb, total_ro_revokes);
+		CTDB_INCREMENT_DB_STAT(ctdb_db, db_ro_revokes);
 		if (ctdb_ltdb_store(ctdb_db, key, &header, data) != 0) {
 			ctdb_fatal(ctdb, "Failed to write header with cleared REVOKE flag");
 		}
diff --git a/ctdb/server/ctdb_ltdb_server.c b/ctdb/server/ctdb_ltdb_server.c
index 9a0132a362..e5437b99a2 100644
--- a/ctdb/server/ctdb_ltdb_server.c
+++ b/ctdb/server/ctdb_ltdb_server.c
@@ -1464,3 +1464,20 @@ int32_t ctdb_control_set_db_priority(struct ctdb_context *ctdb, TDB_DATA indata)
 	return 0;
 }
 
+int32_t ctdb_control_get_db_statistics(struct ctdb_context *ctdb,
+				uint32_t db_id,
+				TDB_DATA *outdata)
+{
+	struct ctdb_db_context *ctdb_db;
+
+	ctdb_db = find_ctdb_db(ctdb, db_id);
+	if (!ctdb_db) {
+		DEBUG(DEBUG_ERR,("Unknown db_id 0x%x in get_db_statistics\n", db_id));
+		return -1;
+	}
+
+	outdata->dptr  = (uint8_t *)&(ctdb_db->statistics);
+	outdata->dsize = sizeof(ctdb_db->statistics);
+
+	return 0;
+}
diff --git a/ctdb/tests/src/libctdb_test.c b/ctdb/tests/src/libctdb_test.c
index 73ba155a29..56c007e8e3 100644
--- a/ctdb/tests/src/libctdb_test.c
+++ b/ctdb/tests/src/libctdb_test.c
@@ -487,3 +487,20 @@ void ctdb_disconnect(struct ctdb_connection *ctdb)
 	free(ctdb);
 }
 
+bool ctdb_getdbstat(struct ctdb_connection *ctdb,
+		    uint32_t destnode,
+		    uint32_t db_id,
+		    struct ctdb_db_statistics **dbstatistics)
+{
+	if (!current_node_is_connected(ctdb)) {
+		return false;
+	}
+
+	*dbstatistics = malloc(sizeof(struct ctdb_db_statistics));
+	return true;
+}
+
+void ctdb_free_dbstat(struct ctdb_db_statistics *dbstatistics)
+{
+	free(dbstatistics);
+}
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index ceab2979ff..051a270d6e 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -566,6 +566,55 @@ static int control_stats(struct ctdb_context *ctdb, int argc, const char **argv)
 
 
 /*
+  display remote ctdb db statistics
+ */
+static int control_dbstatistics(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+	struct ctdb_db_statistics *dbstatistics;
+	struct ctdb_dbid_map *dbmap=NULL;
+	int i, ret;
+
+	if (argc < 1) {
+		usage();
+	}
+
+	ret = ctdb_ctrl_getdbmap(ctdb, TIMELIMIT(), options.pnn, tmp_ctx, &dbmap);
+	if (ret != 0) {
+		DEBUG(DEBUG_ERR, ("Unable to get dbids from node %u\n", options.pnn));
+		return ret;
+	}
+	for(i=0;i<dbmap->num;i++){
+		const char *name;
+
+		ctdb_ctrl_getdbname(ctdb, TIMELIMIT(), options.pnn, dbmap->dbs[i].dbid, tmp_ctx, &name);
+		if(!strcmp(argv[0], name)){
+			talloc_free(discard_const(name));
+			break;
+		}
+		talloc_free(discard_const(name));
+	}
+	if (i == dbmap->num) {
+		DEBUG(DEBUG_ERR,("No database with name '%s' found\n", argv[0]));
+		talloc_free(tmp_ctx);
+		return -1;
+	}
+
+	if (!ctdb_getdbstat(ctdb_connection, options.pnn, dbmap->dbs[i].dbid, &dbstatistics)) {
+		DEBUG(DEBUG_ERR,("Failed to read db statistics from node\n"));
+		talloc_free(tmp_ctx);
+		return -1;
+	}
+
+	printf("DB Statistics:\n");
+	printf("RO Delegations: %d\n", dbstatistics->db_ro_delegations);
+	printf("RO Revokes:     %d\n", dbstatistics->db_ro_revokes);
+
+	ctdb_free_dbstat(dbstatistics);
+	return 0;
+}
+
+/*
   display uptime of remote node
  */
 static int control_uptime(struct ctdb_context *ctdb, int argc, const char **argv)
@@ -5434,6 +5483,7 @@ static const struct {
 	{ "checktcpport",    control_chktcpport,      	false,	true,  "check if a service is bound to a specific tcp port or not", "<port>" },
 	{ "getdbseqnum",     control_getdbseqnum,       false,	false, "get the sequence number off a database", "<dbid>" },
 	{ "nodestatus",      control_nodestatus,        true,   false,  "show and return node status" },
+	{ "dbstatistics",    control_dbstatistics,      false,	false, "show db statistics", "<db>" },
 };
 
 /*