add an in memory ringbuffer where we store the last 500000 log entries regardless of log level.

add commandt to extract this in memory buffer and to clear it

(This used to be ctdb commit 29d2ee8d9c6c6f36b2334480f646d6db209f370e)

9 files changed, 312 insertions, 3 deletions

diff --git a/ctdb/Makefile.in b/ctdb/Makefile.in
index 4b2d1007fd..df80e7f27c 100755
--- a/ctdb/Makefile.in
+++ b/ctdb/Makefile.in
@@ -39,7 +39,8 @@ UTIL_OBJ = lib/util/idtree.o lib/util/db_wrap.o lib/util/strlist.o lib/util/util
 
 CTDB_COMMON_OBJ =  common/ctdb_io.o common/ctdb_util.o \
 	common/ctdb_ltdb.o common/ctdb_message.o common/cmdline.o  \
-	lib/util/debug.o common/rb_tree.o @CTDB_SYSTEM_OBJ@ common/system_common.o
+	lib/util/debug.o common/rb_tree.o @CTDB_SYSTEM_OBJ@ common/system_common.o \
+	common/ctdb_logging.c
 
 CTDB_TCP_OBJ = tcp/tcp_connect.o tcp/tcp_io.o tcp/tcp_init.o
 
diff --git a/ctdb/common/ctdb_logging.c b/ctdb/common/ctdb_logging.c
new file mode 100644
index 0000000000..3183fe8931
--- /dev/null
+++ b/ctdb/common/ctdb_logging.c
@@ -0,0 +1,164 @@
+/* 
+   ctdb logging code
+
+   Copyright (C) Ronnie Sahlberg 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "includes.h"
+#include "lib/events/events.h"
+#include "lib/tdb/include/tdb.h"
+#include "../include/ctdb_private.h"
+#include "../include/ctdb.h"
+
+struct ctdb_log_entry {
+	int32_t level;
+	struct timeval t;
+	char *message;
+};
+
+#define MAX_LOG_ENTRIES 500000
+static int first_entry;
+static int last_entry;
+
+static struct ctdb_log_entry log_entries[MAX_LOG_ENTRIES];
+
+/*
+ * this function logs all messages for all levels to a ringbuffer
+ */
+static void log_ringbuffer_v(const char *format, va_list ap)
+{
+	int ret;
+	char *s = NULL;
+
+	if (log_entries[last_entry].message != NULL) {
+		free(log_entries[last_entry].message);
+		log_entries[last_entry].message = NULL;
+	}
+
+	ret = vasprintf(&s, format, ap);
+	if (ret == -1) {
+		return;
+	}
+
+	log_entries[last_entry].level = this_log_level;
+	log_entries[last_entry].t = timeval_current();
+	log_entries[last_entry].message = s;
+
+	last_entry++;
+	if (last_entry >= MAX_LOG_ENTRIES) {
+		last_entry = 0;
+	}
+	if (first_entry == last_entry) {
+		first_entry++;
+	}
+	if (first_entry >= MAX_LOG_ENTRIES) {
+		first_entry = 0;
+	}
+}
+
+void log_ringbuffer(const char *format, ...)
+{
+	va_list ap;
+
+	va_start(ap, format);
+	log_ringbuffer_v(format, ap);
+	va_end(ap);
+}
+
+
+
+static void ctdb_collect_log(struct ctdb_context *ctdb, struct ctdb_get_log_addr *log_addr)
+{
+	char *buf = talloc_size(NULL, 0);
+	struct ctdb_log_entry_wire *log_entry;
+	uint32_t old_size, len;
+	TDB_DATA data;
+
+	DEBUG(DEBUG_INFO,("Marshalling log entries\n"));
+	while (first_entry != last_entry) {
+		int slen = strlen(log_entries[first_entry].message);
+
+		if (log_entries[first_entry].level > log_addr->level) {
+			first_entry++;
+			if (first_entry >= MAX_LOG_ENTRIES) {
+				first_entry = 0;
+			}
+			continue;
+		}
+
+		len = offsetof(struct ctdb_log_entry_wire, message) + slen + 1;
+		/* pad it to uint42 */
+		len = (len+3)&0xfffffffc;
+
+		old_size = talloc_get_size(buf);
+		buf = talloc_realloc_size(NULL, buf, old_size + len);
+
+		log_entry = (struct ctdb_log_entry_wire *)&buf[old_size];
+		log_entry->level       = log_entries[first_entry].level;
+		log_entry->t           = log_entries[first_entry].t;
+		log_entry->message_len = slen;
+		memcpy(log_entry->message, log_entries[first_entry].message, slen);
+		log_entry->message[slen] = 0;
+
+		first_entry++;
+		if (first_entry >= MAX_LOG_ENTRIES) {
+			first_entry = 0;
+		}
+	}
+
+	data.dptr  = (uint8_t *)buf;
+	data.dsize = talloc_get_size(buf);
+	DEBUG(DEBUG_INFO,("Marshalling log entries into a blob of %d bytes\n", (int)data.dsize));
+
+	DEBUG(DEBUG_INFO,("Send log to %d:%d\n", (int)log_addr->pnn, (int)log_addr->srvid));
+	ctdb_send_message(ctdb, log_addr->pnn, log_addr->srvid, data);
+}
+
+int32_t ctdb_control_get_log(struct ctdb_context *ctdb, TDB_DATA addr)
+{
+	struct ctdb_get_log_addr *log_addr = (struct ctdb_get_log_addr *)addr.dptr;
+	pid_t child;
+
+	/* spawn a child process to marshall the huge log blob and send it back
+	   to the ctdb tool using a MESSAGE
+	*/
+	child = fork();
+	if (child == (pid_t)-1) {
+		DEBUG(DEBUG_ERR,("Failed to fork a log collector child\n"));
+		return -1;
+	}
+
+	if (child == 0) {
+		if (switch_from_server_to_client(ctdb) != 0) {
+			DEBUG(DEBUG_CRIT, (__location__ "ERROR: failed to switch log collector child into client mode.\n"));
+			_exit(1);
+		}
+		ctdb_collect_log(ctdb, log_addr);
+		_exit(0);
+	}
+
+	return 0;
+}
+
+
+int32_t ctdb_control_clear_log(struct ctdb_context *ctdb)
+{
+	first_entry = 0;
+	last_entry  = 0;
+
+	return 0;
+}
+
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index 3fcfdfa84f..48fb29bd0a 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -622,6 +622,8 @@ enum ctdb_controls {CTDB_CONTROL_PROCESS_EXISTS          = 0,
 		    CTDB_CONTROL_REGISTER_NOTIFY         = 114,
 		    CTDB_CONTROL_DEREGISTER_NOTIFY       = 115,
 		    CTDB_CONTROL_TRANS2_ACTIVE           = 116,
+		    CTDB_CONTROL_GET_LOG		 = 117,
+		    CTDB_CONTROL_CLEAR_LOG		 = 118,
 };	
 
 /*
@@ -1520,4 +1522,22 @@ int32_t ctdb_control_deregister_notify(struct ctdb_context *ctdb, uint32_t clien
 
 int start_syslog_daemon(struct ctdb_context *ctdb);
 
+/* Where to send the log messages back to */
+struct ctdb_get_log_addr {
+	uint32_t pnn;
+	uint64_t srvid;
+	int32_t level;
+};
+
+/* wire data for log entries, padded to uint32 */
+struct ctdb_log_entry_wire {
+	int32_t level;
+	struct timeval t;
+	int32_t message_len;
+	char message[1];
+};
+
+int32_t ctdb_control_get_log(struct ctdb_context *ctdb, TDB_DATA addr);
+int32_t ctdb_control_clear_log(struct ctdb_context *ctdb);
+
 #endif
diff --git a/ctdb/include/includes.h b/ctdb/include/includes.h
index 744d117a1a..4b4d2428ef 100644
--- a/ctdb/include/includes.h
+++ b/ctdb/include/includes.h
@@ -28,7 +28,7 @@ enum debug_level {
 	DEBUG_DEBUG   =  4,
 };
 
-#define DEBUG(lvl, x) do { if ((lvl) <= LogLevel) { this_log_level = (lvl); do_debug x; }} while (0)
+#define DEBUG(lvl, x) do { this_log_level = (lvl); log_ringbuffer x; if ((lvl) <= LogLevel) { do_debug x; }} while (0)
 
 #define _PUBLIC_
 
diff --git a/ctdb/lib/util/debug.c b/ctdb/lib/util/debug.c
index 63ca03b5a7..6597570348 100644
--- a/ctdb/lib/util/debug.c
+++ b/ctdb/lib/util/debug.c
@@ -90,7 +90,7 @@ void do_debug_add(const char *format, ...)
 }
 
 #define DEBUGLVL(lvl) ((lvl) <= LogLevel)
-#define DEBUG(lvl, x) do { if ((lvl) <= LogLevel) { this_log_level = (lvl); do_debug x; }} while (0)
+#define DEBUG(lvl, x) do { this_log_level = (lvl); log_ringbuffer x; if ((lvl) <= LogLevel) { do_debug x; }} while (0)
 #define DEBUGADD(lvl, x) do { if ((lvl) <= LogLevel) { this_log_level = (lvl); do_debug_add x; }} while (0)
 
 static void print_asc(int level, const uint8_t *buf, size_t len)
diff --git a/ctdb/lib/util/debug.h b/ctdb/lib/util/debug.h
index a7d89785bb..d91f43b4b4 100644
--- a/ctdb/lib/util/debug.h
+++ b/ctdb/lib/util/debug.h
@@ -19,5 +19,6 @@
 
 void (*do_debug_v)(const char *, va_list ap);
 void (*do_debug_add_v)(const char *, va_list ap);
+void log_ringbuffer(const char *format, ...);
 void do_debug(const char *format, ...) PRINTF_ATTRIBUTE(1, 2);
 void dump_data(int level, const uint8_t *buf1, size_t len);
diff --git a/ctdb/server/ctdb_control.c b/ctdb/server/ctdb_control.c
index bf82e515fb..168e1b5519 100644
--- a/ctdb/server/ctdb_control.c
+++ b/ctdb/server/ctdb_control.c
@@ -564,6 +564,13 @@ static int32_t ctdb_control_dispatch(struct ctdb_context *ctdb,
 		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_client_notify_deregister));
 		return ctdb_control_deregister_notify(ctdb, client_id, indata);
 
+	case CTDB_CONTROL_GET_LOG:
+		CHECK_CONTROL_DATA_SIZE(sizeof(struct ctdb_get_log_addr));
+		return ctdb_control_get_log(ctdb, indata);
+
+	case CTDB_CONTROL_CLEAR_LOG:
+		return ctdb_control_clear_log(ctdb);
+
 	default:
 		DEBUG(DEBUG_CRIT,(__location__ " Unknown CTDB control opcode %u\n", opcode));
 		return -1;
diff --git a/ctdb/server/ctdb_logging.c b/ctdb/server/ctdb_logging.c
index 16f728bd0f..5bb39e394d 100644
--- a/ctdb/server/ctdb_logging.c
+++ b/ctdb/server/ctdb_logging.c
@@ -448,3 +448,8 @@ int ctdb_set_child_logging(struct ctdb_context *ctdb)
 
 	return 0;
 }
+
+
+
+
+	
diff --git a/ctdb/tools/ctdb.c b/ctdb/tools/ctdb.c
index 838efdb7d8..e89640a67d 100644
--- a/ctdb/tools/ctdb.c
+++ b/ctdb/tools/ctdb.c
@@ -2354,6 +2354,115 @@ static int control_catdb(struct ctdb_context *ctdb, int argc, const char **argv)
 }
 
 
+static void log_handler(struct ctdb_context *ctdb, uint64_t srvid, 
+			     TDB_DATA data, void *private_data)
+{
+	struct ctdb_log_entry_wire *entry;
+	int size;
+	struct tm *tm;
+	char tbuf[100];
+
+	DEBUG(DEBUG_ERR,("Log data received\n"));
+	while (data.dsize > 0) {
+		entry = (struct ctdb_log_entry_wire *)data.dptr;
+		size = offsetof(struct ctdb_log_entry_wire, message) + entry->message_len + 1;
+		size = (size+3)&0xfffffffc;
+
+		tm = localtime(&entry->t.tv_sec);
+
+		strftime(tbuf,sizeof(tbuf)-1,"%Y/%m/%d %H:%M:%S", tm);
+
+		printf("%s:%s %s", tbuf, get_debug_by_level(entry->level), entry->message);
+
+		data.dsize -= size;
+		data.dptr  += size;
+	}
+	exit(0);
+}
+
+/*
+  display a list of log messages from the in memory ringbuffer
+ */
+static int control_getlog(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	int32_t res;
+	struct ctdb_get_log_addr log_addr;
+	TDB_DATA data;
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+	char *errmsg;
+	struct timeval tv;
+
+	if (argc != 1) {
+		DEBUG(DEBUG_ERR,("Invalid arguments\n"));
+		talloc_free(tmp_ctx);
+		return -1;
+	}
+
+	log_addr.pnn = ctdb_ctrl_getpnn(ctdb, TIMELIMIT(), CTDB_CURRENT_NODE);
+	log_addr.srvid = getpid();
+	if (isalpha(argv[0][0]) || argv[0][0] == '-') { 
+		log_addr.level = get_debug_by_desc(argv[0]);
+	} else {
+		log_addr.level = strtol(argv[0], NULL, 0);
+	}
+
+
+	data.dptr = (unsigned char *)&log_addr;
+	data.dsize = sizeof(log_addr);
+
+	DEBUG(DEBUG_ERR, ("Pulling logs from node %u\n", options.pnn));
+
+	ctdb_set_message_handler(ctdb, log_addr.srvid, log_handler, NULL);
+	sleep(1);
+
+	DEBUG(DEBUG_ERR,("Listen for response on %d\n", (int)log_addr.srvid));
+
+	ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_GET_LOG,
+			   0, data, tmp_ctx, NULL, &res, NULL, &errmsg);
+	if (ret != 0 || res != 0) {
+		DEBUG(DEBUG_ERR,("Failed to get logs - %s\n", errmsg));
+		talloc_free(tmp_ctx);
+		return -1;
+	}
+
+
+	tv = timeval_current();
+	/* this loop will terminate when we have received the reply */
+	while (timeval_elapsed(&tv) < 3.0) {	
+		event_loop_once(ctdb->ev);
+	}
+
+	DEBUG(DEBUG_INFO,("Timed out waiting for log data.\n"));
+
+	talloc_free(tmp_ctx);
+	return 0;
+}
+
+/*
+  clear the in memory log area
+ */
+static int control_clearlog(struct ctdb_context *ctdb, int argc, const char **argv)
+{
+	int ret;
+	int32_t res;
+	char *errmsg;
+	TALLOC_CTX *tmp_ctx = talloc_new(ctdb);
+
+	ret = ctdb_control(ctdb, options.pnn, 0, CTDB_CONTROL_CLEAR_LOG,
+			   0, tdb_null, tmp_ctx, NULL, &res, NULL, &errmsg);
+	if (ret != 0 || res != 0) {
+		DEBUG(DEBUG_ERR,("Failed to clear logs\n"));
+		talloc_free(tmp_ctx);
+		return -1;
+	}
+
+	talloc_free(tmp_ctx);
+	return 0;
+}
+
+
+
 /*
   display a list of the databases on a remote ctdb
  */
@@ -3460,6 +3569,8 @@ static const struct {
 	{ "enablemonitor",      control_enable_monmode, true,	false,  "set monitoring mode to ACTIVE" },
 	{ "setdebug",        control_setdebug,          true,	false,  "set debug level",                      "<EMERG|ALERT|CRIT|ERR|WARNING|NOTICE|INFO|DEBUG>" },
 	{ "getdebug",        control_getdebug,          true,	false,  "get debug level" },
+	{ "getlog",          control_getlog,            true,	false,  "get the log data from the in memory ringbuffer", "<level>" },
+	{ "clearlog",          control_clearlog,        true,	false,  "clear the log data from the in memory ringbuffer" },
 	{ "attach",          control_attach,            true,	false,  "attach to a database",                 "<dbname>" },
 	{ "dumpmemory",      control_dumpmemory,        true,	false,  "dump memory map to stdout" },
 	{ "rddumpmemory",    control_rddumpmemory,      true,	false,  "dump memory map from the recovery daemon to stdout" },