2006-09-18 Martin Hunt <hunt@redhat.com>

	* print.c (_stp_print_flush): Rewrite so one version works for
	relayfs or procfs. Use proper per-cpu functions.
	(_stp_reserve_bytes): New function. Reserve bytes in the output buffer.
	(_stp_print_binary): New function. Write a variable number of
	64-bit values directly into the output buffer.

	* string.c (_stp_sprintf): Rewrite using new per-cpu buffers.
	(_stp_vsprintf): Ditto.
	(_stp_string_cat_cstr): Ditto.
	(_stp_string_cat_char): Ditto.

	* runtime.h: Set defaults for MAXTRYLOCK and TRYLOCKDELAY to make
	runtime tests in bench2 happy.

4 files changed, 155 insertions, 88 deletions

diff --git a/runtime/ChangeLog b/runtime/ChangeLog
index d830d1a7..74b54860 100644
--- a/runtime/ChangeLog
+++ b/runtime/ChangeLog
@@ -1,3 +1,19 @@
+2006-09-18  Martin Hunt  <hunt@redhat.com>
+
+	* print.c (_stp_print_flush): Rewrite so one version works for
+	relayfs or procfs. Use proper per-cpu functions.
+	(_stp_reserve_bytes): New function. Reserve bytes in the output buffer.
+	(_stp_print_binary): New function. Write a variable number of
+	64-bit values directly into the output buffer.
+
+	* string.c (_stp_sprintf): Rewrite using new per-cpu buffers.
+	(_stp_vsprintf): Ditto.
+	(_stp_string_cat_cstr): Ditto.
+	(_stp_string_cat_char): Ditto.	
+
+	* runtime.h: Set defaults for MAXTRYLOCK and TRYLOCKDELAY to make 
+	runtime tests in bench2 happy.
+	
 2006-08-29  Li Guanglei <guanglei@cn.ibm.com>
 
 	From Gui Jian <guijian@cn.ibm.com> 
diff --git a/runtime/print.c b/runtime/print.c
index f01e27e9..65b2d37f 100644
--- a/runtime/print.c
+++ b/runtime/print.c
@@ -33,85 +33,119 @@
  * @{
  */
 
-static int _stp_pbuf_len[NR_CPUS];
+#ifdef STP_RELAYFS
+#define STP_TIMESTAMP_SIZE (sizeof(uint32_t))
+#else
+#define STP_TIMESTAMP_SIZE 0
+#endif /* STP_RELAYFS */
 
-#ifndef STP_RELAYFS
-#define STP_PRINT_BUF_START 0
 
-/** Size of buffer, not including terminating NULL */
+#define STP_PRINT_BUF_START (STP_TIMESTAMP_SIZE)
 #ifndef STP_PRINT_BUF_LEN
-#define STP_PRINT_BUF_LEN 8191
+#define STP_PRINT_BUF_LEN 8192
 #endif
 
-static char _stp_pbuf[NR_CPUS][STP_PRINT_BUF_LEN + 1];
+typedef struct __stp_pbuf {
+	uint32_t len;			/* bytes used in the buffer */
+	char timestamp[STP_TIMESTAMP_SIZE];
+	char buf[STP_PRINT_BUF_LEN];
+} _stp_pbuf;
+
+DEFINE_PER_CPU(_stp_pbuf, Stp_pbuf);
 
+
+/** Send the print buffer to the transport now.
+ * Output accumulates in the print buffer until it
+ * is filled, or this is called. This MUST be called before returning
+ * from a probe or accumulated output in the print buffer will be lost.
+ *
+ * @note Preemption must be disabled to use this.
+ */
 void _stp_print_flush (void)
 {
-	int cpu = smp_processor_id();
-	char *buf = &_stp_pbuf[cpu][0];
-	int len = _stp_pbuf_len[cpu];
-	int ret;
+	_stp_pbuf *pb = &__get_cpu_var(Stp_pbuf);
 
-	if (len == 0)
+	/* check to see if there is anything in the buffer */
+	if (pb->len == 0)
 		return;
 
-	ret =_stp_transport_write(buf, len);
-	if (unlikely(ret < 0)) {
-#if 0
-		if (!atomic_read(&_stp_transport_failures))
-			_stp_warn("Transport failure - try using a larger buffer size\n");
-#endif
-		atomic_inc (&_stp_transport_failures);
-	}
+#ifdef STP_RELAYFS_MERGE
+	/* In merge-mode, stpd expects relayfs data to start with a 4-byte length */
+	/* followed by a 4-byte sequence number. In non-merge mode, anything goes. */
 
-	_stp_pbuf_len[cpu] = 0;
-	*buf = 0;
-}
+	*((uint32_t *)pb->timestamp) = _stp_seq_inc();
 
-#else /* STP_RELAYFS */
+	if (unlikely(_stp_transport_write(pb, pb->len+4+STP_TIMESTAMP_SIZE) < 0))
+		atomic_inc (&_stp_transport_failures);
+#else
+	if (unlikely(_stp_transport_write(pb->buf, pb->len) < 0))
+		atomic_inc (&_stp_transport_failures);
+#endif
 
-/* size of timestamp, in bytes, including space */
-#define STP_TIMESTAMP_SIZE (sizeof(int))
-#define STP_PRINT_BUF_START (STP_TIMESTAMP_SIZE)
+	pb->len = 0;
+}
 
-/** Size of buffer, not including terminating NULL */
-#ifndef STP_PRINT_BUF_LEN
-#define STP_PRINT_BUF_LEN (8192 - STP_TIMESTAMP_SIZE - 1)
+#ifndef STP_MAXBINARYARGS
+#define STP_MAXBINARYARGS 127
 #endif
 
-static char _stp_pbuf[NR_CPUS][STP_PRINT_BUF_LEN + STP_PRINT_BUF_START + 1];
 
-/** Send the print buffer to the transport now.
- * Output accumulates in the print buffer until it
- * is filled, or this is called. This MUST be called before returning
- * from a probe or accumulated output in the print buffer will be lost.
+/** Reserves space in the output buffer for direct I/O.
  */
 
-void _stp_print_flush (void)
+#if defined STP_RELAYFS && !defined STP_RELAYFS_MERGE
+static void * _stp_reserve_bytes (int numbytes)
+{
+	if (unlikely(numbytes == 0))
+		return NULL;
+	_stp_print_flush();
+	return relay_reserve(_stp_chan, numbytes);
+}
+#else
+static void * _stp_reserve_bytes (int numbytes)
 {
-	int cpu = smp_processor_id();
-	char *buf = &_stp_pbuf[cpu][0];
-	char *ptr = buf + STP_PRINT_BUF_START;
-	int ret;
+	_stp_pbuf *pb = &__get_cpu_var(Stp_pbuf);
+	int size = STP_PRINT_BUF_LEN - pb->len;
+	void * ret;
 
-	if (_stp_pbuf_len[cpu] == 0)
-		return;
+	if (unlikely(numbytes == 0 || numbytes > STP_PRINT_BUF_LEN))
+		return NULL;
 
-	*((int *)buf) = _stp_seq_inc();
-	ret = _stp_transport_write(buf, _stp_pbuf_len[cpu] + STP_TIMESTAMP_SIZE + 1);
-	if (unlikely(ret < 0)) {
-#if 0
-		if (!atomic_read(&_stp_transport_failures))
-			_stp_warn("Transport failure - try using a larger buffer size\n");
-#endif
-		atomic_inc (&_stp_transport_failures);
-	}
+	if (numbytes > size)
+		_stp_print_flush();
 
-	_stp_pbuf_len[cpu] = 0;
-	*ptr = 0;
+	ret = pb->buf + pb->len;
+	pb->len += numbytes;
+	return ret;
 }
 #endif /* STP_RELAYFS */
 
+/** Write 64-bit args directly into the output stream.
+ * This function takes a variable number of 64-bit arguments
+ * and writes them directly into the output stream.  Marginally faster
+ * than doing the same in _stp_vsnprintf().
+ * @sa _stp_vsnprintf()
+ */
+static void _stp_print_binary (int num, ...)
+{
+	va_list vargs;
+	int i;
+	int64_t *args;
+	
+	if (unlikely(num > STP_MAXBINARYARGS))
+		num = STP_MAXBINARYARGS;
+
+	args = _stp_reserve_bytes(num * sizeof(int64_t));
+
+	if (args != NULL) {
+		va_start(vargs, num);
+		for (i = 0; i < num; i++) {
+			args[i] = va_arg(vargs, int64_t);
+		}
+		va_end(vargs);
+	}
+}
+
 /** Print into the print buffer.
  * Like printf, except output goes to the print buffer.
  * Safe because overflowing the buffer is not allowed.
diff --git a/runtime/runtime.h b/runtime/runtime.h
index 84c79f79..dec664d5 100644
--- a/runtime/runtime.h
+++ b/runtime/runtime.h
@@ -63,6 +63,14 @@ static struct
 #define _stp_seq_inc() (atomic_inc_return(&_stp_seq.seq))
 #endif /* RELAYFS */
 
+/* TEST_MODE is always defined by systemtap */
+#ifdef TEST_MODE
+#define SYSTEMTAP 1
+#else
+#define MAXTRYLOCK 1000
+#define TRYLOCKDELAY 100
+#endif
+
 #include "print.c"
 #include "string.c"
 #include "arith.c"
@@ -81,10 +89,6 @@ static const char * (*_stp_kallsyms_lookup)(unsigned long addr,
 					    unsigned long *offset,
 					    char **modname, char *namebuf);
 
-/* TEST_MODE is always defined by systemtap */
-#ifdef TEST_MODE
-#define SYSTEMTAP 1
-#endif
 
 #ifdef SYSTEMTAP
 /* This implementation is used if stap_[num_]symbols are available. */
@@ -161,7 +165,7 @@ int init_module (void)
     _stp_kallsyms_lookup = (const char * (*)(unsigned long,unsigned long *,unsigned long *,char **,char *))
       kallsyms_lookup_name("kallsyms_lookup");
 #endif
-
+    
   return _stp_transport_init();
 }
 
diff --git a/runtime/string.c b/runtime/string.c
index 359d8d45..a413ad76 100644
--- a/runtime/string.c
+++ b/runtime/string.c
@@ -58,28 +58,31 @@ void _stp_sprintf (String str, const char *fmt, ...)
 	int num;
 	va_list args;
 	if (str == _stp_stdout) {
-		int cpu = smp_processor_id();
-		char *buf = &_stp_pbuf[cpu][STP_PRINT_BUF_START] + _stp_pbuf_len[cpu];
-		int size = STP_PRINT_BUF_LEN -_stp_pbuf_len[cpu] + 1;
+		_stp_pbuf *pb = &__get_cpu_var(Stp_pbuf);
+		char *buf = pb->buf + pb->len;
+		int size = STP_PRINT_BUF_LEN - pb->len;
 		va_start(args, fmt);
 		num = _stp_vsnprintf(buf, size, fmt, args);
 		va_end(args);
-		if (unlikely(num >= size)) { 
+		if (unlikely(num > size)) { 
 			/* overflowed the buffer */
-			if (_stp_pbuf_len[cpu] == 0) {
-				_stp_pbuf_len[cpu] = STP_PRINT_BUF_LEN;
+			if (pb->len == 0) {
+				/* A single print request exceeded the buffer size. */
+				/* Should not be possible with Systemtap-generated code. */
+				pb->len = STP_PRINT_BUF_LEN;
 				_stp_print_flush();
+				num = 0;
 			} else {
-				*buf ='\0';
+				/* Need more space. Flush the previous contents */
 				_stp_print_flush();
+
+				/* try again */
 				va_start(args, fmt);
-				_stp_vsprintf(_stp_stdout, fmt, args); 
+				num = _stp_vsnprintf(pb->buf, STP_PRINT_BUF_LEN, fmt, args);
 				va_end(args);
 			}
-		} else {
-			_stp_pbuf_len[cpu] += num;
 		}
-
+		pb->len += num;
 	} else {
 		va_start(args, fmt);
 		num = _stp_vscnprintf(str->buf + str->len, STP_STRING_SIZE - str->len, fmt, args);
@@ -97,16 +100,27 @@ void _stp_vsprintf (String str, const char *fmt, va_list args)
 {
 	int num;
 	if (str == _stp_stdout) {
-		int cpu = smp_processor_id();
-		char *buf = &_stp_pbuf[cpu][STP_PRINT_BUF_START] + _stp_pbuf_len[cpu];
-		int size = STP_PRINT_BUF_LEN -_stp_pbuf_len[cpu] + 1;
+		_stp_pbuf *pb = &__get_cpu_var(Stp_pbuf);
+		char *buf = pb->buf + pb->len;
+		int size = STP_PRINT_BUF_LEN - pb->len;
 		num = _stp_vsnprintf(buf, size, fmt, args);
-		if (num < size)
-			_stp_pbuf_len[cpu] += num;
-		else {
-			_stp_pbuf_len[cpu] = STP_PRINT_BUF_LEN;
-			_stp_print_flush();
+		if (unlikely(num > size)) { 
+			/* overflowed the buffer */
+			if (pb->len == 0) {
+				/* A single print request exceeded the buffer size. */
+				/* Should not be possible with Systemtap-generated code. */
+				pb->len = STP_PRINT_BUF_LEN;
+				_stp_print_flush();
+				num = 0;
+			} else {
+				/* Need more space. Flush the previous contents */
+				_stp_print_flush();
+
+				/* try again */
+				num = _stp_vsnprintf(pb->buf, STP_PRINT_BUF_LEN, fmt, args);
+			}
 		}
+		pb->len += num;
 	} else {
 		num = _stp_vscnprintf(str->buf + str->len, STP_STRING_SIZE - str->len, fmt, args);
 		if (num > 0)
@@ -124,17 +138,15 @@ void _stp_string_cat_cstr (String str1, const char *str2)
 {
 	int num = strlen (str2);
 	if (str1 == _stp_stdout) {
-		char *buf;
-		int cpu = smp_processor_id();
-		int size = STP_PRINT_BUF_LEN -_stp_pbuf_len[cpu];
-		if (num >= size) {
+		_stp_pbuf *pb = &__get_cpu_var(Stp_pbuf);
+		int size = STP_PRINT_BUF_LEN - pb->len;
+		if (num > size) {
 			_stp_print_flush();
 			if (num > STP_PRINT_BUF_LEN)
 				num = STP_PRINT_BUF_LEN;
 		}
-		buf = &_stp_pbuf[cpu][STP_PRINT_BUF_START] + _stp_pbuf_len[cpu];
-		memcpy (buf, str2, num);
-		_stp_pbuf_len[cpu] += num;
+		memcpy (pb->buf + pb->len, str2, num);
+		pb->len += num;
 	} else {
 		int size = STP_STRING_SIZE - str1->len - 1; 
 		if (num > size)
@@ -161,15 +173,16 @@ void _stp_string_cat_string (String str1, String str2)
 void _stp_string_cat_char (String str1, const char c)
 {
 	if (str1 == _stp_stdout) {
+		_stp_pbuf *pb = &__get_cpu_var(Stp_pbuf);
+		int size = STP_PRINT_BUF_LEN - pb->len;
 		char *buf;
-		int cpu = smp_processor_id();
-		int size = STP_PRINT_BUF_LEN -_stp_pbuf_len[cpu];
+
 		if (1 >= size)
 			_stp_print_flush();
-		buf = &_stp_pbuf[cpu][STP_PRINT_BUF_START] + _stp_pbuf_len[cpu];
+			
+		buf = pb->buf + pb->len;
 		buf[0] = c;
-		buf[1] = 0;
-		_stp_pbuf_len[cpu] ++;
+		pb->len ++;
 	} else {
 		int size = STP_STRING_SIZE - str1->len - 1; 
 		if (size > 0) {