6 files changed, 105 insertions, 11 deletions
diff --git a/testsuite/systemtap.examples/index.html b/testsuite/systemtap.examples/index.html
index 0f4b2572..9880c5b6 100644
--- a/testsuite/systemtap.examples/index.html
+++ b/testsuite/systemtap.examples/index.html
@@ -60,7 +60,7 @@ keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#BAC
 <p>When a reschedule occurs during an AIO io_submit call, accumulate the traceback in a histogram. When the script exits prints out a sorted list from most common to least common backtrace.</p></li>
 <li><a href="io/ioblktime.stp">io/ioblktime.stp</a> - Average Time Block IO Requests Spend in Queue <br>
 keywords: <a href="keyword-index.html#IO">IO</a> <br>
-<p>The ioblktime.stp script tracks the amount of time that each block IO requests spend waiting for completion. The script compute the average time waiting time for block IO per device and prints list every 10 seconds. In some cases there can be too many oustanding block IO operations and the script may exceed the default number of MAXMAPENTRIES allowed. In this case the allowed number can be increased with "-DMAXMAPENTRIES=10000" option on the stap command line.</p></li>
+<p>The ioblktime.stp script tracks the amount of time that each block IO requests spend waiting for completion. The script computes the average time waiting time for block IO per device and prints list every 10 seconds. In some cases there can be too many oustanding block IO operations and the script may exceed the default number of MAXMAPENTRIES allowed. In this case the allowed number can be increased with "-DMAXMAPENTRIES=10000" option on the stap command line.</p></li>
 <li><a href="io/iostats.stp">io/iostats.stp</a> - List Executables Reading and Writing the Most Data<br>
 keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#PROFILING">PROFILING</a> <br>
 <p> The iostat.stp script measures the amount of data successfully read and written by all the executables on the system.  The output is sorted from most greatest sum of bytes read and written by an executable to the least. The output contains  the count of operations (opens, reads, and writes), the totals and averages for the number of bytes read and written.</p></li>
@@ -91,6 +91,9 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 <li><a href="network/tcp_connections.stp">network/tcp_connections.stp</a> - Track Creation of Incoming TCP Connections<br>
 keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-index.html#TCP">TCP</a> <a href="keyword-index.html#SOCKET">SOCKET</a> <br>
 <p>The tcp_connections.stp script prints information for each new incoming TCP connection accepted by the computer. The information includes the UID, the command accepting the connection, the PID of the command, the port the connection is on, and the IP address of the originator of the request.</p></li>
+<li><a href="process/errsnoop.stp">process/errsnoop.stp</a> - tabulate system call errors<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SYSCALL">SYSCALL</a> <br>
+<p>The script prints a periodic tabular report about failing system calls, by process and by syscall failure.  The first optional argument specifies the reporting interval (in seconds, default 5); the second optional argument gives a screen height (number of lines in the report, default 20).</p></li>
 <li><a href="process/futexes.stp">process/futexes.stp</a> - System-Wide Futex Contention<br>
 keywords: <a href="keyword-index.html#SYSCALL">SYSCALL</a> <a href="keyword-index.html#LOCKING">LOCKING</a> <a href="keyword-index.html#FUTEX">FUTEX</a> <br>
 <p>The script watches the futex syscall on the system. On exit the futexes address, the number of contentions, and the average time for each contention on the futex are printed from lowest pid number to highest.</p></li>
diff --git a/testsuite/systemtap.examples/index.txt b/testsuite/systemtap.examples/index.txt
index e31baf4f..96ac5ab0 100644
--- a/testsuite/systemtap.examples/index.txt
+++ b/testsuite/systemtap.examples/index.txt
@@ -56,10 +56,10 @@ io/ioblktime.stp - Average Time Block IO Requests Spend in Queue
 keywords: io
 
   The ioblktime.stp script tracks the amount of time that each block IO
-  requests spend waiting for completion. The script compute the average
-  time waiting time for block IO per device and prints list every 10
-  seconds. In some cases there can be too many oustanding block IO
-  operations and the script may exceed the default number of
+  requests spend waiting for completion. The script computes the
+  average time waiting time for block IO per device and prints list
+  every 10 seconds. In some cases there can be too many oustanding
+  block IO operations and the script may exceed the default number of
   MAXMAPENTRIES allowed. In this case the allowed number can be
   increased with "-DMAXMAPENTRIES=10000" option on the stap command
   line.
@@ -166,6 +166,16 @@ keywords: network tcp socket
   originator of the request.
 
 
+process/errsnoop.stp - tabulate system call errors
+keywords: process syscall
+
+  The script prints a periodic tabular report about failing system
+  calls, by process and by syscall failure.  The first optional
+  argument specifies the reporting interval (in seconds, default 5);
+  the second optional argument gives a screen height (number of lines
+  in the report, default 20).
+
+
 process/futexes.stp - System-Wide Futex Contention
 keywords: syscall locking futex
 
diff --git a/testsuite/systemtap.examples/keyword-index.html b/testsuite/systemtap.examples/keyword-index.html
index 75768709..78d467cd 100644
--- a/testsuite/systemtap.examples/keyword-index.html
+++ b/testsuite/systemtap.examples/keyword-index.html
@@ -39,7 +39,7 @@
          	</ul>
 
 <h2>Examples by Keyword</h2>
-<p><tt><a href="#BACKTRACE">BACKTRACE</a> <a href="#CALLGRAPH">CALLGRAPH</a> <a href="#CPU">CPU</a> <a href="#DISK">DISK</a> <a href="#FUNCTIONS">FUNCTIONS</a> <a href="#FUTEX">FUTEX</a> <a href="#GRAPH">GRAPH</a> <a href="#INTERRUPT">INTERRUPT</a> <a href="#IO">IO</a> <a href="#LOCKING">LOCKING</a> <a href="#MEMORY">MEMORY</a> <a href="#NETWORK">NETWORK</a> <a href="#PER-PROCESS">PER-PROCESS</a> <a href="#PROFILING">PROFILING</a> <a href="#READ">READ</a> <a href="#SCHEDULER">SCHEDULER</a> <a href="#SIGNALS">SIGNALS</a> <a href="#SIMPLE">SIMPLE</a> <a href="#SLEEP">SLEEP</a> <a href="#SOCKET">SOCKET</a> <a href="#SYSCALL">SYSCALL</a> <a href="#TCP">TCP</a> <a href="#TIME">TIME</a> <a href="#TRACE">TRACE</a> <a href="#TRAFFIC">TRAFFIC</a> <a href="#USE">USE</a> <a href="#WAIT4">WAIT4</a> <a href="#WRITE">WRITE</a> </tt></p>
+<p><tt><a href="#BACKTRACE">BACKTRACE</a> <a href="#CALLGRAPH">CALLGRAPH</a> <a href="#CPU">CPU</a> <a href="#DISK">DISK</a> <a href="#FUNCTIONS">FUNCTIONS</a> <a href="#FUTEX">FUTEX</a> <a href="#GRAPH">GRAPH</a> <a href="#INTERRUPT">INTERRUPT</a> <a href="#IO">IO</a> <a href="#LOCKING">LOCKING</a> <a href="#MEMORY">MEMORY</a> <a href="#NETWORK">NETWORK</a> <a href="#PER-PROCESS">PER-PROCESS</a> <a href="#PROCESS">PROCESS</a> <a href="#PROFILING">PROFILING</a> <a href="#READ">READ</a> <a href="#SCHEDULER">SCHEDULER</a> <a href="#SIGNALS">SIGNALS</a> <a href="#SIMPLE">SIMPLE</a> <a href="#SLEEP">SLEEP</a> <a href="#SOCKET">SOCKET</a> <a href="#SYSCALL">SYSCALL</a> <a href="#TCP">TCP</a> <a href="#TIME">TIME</a> <a href="#TRACE">TRACE</a> <a href="#TRAFFIC">TRAFFIC</a> <a href="#USE">USE</a> <a href="#WAIT4">WAIT4</a> <a href="#WRITE">WRITE</a> </tt></p>
 <h3><a name="BACKTRACE">BACKTRACE</a></h3>
 <ul>
 <li><a href="interrupt/scf.stp">interrupt/scf.stp</a> - Tally Backtraces for Inter-Processor Interrupt (IPI)<br>
@@ -104,7 +104,7 @@ keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#BAC
 <p>When a reschedule occurs during an AIO io_submit call, accumulate the traceback in a histogram. When the script exits prints out a sorted list from most common to least common backtrace.</p></li>
 <li><a href="io/ioblktime.stp">io/ioblktime.stp</a> - Average Time Block IO Requests Spend in Queue <br>
 keywords: <a href="keyword-index.html#IO">IO</a> <br>
-<p>The ioblktime.stp script tracks the amount of time that each block IO requests spend waiting for completion. The script compute the average time waiting time for block IO per device and prints list every 10 seconds. In some cases there can be too many oustanding block IO operations and the script may exceed the default number of MAXMAPENTRIES allowed. In this case the allowed number can be increased with "-DMAXMAPENTRIES=10000" option on the stap command line.</p></li>
+<p>The ioblktime.stp script tracks the amount of time that each block IO requests spend waiting for completion. The script computes the average time waiting time for block IO per device and prints list every 10 seconds. In some cases there can be too many oustanding block IO operations and the script may exceed the default number of MAXMAPENTRIES allowed. In this case the allowed number can be increased with "-DMAXMAPENTRIES=10000" option on the stap command line.</p></li>
 <li><a href="io/iostats.stp">io/iostats.stp</a> - List Executables Reading and Writing the Most Data<br>
 keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#PROFILING">PROFILING</a> <br>
 <p> The iostat.stp script measures the amount of data successfully read and written by all the executables on the system.  The output is sorted from most greatest sum of bytes read and written by an executable to the least. The output contains  the count of operations (opens, reads, and writes), the totals and averages for the number of bytes read and written.</p></li>
@@ -157,6 +157,12 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-index.html#TRAFFIC">TRAFFIC</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <br>
 <p>Every five seconds the nettop.stp script prints out a list of processed (PID and command) with the number of packets sent/received and the amount of data sent/received by the process during that interval.</p></li>
 </ul>
+<h3><a name="PROCESS">PROCESS</a></h3>
+<ul>
+<li><a href="process/errsnoop.stp">process/errsnoop.stp</a> - tabulate system call errors<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SYSCALL">SYSCALL</a> <br>
+<p>The script prints a periodic tabular report about failing system calls, by process and by syscall failure.  The first optional argument specifies the reporting interval (in seconds, default 5); the second optional argument gives a screen height (number of lines in the report, default 20).</p></li>
+</ul>
 <h3><a name="PROFILING">PROFILING</a></h3>
 <ul>
 <li><a href="io/iostats.stp">io/iostats.stp</a> - List Executables Reading and Writing the Most Data<br>
@@ -231,6 +237,9 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 <li><a href="io/iotime.stp">io/iotime.stp</a> - Trace Time Spent in Read and Write for Files <br>
 keywords: <a href="keyword-index.html#SYSCALL">SYSCALL</a> <a href="keyword-index.html#READ">READ</a> <a href="keyword-index.html#WRITE">WRITE</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#IO">IO</a> <br>
 <p>The script watches each open, close, read, and write syscalls on the system. For each file the scripts observes opened it accumulates the amount of wall clock time spend in read and write operations and the number of bytes read and written. When a file is closed the script prints out a pair of lines for the file. Both lines begin with a timestamp in microseconds, the PID number, and the executable name in parenthesese. The first line with the "access" keyword lists the file name, the attempted number of bytes for the read and write operations. The second line with the "iotime" keyword list the file name and the number of microseconds accumulated in the read and write syscalls.</p></li>
+<li><a href="process/errsnoop.stp">process/errsnoop.stp</a> - tabulate system call errors<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SYSCALL">SYSCALL</a> <br>
+<p>The script prints a periodic tabular report about failing system calls, by process and by syscall failure.  The first optional argument specifies the reporting interval (in seconds, default 5); the second optional argument gives a screen height (number of lines in the report, default 20).</p></li>
 <li><a href="process/futexes.stp">process/futexes.stp</a> - System-Wide Futex Contention<br>
 keywords: <a href="keyword-index.html#SYSCALL">SYSCALL</a> <a href="keyword-index.html#LOCKING">LOCKING</a> <a href="keyword-index.html#FUTEX">FUTEX</a> <br>
 <p>The script watches the futex syscall on the system. On exit the futexes address, the number of contentions, and the average time for each contention on the futex are printed from lowest pid number to highest.</p></li>
diff --git a/testsuite/systemtap.examples/keyword-index.txt b/testsuite/systemtap.examples/keyword-index.txt
index d3a9617f..17334252 100644
--- a/testsuite/systemtap.examples/keyword-index.txt
+++ b/testsuite/systemtap.examples/keyword-index.txt
@@ -129,10 +129,10 @@ io/ioblktime.stp - Average Time Block IO Requests Spend in Queue
 keywords: io
 
   The ioblktime.stp script tracks the amount of time that each block IO
-  requests spend waiting for completion. The script compute the average
-  time waiting time for block IO per device and prints list every 10
-  seconds. In some cases there can be too many oustanding block IO
-  operations and the script may exceed the default number of
+  requests spend waiting for completion. The script computes the
+  average time waiting time for block IO per device and prints list
+  every 10 seconds. In some cases there can be too many oustanding
+  block IO operations and the script may exceed the default number of
   MAXMAPENTRIES allowed. In this case the allowed number can be
   increased with "-DMAXMAPENTRIES=10000" option on the stap command
   line.
@@ -274,6 +274,18 @@ keywords: network traffic per-process
   interval.
 
 
+= PROCESS =
+
+process/errsnoop.stp - tabulate system call errors
+keywords: process syscall
+
+  The script prints a periodic tabular report about failing system
+  calls, by process and by syscall failure.  The first optional
+  argument specifies the reporting interval (in seconds, default 5);
+  the second optional argument gives a screen height (number of lines
+  in the report, default 20).
+
+
 = PROFILING =
 
 io/iostats.stp - List Executables Reading and Writing the Most Data
@@ -458,6 +470,16 @@ keywords: syscall read write time io
   syscalls.
 
 
+process/errsnoop.stp - tabulate system call errors
+keywords: process syscall
+
+  The script prints a periodic tabular report about failing system
+  calls, by process and by syscall failure.  The first optional
+  argument specifies the reporting interval (in seconds, default 5);
+  the second optional argument gives a screen height (number of lines
+  in the report, default 20).
+
+
 process/futexes.stp - System-Wide Futex Contention
 keywords: syscall locking futex
 
diff --git a/testsuite/systemtap.examples/process/errsnoop.meta b/testsuite/systemtap.examples/process/errsnoop.meta
new file mode 100644
index 00000000..34b8cb7c
--- /dev/null
+++ b/testsuite/systemtap.examples/process/errsnoop.meta
@@ -0,0 +1,7 @@
+title: tabulate system call errors
+name: errsnoop.stp
+keywords: process syscall
+subsystem: general
+description: The script prints a periodic tabular report about failing system calls, by process and by syscall failure.  The first optional argument specifies the reporting interval (in seconds, default 5); the second optional argument gives a screen height (number of lines in the report, default 20).
+test_check: stap -p4 errsnoop.stp
+test_installcheck: stap errsnoop.stp 1 10 -c "sleep 3"
diff --git a/testsuite/systemtap.examples/process/errsnoop.stp b/testsuite/systemtap.examples/process/errsnoop.stp
new file mode 100644
index 00000000..d9fba3b7
--- /dev/null
+++ b/testsuite/systemtap.examples/process/errsnoop.stp
@@ -0,0 +1,43 @@
+#!/usr/bin/env stap
+# errsnoop.stp
+# Copyright (C) 2009 Red Hat, Inc., Eugene Teo <eteo@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# attack "stupid userspace" apps
+#
+
+global error, trace
+
+probe syscall.* {
+	# assume syscall don't nest
+	trace[tid()] = argstr
+}
+
+probe syscall.*.return {
+	errno = errno_p(returnval())
+	if (errno != 0) {
+		t = tid()
+		argstr = trace[t]
+		delete trace[t]
+
+		error[name, execname(), pid(), errno, argstr] <<< 1
+	}
+}
+
+probe timer.s(%( $# > 0 %? $1 %: 5 %)) {
+	ansi_clear_screen()
+	printf("%17s %15s %5s %4s %-12s %s\n",
+			"SYSCALL", "PROCESS", "PID", "HITS", "ERRSTR", "ARGSTR")
+	foreach([fn, comm, pid, errno, argstr] in error- limit %( $# > 1 %? $2 %: 20 %)) {
+		errstr = sprintf("%3d (%s)", errno, errno_str(errno))
+		printf("%17s %15s %5d %4d %-12s %s\n", fn, comm, pid,
+				@count(error[fn, comm, pid, errno, argstr]),
+#				errstr, substr(argstr,0,22)) # within cols#80
+				errstr, argstr)
+
+	}
+	delete error
+}