Merge commit 'origin/master' into pr7043

8 files changed, 350 insertions, 1 deletions

diff --git a/testsuite/systemtap.examples/index.html b/testsuite/systemtap.examples/index.html
index 0df681ac..b2ed3a3a 100644
--- a/testsuite/systemtap.examples/index.html
+++ b/testsuite/systemtap.examples/index.html
@@ -82,6 +82,9 @@ keywords: <a href="keyword-index.html#IO">IO</a> <br>
 <li><a href="io/traceio2.stp">io/traceio2.stp</a> - Watch I/O Activity on a Particular Device<br>
 keywords: <a href="keyword-index.html#IO">IO</a> <br>
 <p>Print out the executable name and process number as reads and writes to the specified device occur.</p></li>
+<li><a href="io/ttyspy.stp">io/ttyspy.stp</a> - Monitor tty typing.<br>
+keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#TTY">TTY</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <a href="keyword-index.html#MONITOR">MONITOR</a> <br>
+<p>The ttyspy.stp script uses tty_audit hooks to monitor recent typing activity on the system, printing a scrolling record of recent keystrokes, on a per-tty basis.</p></li>
 <li><a href="memory/kmalloc-top">memory/kmalloc-top</a> - Show Paths to Kernel Malloc (kmalloc) Invocations<br>
 keywords: <a href="keyword-index.html#MEMORY">MEMORY</a> <br>
 <p>The kmalloc-top perl program runs a small systemtap script to collect stack traces for each call to the kmalloc function and counts the time that each stack trace is observed. When kmalloc-top exits it prints out sorted list. The output can be be filtered to print only only the first stack traces (-t) stack traces with more a minimum counts (-m), or exclude certain stack traces (-e).</p></li>
@@ -112,6 +115,9 @@ keywords: <a href="keyword-index.html#SYSCALL">SYSCALL</a> <a href="keyword-inde
 <li><a href="process/pf2.stp">process/pf2.stp</a> - Profile kernel functions<br>
 keywords: <a href="keyword-index.html#PROFILING">PROFILING</a> <br>
 <p>The pf2.stp script sets up time-based sampling. Every five seconds it prints out a sorted list with the top ten kernel functions with samples.</p></li>
+<li><a href="process/schedtimes.stp">process/schedtimes.stp</a> - Track Time Processes Spend in Various States using Tracepoints<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#TRACEPOINT">TRACEPOINT</a> <br>
+<p>The schedtimes.stp script instruments the scheduler to track the amount of time that each process spends running, sleeping, queued, and waiting for io. On exit the script prints out the accumulated time for each state of processes observed.  Optionally, this script can be used with the '-c' or '-x' options to focus on a specific PID.</p></li>
 <li><a href="process/sig_by_pid.stp">process/sig_by_pid.stp</a> -  Signal Counts by Process ID<br>
 keywords: <a href="keyword-index.html#SIGNALS">SIGNALS</a> <br>
 <p>Print signal counts by process ID in descending order.</p></li>
diff --git a/testsuite/systemtap.examples/index.txt b/testsuite/systemtap.examples/index.txt
index fa344933..91fc66ae 100644
--- a/testsuite/systemtap.examples/index.txt
+++ b/testsuite/systemtap.examples/index.txt
@@ -129,6 +129,14 @@ keywords: io
   to the specified device occur.
 
 
+io/ttyspy.stp - Monitor tty typing.
+keywords: io tty per-process monitor
+
+  The ttyspy.stp script uses tty_audit hooks to monitor recent typing
+  activity on the system, printing a scrolling record of recent
+  keystrokes, on a per-tty basis.
+
+
 memory/kmalloc-top - Show Paths to Kernel Malloc (kmalloc) Invocations
 keywords: memory
 
@@ -224,6 +232,16 @@ keywords: profiling
   samples.
 
 
+process/schedtimes.stp - Track Time Processes Spend in Various States using Tracepoints
+keywords: process scheduler time tracepoint
+
+  The schedtimes.stp script instruments the scheduler to track the
+  amount of time that each process spends running, sleeping, queued,
+  and waiting for io. On exit the script prints out the accumulated
+  time for each state of processes observed.  Optionally, this script
+  can be used with the '-c' or '-x' options to focus on a specific PID.
+
+
 process/sig_by_pid.stp -  Signal Counts by Process ID
 keywords: signals
 
diff --git a/testsuite/systemtap.examples/io/ttyspy.meta b/testsuite/systemtap.examples/io/ttyspy.meta
new file mode 100644
index 00000000..e29add1b
--- /dev/null
+++ b/testsuite/systemtap.examples/io/ttyspy.meta
@@ -0,0 +1,6 @@
+title: Monitor tty typing.
+name: ttyspy.stp
+keywords: io tty per-process monitor
+description: The ttyspy.stp script uses tty_audit hooks to monitor recent typing activity on the system, printing a scrolling record of recent keystrokes, on a per-tty basis.
+test_check: stap -gp4 ttyspy.stp
+test_installcheck: stap --skip-badvars -g ttyspy.stp -c "sleep 7"
diff --git a/testsuite/systemtap.examples/io/ttyspy.stp b/testsuite/systemtap.examples/io/ttyspy.stp
new file mode 100755
index 00000000..272d82e9
--- /dev/null
+++ b/testsuite/systemtap.examples/io/ttyspy.stp
@@ -0,0 +1,46 @@
+#! /usr/bin/stap -g
+# May also need --skip-badvars
+
+global activity_time, activity_log
+
+/* Concatenate head and tail, to a max of @num chars, preferring to keep the tail
+   (as if it were a recent history buffer). */
+function strcattail:string(head:string,tail:string,num:long) %{
+         unsigned taillen = strlen(THIS->tail);
+         unsigned headlen = strlen(THIS->head);
+         unsigned maxlen = THIS->num < MAXSTRINGLEN ? THIS->num : MAXSTRINGLEN;
+         unsigned headkeep = min(maxlen-taillen,headlen);
+         unsigned headdrop = headlen-headkeep;
+
+         if (headkeep)
+            strlcpy (THIS->__retvalue, &THIS->head[headdrop], headkeep+1); /* includes \0 */
+         strlcat (THIS->__retvalue, THIS->tail, maxlen);
+%}
+
+probe kernel.function("tty_audit_add_data") {
+      major=$tty->driver->major;
+      minor=$tty->driver->minor_start + $tty->index;
+      pgrp=$tty->pgrp %( kernel_v >= "2.6.24" %? ->numbers[0]->nr %: %);
+      data=kernel_string_n($data,$size);
+      uid=uid()
+
+      activity_time[major,minor,pgrp,uid] = gettimeofday_s();
+      activity_log[major,minor,pgrp,uid]
+       = strcattail(activity_log[major,minor,pgrp,uid],data,40);
+}
+
+probe timer.s(3) {
+      ansi_clear_screen ()
+      printf("(%3s,%2s,%5s,%5s)\n", "maj","min","pgrp","uid");
+      foreach ([x,y,z,u] in activity_time-) {
+              printf("(%3d,%3d,%5d,%5d) %s\n", x,y,z,u,
+               text_str(activity_log[x,y,z,u]))
+      }
+
+      /* delete last record, if older than 60 seconds */
+      if (activity_time[x,y,z,u]+60 < gettimeofday_s()) {
+         delete activity_time[x,y,z,u]
+         delete activity_log[x,y,z,u]
+      }
+}
+
diff --git a/testsuite/systemtap.examples/keyword-index.html b/testsuite/systemtap.examples/keyword-index.html
index 7edbec21..7306c164 100644
--- a/testsuite/systemtap.examples/keyword-index.html
+++ b/testsuite/systemtap.examples/keyword-index.html
@@ -39,7 +39,7 @@
          	</ul>
 
 <h2>Examples by Keyword</h2>
-<p><tt><a href="#BACKTRACE">BACKTRACE</a> <a href="#BUFFER">BUFFER</a> <a href="#CALLGRAPH">CALLGRAPH</a> <a href="#CPU">CPU</a> <a href="#DISK">DISK</a> <a href="#FORMAT">FORMAT</a> <a href="#FREE">FREE</a> <a href="#FUNCTIONS">FUNCTIONS</a> <a href="#FUTEX">FUTEX</a> <a href="#GRAPH">GRAPH</a> <a href="#INTERRUPT">INTERRUPT</a> <a href="#IO">IO</a> <a href="#LOCKING">LOCKING</a> <a href="#MEMORY">MEMORY</a> <a href="#NETWORK">NETWORK</a> <a href="#PER-PROCESS">PER-PROCESS</a> <a href="#PROCESS">PROCESS</a> <a href="#PROFILING">PROFILING</a> <a href="#READ">READ</a> <a href="#SCHEDULER">SCHEDULER</a> <a href="#SIGNALS">SIGNALS</a> <a href="#SIMPLE">SIMPLE</a> <a href="#SLEEP">SLEEP</a> <a href="#SOCKET">SOCKET</a> <a href="#SYSCALL">SYSCALL</a> <a href="#TCP">TCP</a> <a href="#TIME">TIME</a> <a href="#TRACE">TRACE</a> <a href="#TRACEPOINT">TRACEPOINT</a> <a href="#TRAFFIC">TRAFFIC</a> <a href="#USE">USE</a> <a href="#WAIT4">WAIT4</a> <a href="#WRITE">WRITE</a> </tt></p>
+<p><tt><a href="#BACKTRACE">BACKTRACE</a> <a href="#BUFFER">BUFFER</a> <a href="#CALLGRAPH">CALLGRAPH</a> <a href="#CPU">CPU</a> <a href="#DISK">DISK</a> <a href="#FORMAT">FORMAT</a> <a href="#FREE">FREE</a> <a href="#FUNCTIONS">FUNCTIONS</a> <a href="#FUTEX">FUTEX</a> <a href="#GRAPH">GRAPH</a> <a href="#INTERRUPT">INTERRUPT</a> <a href="#IO">IO</a> <a href="#LOCKING">LOCKING</a> <a href="#MEMORY">MEMORY</a> <a href="#MONITOR">MONITOR</a> <a href="#NETWORK">NETWORK</a> <a href="#PER-PROCESS">PER-PROCESS</a> <a href="#PROCESS">PROCESS</a> <a href="#PROFILING">PROFILING</a> <a href="#READ">READ</a> <a href="#SCHEDULER">SCHEDULER</a> <a href="#SIGNALS">SIGNALS</a> <a href="#SIMPLE">SIMPLE</a> <a href="#SLEEP">SLEEP</a> <a href="#SOCKET">SOCKET</a> <a href="#SYSCALL">SYSCALL</a> <a href="#TCP">TCP</a> <a href="#TIME">TIME</a> <a href="#TRACE">TRACE</a> <a href="#TRACEPOINT">TRACEPOINT</a> <a href="#TRAFFIC">TRAFFIC</a> <a href="#TTY">TTY</a> <a href="#USE">USE</a> <a href="#WAIT4">WAIT4</a> <a href="#WRITE">WRITE</a> </tt></p>
 <h3><a name="BACKTRACE">BACKTRACE</a></h3>
 <ul>
 <li><a href="interrupt/scf.stp">interrupt/scf.stp</a> - Tally Backtraces for Inter-Processor Interrupt (IPI)<br>
@@ -141,6 +141,9 @@ keywords: <a href="keyword-index.html#IO">IO</a> <br>
 <li><a href="io/traceio2.stp">io/traceio2.stp</a> - Watch I/O Activity on a Particular Device<br>
 keywords: <a href="keyword-index.html#IO">IO</a> <br>
 <p>Print out the executable name and process number as reads and writes to the specified device occur.</p></li>
+<li><a href="io/ttyspy.stp">io/ttyspy.stp</a> - Monitor tty typing.<br>
+keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#TTY">TTY</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <a href="keyword-index.html#MONITOR">MONITOR</a> <br>
+<p>The ttyspy.stp script uses tty_audit hooks to monitor recent typing activity on the system, printing a scrolling record of recent keystrokes, on a per-tty basis.</p></li>
 <li><a href="process/sleepingBeauties.stp">process/sleepingBeauties.stp</a> - Generating Backtraces of Threads Waiting for IO Operations<br>
 keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#BACKTRACE">BACKTRACE</a> <br>
 <p>The script monitors the time that threads spend waiting for IO operations (in "D" state) in the wait_for_completion function.  If a thread spends over 10ms, its name and backtrace is printed, and later so is the total delay.</p></li>
@@ -160,6 +163,12 @@ keywords: <a href="keyword-index.html#MEMORY">MEMORY</a> <br>
 keywords: <a href="keyword-index.html#MEMORY">MEMORY</a> <br>
 <p>The pfaults.stp script generates a simple log for each major and minor page fault that occurs on the system. Each line contains a timestamp (in microseconds) when the page fault servicing was completed, the pid of the process, the address of the page fault, the type of access (read or write), the type of fault (major or minor), and the elapsed time for page fault. This log can be examined to determine where the page faults are occuring.</p></li>
 </ul>
+<h3><a name="MONITOR">MONITOR</a></h3>
+<ul>
+<li><a href="io/ttyspy.stp">io/ttyspy.stp</a> - Monitor tty typing.<br>
+keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#TTY">TTY</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <a href="keyword-index.html#MONITOR">MONITOR</a> <br>
+<p>The ttyspy.stp script uses tty_audit hooks to monitor recent typing activity on the system, printing a scrolling record of recent keystrokes, on a per-tty basis.</p></li>
+</ul>
 <h3><a name="NETWORK">NETWORK</a></h3>
 <ul>
 <li><a href="network/dropwatch.stp">network/dropwatch.stp</a> - Watch Where Socket Buffers are Freed in the Kernel<br>
@@ -180,6 +189,9 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 </ul>
 <h3><a name="PER-PROCESS">PER-PROCESS</a></h3>
 <ul>
+<li><a href="io/ttyspy.stp">io/ttyspy.stp</a> - Monitor tty typing.<br>
+keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#TTY">TTY</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <a href="keyword-index.html#MONITOR">MONITOR</a> <br>
+<p>The ttyspy.stp script uses tty_audit hooks to monitor recent typing activity on the system, printing a scrolling record of recent keystrokes, on a per-tty basis.</p></li>
 <li><a href="network/nettop.stp">network/nettop.stp</a> - Periodic Listing of Processes Using Network Interfaces<br>
 keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-index.html#TRAFFIC">TRAFFIC</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <br>
 <p>Every five seconds the nettop.stp script prints out a list of processed (PID and command) with the number of packets sent/received and the amount of data sent/received by the process during that interval.</p></li>
@@ -189,6 +201,9 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 <li><a href="process/errsnoop.stp">process/errsnoop.stp</a> - tabulate system call errors<br>
 keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SYSCALL">SYSCALL</a> <br>
 <p>The script prints a periodic tabular report about failing system calls, by process and by syscall failure.  The first optional argument specifies the reporting interval (in seconds, default 5); the second optional argument gives a screen height (number of lines in the report, default 20).</p></li>
+<li><a href="process/schedtimes.stp">process/schedtimes.stp</a> - Track Time Processes Spend in Various States using Tracepoints<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#TRACEPOINT">TRACEPOINT</a> <br>
+<p>The schedtimes.stp script instruments the scheduler to track the amount of time that each process spends running, sleeping, queued, and waiting for io. On exit the script prints out the accumulated time for each state of processes observed.  Optionally, this script can be used with the '-c' or '-x' options to focus on a specific PID.</p></li>
 </ul>
 <h3><a name="PROFILING">PROFILING</a></h3>
 <ul>
@@ -219,6 +234,9 @@ keywords: <a href="keyword-index.html#SYSCALL">SYSCALL</a> <a href="keyword-inde
 </ul>
 <h3><a name="SCHEDULER">SCHEDULER</a></h3>
 <ul>
+<li><a href="process/schedtimes.stp">process/schedtimes.stp</a> - Track Time Processes Spend in Various States using Tracepoints<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#TRACEPOINT">TRACEPOINT</a> <br>
+<p>The schedtimes.stp script instruments the scheduler to track the amount of time that each process spends running, sleeping, queued, and waiting for io. On exit the script prints out the accumulated time for each state of processes observed.  Optionally, this script can be used with the '-c' or '-x' options to focus on a specific PID.</p></li>
 <li><a href="process/sleepingBeauties.stp">process/sleepingBeauties.stp</a> - Generating Backtraces of Threads Waiting for IO Operations<br>
 keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#BACKTRACE">BACKTRACE</a> <br>
 <p>The script monitors the time that threads spend waiting for IO operations (in "D" state) in the wait_for_completion function.  If a thread spends over 10ms, its name and backtrace is printed, and later so is the total delay.</p></li>
@@ -294,6 +312,9 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 <li><a href="io/iotime.stp">io/iotime.stp</a> - Trace Time Spent in Read and Write for Files <br>
 keywords: <a href="keyword-index.html#SYSCALL">SYSCALL</a> <a href="keyword-index.html#READ">READ</a> <a href="keyword-index.html#WRITE">WRITE</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#IO">IO</a> <br>
 <p>The script watches each open, close, read, and write syscalls on the system. For each file the scripts observes opened it accumulates the amount of wall clock time spend in read and write operations and the number of bytes read and written. When a file is closed the script prints out a pair of lines for the file. Both lines begin with a timestamp in microseconds, the PID number, and the executable name in parenthesese. The first line with the "access" keyword lists the file name, the attempted number of bytes for the read and write operations. The second line with the "iotime" keyword list the file name and the number of microseconds accumulated in the read and write syscalls.</p></li>
+<li><a href="process/schedtimes.stp">process/schedtimes.stp</a> - Track Time Processes Spend in Various States using Tracepoints<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#TRACEPOINT">TRACEPOINT</a> <br>
+<p>The schedtimes.stp script instruments the scheduler to track the amount of time that each process spends running, sleeping, queued, and waiting for io. On exit the script prints out the accumulated time for each state of processes observed.  Optionally, this script can be used with the '-c' or '-x' options to focus on a specific PID.</p></li>
 </ul>
 <h3><a name="TRACE">TRACE</a></h3>
 <ul>
@@ -306,6 +327,9 @@ keywords: <a href="keyword-index.html#TRACE">TRACE</a> <a href="keyword-index.ht
 <li><a href="network/dropwatch.stp">network/dropwatch.stp</a> - Watch Where Socket Buffers are Freed in the Kernel<br>
 keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-index.html#TRACEPOINT">TRACEPOINT</a> <a href="keyword-index.html#BUFFER">BUFFER</a> <a href="keyword-index.html#FREE">FREE</a> <br>
 <p>Every five seconds the dropwatch.stp script lists the number of socket buffers freed at locations in the kernel.</p></li>
+<li><a href="process/schedtimes.stp">process/schedtimes.stp</a> - Track Time Processes Spend in Various States using Tracepoints<br>
+keywords: <a href="keyword-index.html#PROCESS">PROCESS</a> <a href="keyword-index.html#SCHEDULER">SCHEDULER</a> <a href="keyword-index.html#TIME">TIME</a> <a href="keyword-index.html#TRACEPOINT">TRACEPOINT</a> <br>
+<p>The schedtimes.stp script instruments the scheduler to track the amount of time that each process spends running, sleeping, queued, and waiting for io. On exit the script prints out the accumulated time for each state of processes observed.  Optionally, this script can be used with the '-c' or '-x' options to focus on a specific PID.</p></li>
 </ul>
 <h3><a name="TRAFFIC">TRAFFIC</a></h3>
 <ul>
@@ -316,6 +340,12 @@ keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-inde
 keywords: <a href="keyword-index.html#NETWORK">NETWORK</a> <a href="keyword-index.html#TRAFFIC">TRAFFIC</a> <br>
 <p>The tcpdumplike.stp prints out a line for each TCP packet received. Each line includes the source and destination IP addresses, the source and destination ports, and flags.</p></li>
 </ul>
+<h3><a name="TTY">TTY</a></h3>
+<ul>
+<li><a href="io/ttyspy.stp">io/ttyspy.stp</a> - Monitor tty typing.<br>
+keywords: <a href="keyword-index.html#IO">IO</a> <a href="keyword-index.html#TTY">TTY</a> <a href="keyword-index.html#PER-PROCESS">PER-PROCESS</a> <a href="keyword-index.html#MONITOR">MONITOR</a> <br>
+<p>The ttyspy.stp script uses tty_audit hooks to monitor recent typing activity on the system, printing a scrolling record of recent keystrokes, on a per-tty basis.</p></li>
+</ul>
 <h3><a name="USE">USE</a></h3>
 <ul>
 <li><a href="general/graphs.stp">general/graphs.stp</a> - Graphing Disk and CPU Utilization<br>
diff --git a/testsuite/systemtap.examples/keyword-index.txt b/testsuite/systemtap.examples/keyword-index.txt
index b53e776f..eee89e22 100644
--- a/testsuite/systemtap.examples/keyword-index.txt
+++ b/testsuite/systemtap.examples/keyword-index.txt
@@ -222,6 +222,14 @@ keywords: io
   to the specified device occur.
 
 
+io/ttyspy.stp - Monitor tty typing.
+keywords: io tty per-process monitor
+
+  The ttyspy.stp script uses tty_audit hooks to monitor recent typing
+  activity on the system, printing a scrolling record of recent
+  keystrokes, on a per-tty basis.
+
+
 process/sleepingBeauties.stp - Generating Backtraces of Threads Waiting for IO Operations
 keywords: io scheduler backtrace
 
@@ -267,6 +275,16 @@ keywords: memory
   determine where the page faults are occuring.
 
 
+= MONITOR =
+
+io/ttyspy.stp - Monitor tty typing.
+keywords: io tty per-process monitor
+
+  The ttyspy.stp script uses tty_audit hooks to monitor recent typing
+  activity on the system, printing a scrolling record of recent
+  keystrokes, on a per-tty basis.
+
+
 = NETWORK =
 
 network/dropwatch.stp - Watch Where Socket Buffers are Freed in the Kernel
@@ -316,6 +334,14 @@ keywords: network traffic
 
 = PER-PROCESS =
 
+io/ttyspy.stp - Monitor tty typing.
+keywords: io tty per-process monitor
+
+  The ttyspy.stp script uses tty_audit hooks to monitor recent typing
+  activity on the system, printing a scrolling record of recent
+  keystrokes, on a per-tty basis.
+
+
 network/nettop.stp - Periodic Listing of Processes Using Network Interfaces
 keywords: network traffic per-process
 
@@ -337,6 +363,16 @@ keywords: process syscall
   in the report, default 20).
 
 
+process/schedtimes.stp - Track Time Processes Spend in Various States using Tracepoints
+keywords: process scheduler time tracepoint
+
+  The schedtimes.stp script instruments the scheduler to track the
+  amount of time that each process spends running, sleeping, queued,
+  and waiting for io. On exit the script prints out the accumulated
+  time for each state of processes observed.  Optionally, this script
+  can be used with the '-c' or '-x' options to focus on a specific PID.
+
+
 = PROFILING =
 
 io/iostats.stp - List Executables Reading and Writing the Most Data
@@ -417,6 +453,16 @@ keywords: syscall read write time io
 
 = SCHEDULER =
 
+process/schedtimes.stp - Track Time Processes Spend in Various States using Tracepoints
+keywords: process scheduler time tracepoint
+
+  The schedtimes.stp script instruments the scheduler to track the
+  amount of time that each process spends running, sleeping, queued,
+  and waiting for io. On exit the script prints out the accumulated
+  time for each state of processes observed.  Optionally, this script
+  can be used with the '-c' or '-x' options to focus on a specific PID.
+
+
 process/sleepingBeauties.stp - Generating Backtraces of Threads Waiting for IO Operations
 keywords: io scheduler backtrace
 
@@ -606,6 +652,16 @@ keywords: syscall read write time io
   syscalls.
 
 
+process/schedtimes.stp - Track Time Processes Spend in Various States using Tracepoints
+keywords: process scheduler time tracepoint
+
+  The schedtimes.stp script instruments the scheduler to track the
+  amount of time that each process spends running, sleeping, queued,
+  and waiting for io. On exit the script prints out the accumulated
+  time for each state of processes observed.  Optionally, this script
+  can be used with the '-c' or '-x' options to focus on a specific PID.
+
+
 = TRACE =
 
 general/para-callgraph.stp - Callgraph tracing with arguments
@@ -628,6 +684,16 @@ keywords: network tracepoint buffer free
   socket buffers freed at locations in the kernel.
 
 
+process/schedtimes.stp - Track Time Processes Spend in Various States using Tracepoints
+keywords: process scheduler time tracepoint
+
+  The schedtimes.stp script instruments the scheduler to track the
+  amount of time that each process spends running, sleeping, queued,
+  and waiting for io. On exit the script prints out the accumulated
+  time for each state of processes observed.  Optionally, this script
+  can be used with the '-c' or '-x' options to focus on a specific PID.
+
+
 = TRAFFIC =
 
 network/nettop.stp - Periodic Listing of Processes Using Network Interfaces
@@ -647,6 +713,16 @@ keywords: network traffic
   source and destination ports, and flags.
 
 
+= TTY =
+
+io/ttyspy.stp - Monitor tty typing.
+keywords: io tty per-process monitor
+
+  The ttyspy.stp script uses tty_audit hooks to monitor recent typing
+  activity on the system, printing a scrolling record of recent
+  keystrokes, on a per-tty basis.
+
+
 = USE =
 
 general/graphs.stp - Graphing Disk and CPU Utilization
diff --git a/testsuite/systemtap.examples/process/schedtimes.meta b/testsuite/systemtap.examples/process/schedtimes.meta
new file mode 100644
index 00000000..0074731f
--- /dev/null
+++ b/testsuite/systemtap.examples/process/schedtimes.meta
@@ -0,0 +1,13 @@
+title: Track Time Processes Spend in Various States using Tracepoints
+name: schedtimes.stp
+version: 1.0
+author: Jason Baron
+keywords: process scheduler time tracepoint
+subsystem: scheduler
+status: production
+exit: user-controlled
+output: sorted-list
+scope: system-wide
+description: The schedtimes.stp script instruments the scheduler to track the amount of time that each process spends running, sleeping, queued, and waiting for io. On exit the script prints out the accumulated time for each state of processes observed.  Optionally, this script can be used with the '-c' or '-x' options to focus on a specific PID.
+test_check: stap -p4 schedtimes.stp
+test_installcheck: stap schedtimes.stp -c "sleep 1"
diff --git a/testsuite/systemtap.examples/process/schedtimes.stp b/testsuite/systemtap.examples/process/schedtimes.stp
new file mode 100755
index 00000000..e964dd58
--- /dev/null
+++ b/testsuite/systemtap.examples/process/schedtimes.stp
@@ -0,0 +1,154 @@
+#! /usr/bin/env stap
+
+############################################################
+# Schedtimes.stp
+# Author: Jason Baron <jbaron@redhat.com>
+# profiles threads and displays their run times, queued times,
+# wait times, including i/o wait times.
+# Has two modes. When no arguments are given it profiles all
+# threads. Alternatively, you can pass -c "program name"
+############################################################
+
+//constants
+global RUNNING=0, QUEUED=1, SLEEPING=2
+
+global traced_pid
+global run_time, queued_time,  sleep_time, io_wait_time
+global pid_state, pid_names
+global previous_timestamp
+global io_wait_count
+global io_wait_incremented
+
+function get_iowait:long(queue:long)
+{
+  return @cast(queue,"rq","kernel")->nr_iowait->counter;
+}
+
+probe kernel.trace("sched_switch") {
+  previous_pid = $prev->pid;
+  next_pid = $next->pid;
+  if (traced_pid) {
+    if (previous_pid != traced_pid) {
+      previous_pid = 0;
+    }
+    if (next_pid != traced_pid) {
+      next_pid = 0;
+    }
+  }
+  if (previous_pid) {
+    if (!([previous_pid] in pid_state)) {
+      //use this state as entry into state machine
+      previous_timestamp[previous_pid] = gettimeofday_us();
+      pid_names[previous_pid] = kernel_string($prev->comm);
+      if ($prev->state > 0) {
+        pid_state[previous_pid] = SLEEPING;
+      } else if ($prev->state == 0) {
+        pid_state[previous_pid] = QUEUED;
+      } else {
+        printf("unknown transition:\n");
+        printf("pid state: %d our state: %d\n",
+          $prev->state, pid_state[previous_pid]);
+      }
+    } else if (pid_state[previous_pid] == RUNNING) {
+      pid_names[previous_pid] = kernel_string($prev->comm);
+      t = gettimeofday_us();  
+      run_time[previous_pid] += (t - previous_timestamp[previous_pid]);
+      previous_timestamp[previous_pid] = t;
+      if ($prev->state > 0) {
+        if ((get_iowait($rq) - io_wait_count[previous_pid]) > 0)
+          io_wait_incremented[previous_pid] = 1;  
+        pid_state[previous_pid] = SLEEPING;
+      } else if ($prev->state == 0) {
+        pid_state[previous_pid] = QUEUED;
+      } else {
+        printf("unknown transition:\n");
+        printf("pid state: %d our state: %d\n",
+          $prev->state, pid_state[previous_pid]);
+      }
+    } else {
+      printf("unknown transition:\n");
+      printf("%s pid state: %d our state: %d\n",
+           pid_names[previous_pid],
+           $prev->state, pid_state[previous_pid]);
+    }
+  }
+  if (next_pid) {
+    io_wait_count[next_pid] = get_iowait($rq);
+    if (!([next_pid] in pid_state)) {
+      //use this state as entry into state machine
+      previous_timestamp[next_pid] =  gettimeofday_us();
+      pid_state[next_pid] = RUNNING;
+      pid_names[next_pid] = kernel_string($next->comm);
+    } else if (pid_state[next_pid] == QUEUED) {
+      t = gettimeofday_us();
+      queued_time[next_pid] += (t - previous_timestamp[next_pid]);
+      previous_timestamp[next_pid] = t;
+      pid_state[next_pid] = RUNNING;
+      pid_names[next_pid] = kernel_string($next->comm);
+    } else {
+      printf("unknown transition:\n");
+      printf("%s pid state: %d our state: %d\n",
+        pid_names[next_pid],
+        $next->state, pid_state[next_pid]);
+    }
+  }
+}
+
+probe kernel.trace("sched_wakeup") {
+  wakeup_pid = $p->pid;
+  if (traced_pid && (wakeup_pid != traced_pid)) next
+  if ((!$success) && (pid_state[wakeup_pid] != SLEEPING)) next
+  if (!wakeup_pid) next
+
+  if (!([wakeup_pid] in pid_state)) {
+    //use this state as entry into state machine
+    previous_timestamp[wakeup_pid] =  gettimeofday_us();
+    pid_state[wakeup_pid] = QUEUED;
+    pid_names[wakeup_pid] = kernel_string($p->comm);
+  } else if (pid_state[wakeup_pid] == SLEEPING) {
+    t = gettimeofday_us();
+    sleep_time[wakeup_pid] += (t - previous_timestamp[wakeup_pid]);
+    if (io_wait_incremented[wakeup_pid] == 1) {
+      io_wait_time[wakeup_pid] += (t - previous_timestamp[wakeup_pid]);
+      io_wait_incremented[wakeup_pid] = 0;
+    }
+    previous_timestamp[wakeup_pid] = t;
+    pid_state[wakeup_pid] = QUEUED;
+    pid_names[wakeup_pid] = kernel_string($p->comm);
+  } else {
+    printf("unknown transition:\n");
+    printf("pid state: %d our state: %d\n",
+      $p->state, pid_state[wakeup_pid]);
+  }
+}
+
+probe begin {
+  traced_pid = target();
+  if (traced_pid == 0) {
+    printf("all mode\n");
+  } else {
+    printf("target mode\n");
+    printf("traced pid: %d\n", traced_pid);
+  }
+}
+
+probe end {
+  t = gettimeofday_us();
+  foreach (pid in pid_state) {
+    if (pid_state[pid] == SLEEPING)
+      sleep_time[pid] += (t - previous_timestamp[pid]);
+    if (pid_state[pid] == QUEUED)
+      queued_time[pid] += (t - previous_timestamp[pid]);
+    if (pid_state[pid] == RUNNING)
+      run_time[pid] += (t - previous_timestamp[pid]);
+  }
+  printf ("%16s: %6s %10s %10s %10s %10s %10s\n\n",
+         "execname", "pid", "run(us)", "sleep(us)", "io_wait(us)",
+         "queued(us)", "total(us)")
+  foreach (pid+ in run_time) {
+    printf("%16s: %6d %10d %10d %10d %10d %10d\n", 
+      pid_names[pid], pid, run_time[pid], sleep_time[pid], 
+      io_wait_time[pid], queued_time[pid],
+      (run_time[pid] + sleep_time[pid] + queued_time[pid]));
+  }
+}