Merge branch 'master' of ssh://kenistoj@sources.redhat.com/git/systemtap

21 files changed, 403 insertions, 227 deletions

diff --git a/ChangeLog b/ChangeLog
index d62f154f..434a2e9b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2008-09-30  Mark Wielaard  <mjw@redhat.com>
+
+	* tapsets.cxx (literal_stmt_for_local): Check if alternatives can be
+	provided after calling dwarf_formref_die.
+	(literal_stmt_for_return): Likewise.
+
+2008-09-26  Frank Ch. Eigler  <fche@elastic.org>
+
+	* elaborate.cxx (add_global_var_display): Implicitly sort arrays.
+
 2008-09-26  Frank Ch. Eigler  <fche@elastic.org>
 
 	PR 6916
diff --git a/README b/README
index 6b62c772..b7434256 100644
--- a/README
+++ b/README
@@ -10,10 +10,10 @@ See the INSTALL file for generic build instructions.
 Prerequisites:
 
 - linux kernel with kprobes (mainline 2.6.11+ or backport)
-- kernel module build environment (kernel-devel or kernel-smp-devel rpm)
+- kernel module build environment (kernel-devel rpm)
 - kernel debugging information (kernel-debuginfo rpm)
 - C compiler (same as what kernel was compiled with)
-- elfutils with libdwfl for debugging informatin parsing
+- elfutils with libdwfl for debugging information parsing
 - root privileges
 
 Installation steps:
@@ -23,12 +23,12 @@ Installation steps:
 
 Build steps:
 
-- Install the kernel-debuginfo, kernel-[smp-]devel, gcc and libcap-devel
+- Install the kernel-debuginfo, kernel-devel, gcc and dependent
   packages (or see below if you are building your own kernels from source).
-  If available, install your distribution's copy of elfutils and its
-  development headers/libraries.
 
-- Or if desired, download an elfutils source release to build in
+- If available, install your distribution's copy of elfutils and its
+  development headers/libraries.
+  Or if desired, download an elfutils source release to build in
   "bundled mode" (below), and untar it into some new directory.
   Or if desired, build elfutils separately one time, and install
   it to /usr/local.
@@ -45,15 +45,39 @@ Build steps:
   % .../configure [other autoconf options]
   Or, with build it with a bundled internal copy of elfutils:
   % .../configure --with-elfutils=ELFUTILS-SOURCE-DIR [other autoconf options]
+
   Consider configuring with "--enable-dejazilla" to automatically
   contribute to our public test result database.
 
+  Consider configuring with "--prefix=DIRECTORY" to specify an
+  installation directory other than /usr/local.  It can be an ordinary
+  personal directory.
+
   % make all
   % sudo make install
 
-  To run the full test suite:
+  To uninstall systemtap:
+  % sudo make uninstall
+
+- Run systemtap:
+
+  To run systemtap after installation, add $prefix/bin to your $PATH, or
+  refer to $prefix/bin/stap directly.  If you keep your build tree
+  around, you can also use the "stap" binary there.  
+
+  Some samples should be available under $prefix/share/doc/systemtap/examples.
+
+  Normally, run "stap" as root.  If desired, create "stapdev" and
+  "stapusr" entries in /etc/groups.  Any users in "stapdev" will be
+  able to run systemtap as if with root privileges.  Users in "stapusr"
+  can only launch (with "staprun") pre-compiled probe modules (created
+  by "stap -p4 ...") that a system administrator copied under
+  /lib/modules/`uname -r`/systemtap.
+
+  To run the full test suite from the build tree.
   % sudo make installcheck
 
+
 Tips:
 
 - By default, systemtap looks for the debug info in these locations:
diff --git a/doc/SystemTap_Beginners_Guide/en-US/CrossInstrumenting.xml b/doc/SystemTap_Beginners_Guide/en-US/CrossInstrumenting.xml
index cf57f5b2..cc451ec4 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/CrossInstrumenting.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/CrossInstrumenting.xml
@@ -77,7 +77,7 @@
 	<title>Configuring a Host System and Target Systems</title>
 	
 <step>
-		<para>Configure <command>yum</command> on the <emphasis>host system</emphasis> to point to a repository containing the necessary debug RPMs for the <emphasis>target kernels</emphasis>. The following <command>yum</command> repository file (which you can add to <filename>/etc/yum.repos.d/</filename> points to a popular debug RPM repository for Red Hat Enterprise Linux 5:</para>
+		<para>Configure <command>yum</command> on the <emphasis>host system</emphasis> to point to a repository containing the necessary debug RPMs for the <emphasis>target kernels</emphasis>. The following <command>yum</command> repository file (which you can add to <filename>/etc/yum.repos.d/</filename> points to a popular debug RPM repository for i386 systems running Red Hat Enterprise Linux 5:</para>
 <screen>
 [rhel-debuginfo]
 name=Red Hat Enterprise Linux $releasever - $basearch - Debug
@@ -113,18 +113,19 @@ enabled=1
 
 <para>To build the <emphasis>instrumentation module</emphasis>, run the following command on the <emphasis>host system</emphasis> (be sure to specify the appropriate values):</para>
 
-<para><command>stap -r <replaceable>[kernel version]</replaceable> <replaceable>[script]</replaceable> -m <replaceable>[module name]</replaceable></command></para>
+<para><command>stap -r <replaceable>kernel version</replaceable> <replaceable>script</replaceable> -m <replaceable>module name</replaceable></command></para>
 
-<para>Here, <command><replaceable>[kernel version]</replaceable></command> refers to the version of <emphasis>target kernel</emphasis> (including the architecture notation), <command><replaceable>[script]</replaceable></command> refers to the script to be converted into an <emphasis>instrumentation module</emphasis>, and <command><replaceable>[instrumentation name]</replaceable></command> is the desired name of the <emphasis>instrumentation module</emphasis>.</para>
+<para>Here, <command><replaceable>kernel version</replaceable></command> refers to the version of <emphasis>target kernel</emphasis> (including the architecture notation), <command><replaceable>script</replaceable></command> refers to the script to be converted into an <emphasis>instrumentation module</emphasis>, and <command><replaceable>instrumentation name</replaceable></command> is the desired name of the <emphasis>instrumentation module</emphasis>.</para>
 
 <note>
 	<title>Note</title>
-	<para>To determine the version of a running kernel, run <command>uname -r</command>. To determine the architecture notation of a running kernel, run <command>uname -m</command>.</para>
+	<para>To determine the architecture notation of a running kernel, run <command>uname -m</command>.</para>
+<!--	<para>To determine the version of a running kernel, run <command>uname -r</command>. To determine the architecture notation of a running kernel, run <command>uname -m</command>.</para>-->
 </note>	
 
-<para>Once the the <emphasis>instrumentation module</emphasis>is compiled, copy it to the <emphasis>target system</emphasis> and load it using:</para>
+<para>Once the the <emphasis>instrumentation module</emphasis> is compiled, copy it to the <emphasis>target system</emphasis> and load it using:</para>
 
-<para><command>staprun <replaceable>[instrumentation]</replaceable></command></para>
+<para><command>staprun <replaceable>instrumentation</replaceable></command></para>
 
 <para>
 	For example, to create the <emphasis>instrumentation module</emphasis> <filename>module.ko</filename> from the SystemTap script <filename>script.stp</filename> for the <emphasis>target kernel</emphasis> 2.6.25.9-76.fc9 (on i686 architecture), use the following command:
@@ -142,6 +143,9 @@ enabled=1
 <step>
 	<para>Note the version of the target system's kernel on which you wish to use SystemTap. You can do this by logging onto the target system and running <command>uname -r</command> (assuming the system is running the kernel on which you wish to use SystemTap), or by inspecting <filename>/boot</filename>.</para>
 </step>	-->
-		
+<important>
+	<title>Important</title>
+	<para>The <emphasis>host system</emphasis> must be the same architecture as the <emphasis>target system</emphasis> in order for the <emphasis>instrumentation module</emphasis> to work.</para>
+</important>	
 
 	</section>	
\ No newline at end of file
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Installation.xml b/doc/SystemTap_Beginners_Guide/en-US/Installation.xml
new file mode 100644
index 00000000..0ccefa3a
--- /dev/null
+++ b/doc/SystemTap_Beginners_Guide/en-US/Installation.xml
@@ -0,0 +1,95 @@
+<?xml version='1.0'?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+]>
+
+	
+<section id="using-setup">
+		<title>Setup and Installation</title>
+		<remark>
+			required packages, installation thru yum, repos (?); possibly, a script to install all required packages
+		</remark>
+		
+		<remark>
+			notes in ~/Desktop/SystemTap/aug21chatlog and ~/Desktop/SystemTap/noted_wcohenmeeting 
+		</remark>	
+		
+		<para>
+			To deploy SystemTap, you need to install the SystemTap packages along with the corresponding set of <filename>-devel</filename> and <filename>-debuginfo</filename> packages for your kernel. This means that if your system has multiple kernels installed, and you wish to use SystemTap on more than one kernel kernel, you will need to install the <filename>-devel</filename> and <filename>-debuginfo</filename> packages for <emphasis>each</emphasis> of those kernel versions.
+		</para>
+<section id="installprep">
+	<title>Preparing For Installation</title>
+		<para>
+			To view what kernels and kernel versions are installed on your system, check the contents of <filename>/boot</filename>. Each installed kernel/kernel version has a corresponding <filename>vmlinuz-<replaceable>kernel version</replaceable></filename> there.
+		</para>
+<!-- </formalpara>		 -->
+		<para>
+			To determine what kernel your system is currently using, use:
+		</para>
+		
+<screen>
+uname -r
+</screen>
+<!--		
+		<para>
+			Red Hat recommends that you configure <command>yum</command> to point to a repository that houses the necessary debuginfo packages. Otherwise, you will have to manually download the debuginfo packages yourself.
+		</para>-->
+		
+		<para>
+			Most <filename>-debuginfo</filename> packages for Red Hat Enterprise Linux 5 can be found at the following link (under <filename><replaceable>arch</replaceable>/Debuginfo</filename>, where <filename>arch</filename> is the appropriate architecture for your system:
+		</para>
+		
+		<para><ulink url="ftp://ftp.redhat.com/pub/redhat/linux/enterprise/5Client/en/os/">ftp://ftp.redhat.com/pub/redhat/linux/enterprise/5Client/en/os/</ulink></para>
+		
+		<remark>find any other such repository, if only for RHEL</remark>	
+		
+		
+		
+	</section>			
+			
+<section id="installproper">
+	<title>Deploying SystemTap</title>
+	
+<!-- <step> -->
+	<para>Once you've decided which kernels with which you need to use SystemTap with, install the following packages:</para>
+	
+	<itemizedlist>
+		<listitem><para><filename>systemtap</filename></para></listitem>
+		<listitem><para><filename>systemtap-runtime</filename></para></listitem>
+	</itemizedlist>
+	
+	<para>This will install the SystemTap suite of tools.</para> 
+<!-- </step> -->
+<!--<procedure id="deployproper">
+	<title>Deploying SystemTap</title>
+	<step>-->
+		<para>Next, you'll need to download and install the necessary  <filename>-devel</filename> and <filename>-debuginfo</filename> packages for your kernel. The necessary  <filename>-devel</filename> and <filename>-debuginfo</filename> packages for the ordinary "vanilla" kernel are as follows:</para>
+	<!-- Most debugging RPMs for Red Hat Enterprise Linux 5 can be found at the following link:</para>
+
+	<para>	 -->
+<itemizedlist>
+	<listitem><para><filename>kernel-debuginfo</filename></para></listitem>
+	<listitem><para><filename>kernel-debuginfo-common</filename></para></listitem>
+	<listitem><para><filename>kernel-devel</filename></para></listitem>
+</itemizedlist>
+
+<para>For example, if you wish to use SystemTap on kernel version <filename>2.6.18-53.el5</filename>, then you need to download and install the following RPMs:</para>
+
+<example id="debuggingrpmlist">
+	<title>Sample List of Debugging RPMs</title>
+<itemizedlist>
+	<listitem><para><filename>kernel-debuginfo-2.6.18-53.1.13.el5.i686.rpm</filename></para></listitem>
+	<listitem><para><filename>kernel-debuginfo-common-2.6.18-53.1.13.el5.i686.rpm</filename></para></listitem>
+	<listitem><para><filename>kernel-devel-2.6.18-53.1.13.el5.i686.rpm</filename></para></listitem>
+</itemizedlist>
+</example>
+<!--</step>
+</section>
+
+ 		
+<itemizedlist>
+	<listitem><para><filename>systemtap</filename></para></listitem>
+	<listitem><para><filename>systemtap-runtime</filename></para></listitem>
+</itemizedlist>	
+		-->		
+</section>
+</section>
\ No newline at end of file
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Introduction.xml b/doc/SystemTap_Beginners_Guide/en-US/Introduction.xml
index b19f5e48..28abf439 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Introduction.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Introduction.xml
@@ -30,7 +30,7 @@
 	<itemizedlist>
 		<listitem><para>To introduce users to SystemTap, familiarize them with its architecture, and provide setup instructions for all kernel types.</para></listitem>
 		
-		<listitem><para>To provide pre-written SystemTap scripts for monitoring and forensic tasks, along with instructions on how to analyze their output.</para></listitem>  
+		<listitem><para>To provide pre-written SystemTap scripts for monitoring detailed activity in different components of the system, along with instructions on how to analyze their output.</para></listitem>  
 	</itemizedlist>	
 <!-- </formalpara>	 -->
 <remark>above,  Short description on the underlying goals of SystemTap_Beginners_Guide, what we want to teach users.</remark>
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Scripts.xml b/doc/SystemTap_Beginners_Guide/en-US/Scripts.xml
index 160597bf..b928372b 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Scripts.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Scripts.xml
@@ -6,7 +6,7 @@
 	<title>SystemTap Scripts</title>
 	
 	<para>
-		For the most part, SystemTap scripts are the foundation of each SystemTap session. SystemTap scripts instruct SystemTap on what type of information to trap, and what to do once that information is trapped.
+		For the most part, SystemTap scripts are the foundation of each SystemTap session. SystemTap scripts instruct SystemTap on what type of information to collect, and what to do once that information is collected.
 	</para>
 	
 	<para>
@@ -21,9 +21,12 @@
 	<para>A probe's handler is also commonly referred to as a <emphasis>probe body</emphasis>.</para>
 </note>	
 	
-	<para>
-		In terms of application development, using events and handlers is similar to inserting <command>print</command> statements in a program's sequence of commands. These <command>print</command> statements allow you to view a history of commands executed once the program is run. 
+<para>
+	In terms of application development, using events and handlers is similar to inserting diagnostic print statements in a program's sequence of commands. These diagnostic print statements allow you to view a history of commands executed once the program is run. 
 	</para>	
+<!--	<para>
+		In terms of application development, using events and handlers is similar to inserting <command>print</command> statements in a program's sequence of commands. These <command>print</command> statements allow you to view a history of commands executed once the program is run. 
+	</para>	-->
 	
 	<para>
 		SystemTap scripts go one step further by allowing you more flexibility with regard to handlers. Events serve as the triggers for handlers to run; handlers can be specified to trap specified data and print it in a certain manner. 
@@ -36,17 +39,15 @@
 	</para>
 </formalpara>	
 <programlisting>
-probe	<replaceable>[event]</replaceable>,
-<replaceable>[another event]</replaceable>
+probe	<replaceable>event</replaceable>,
 
 {
-	<replaceable>[handler]</replaceable>
-
-	exit()
+	<replaceable>handler</replaceable>
 }
 </programlisting>		
-	<para>The <replaceable>exit()</replaceable> condition is optional; this condition safely terminates the session once the script successfully traps the required information the first time.</para>	
-	
+<!--
+	<para>The <replaceable>exit()</replaceable> condition is optional; this condition safely terminates the session once the script successfully collects the required information the first time.</para>	
+	-->
 <important>
 	<title>Important</title>
 	<para>
@@ -59,18 +60,22 @@ probe	<replaceable>[event]</replaceable>,
 <para>SystemTap events can be broadly classified into two types: <firstterm>synchronous</firstterm> and <firstterm>asynchronous</firstterm>.</para>
 
 <formalpara><title>Synchronous Events</title>
-	<para>A <firstterm>synchronous</firstterm> event occurs when any processor executes an instruction matched by the specification. This gives other events a reference point (or instruction address) from which more contextual data may be available.</para>
+<para>A <firstterm>synchronous</firstterm> event occurs when any process executes an instruction that references a particular location in kernel code. This gives other events a reference point from which more contextual data may be available. </para>
 </formalpara>
 
+<!--<para>A <firstterm>synchronous</firstterm> event occurs when any processor executes an instruction matched by the specification. This gives other events a reference point (or instruction address) from which more contextual data may be available.</para>-->
+
+<!--<para>Synchronous events reference particular locations in kernel code. As a result, when synchronous events are used SystemTap can determine contextual  information regarding the location (such as function parameters).</para>-->
+
 <para>Examples of synchronous events include:</para>
 
 <variablelist>
 <varlistentry>
-	<term>kernel.function("<replaceable>[function]</replaceable>")</term>
+	<term>kernel.function("<replaceable>function</replaceable>")</term>
 	<listitem>
 		<para>The entry to the kernel function <replaceable>function</replaceable>. For example, <command>kernel.function("sys_open")</command> refers to the "event" that occurs when the kernel function <command>sys_open</command> is called by any thread in the system. To specify the <emphasis>return</emphasis> of the kernel function <command>sys_open</command>, append the <command>return</command> string to the event statement; i.e. <command>kernel.function("sys_open").return</command>.</para>
 		
-		<para>When defining functions, you can use asterisk (<command>*</command>) for wildcards. You can also trace the entry/exit of a function in a kernel source file. Consider the following example:</para>
+		<para>When defining functions, you can use asterisk (<literal>*</literal>) for wildcards. You can also trace the entry or exit of a function in a kernel source file. Consider the following example:</para>
 <example id="wildcards"><title>Wildcards and Kernel Source Files in an Event</title>
 <programlisting>
 probe kernel.function("*@net/socket.c") { }
@@ -83,16 +88,16 @@ probe kernel.function("*@net/socket.c").return { }
 </varlistentry>
 
 <varlistentry>
-	<term>syscall.<replaceable>[system_call]</replaceable></term>
+	<term>syscall.<replaceable>system_call</replaceable></term>
 	<listitem>
-		<para>The entry to the system call <replaceable>[system_call]</replaceable>. Similar to <command>kernel.function</command>, appending a <command>return</command> to the statement specifies the exit of the system call. For example, to specify the entry of the system call <command>close</command>, use <command>syscall.close.return</command>.</para>
+		<para>The entry to the system call <replaceable>system_call</replaceable>. Similar to <command>kernel.function</command>, appending a <command>return</command> to the statement specifies the exit of the system call. For example, to specify the entry of the system call <command>close</command>, use <command>syscall.close.return</command>.</para>
 		
 		<para>To identify what system calls are made by a specific program/command, use <command>strace <replaceable>command</replaceable></command>.</para>
 	</listitem>	
 </varlistentry>
 	
 <varlistentry>
-	<term>module("<replaceable>[module]</replaceable>").function("<replaceable>[function]</replaceable>")</term>
+	<term>module("<replaceable>module</replaceable>").function("<replaceable>function</replaceable>")</term>
 	<listitem>
 		<para>Allows you to probe functions within modules. For example:</para>
 		
@@ -108,7 +113,7 @@ probe kernel.function("*@net/socket.c").return { }
 		</para>	
 		
 		<para>
-			A system's loaded modules are typically located in <filename>/lib/modules/<replaceable>[kernel version]</replaceable></filename>, where <replaceable>kernel version</replaceable> refers to the currently loaded kernel. Modules use the filename extension <filename>.ko</filename>. 
+			A system's loaded modules are typically located in <filename>/lib/modules/<replaceable>kernel version</replaceable></filename>, where <replaceable>kernel version</replaceable> refers to the currently loaded kernel. Modules use the filename extension <filename>.ko</filename>. 
 		</para>	
 		
 	</listitem>	
@@ -118,7 +123,8 @@ probe kernel.function("*@net/socket.c").return { }
 	
 <formalpara>
 	<title>Asynchronous Events</title>
-	<para><firstterm>Asynchronous</firstterm> events, on the other hand, do not point to any reference point. This family of probe points consists mainly of counters, timers, and similar constructs.</para>
+	<para><firstterm>Asynchronous</firstterm> events, on the other hand, occur as instructed in the probe itself, rather than waiting for a particular instruction in kernel code to be executed by a process. This family of probe points consists mainly of counters, timers, and similar constructs.</para>
+<!--	<para><firstterm>Asynchronous</firstterm> events, on the other hand, do not point to any reference point. This family of probe points consists mainly of counters, timers, and similar constructs.</para>-->
 </formalpara>
 
 	<para>Examples of asynchronous events include:</para>
@@ -144,36 +150,40 @@ probe kernel.function("*@net/socket.c").return { }
 <varlistentry>
 	<term>timer events</term>
 	<listitem>
-		<para>An event that specifies a handler to be executed "after X number of milliseconds". For example:</para>
+		<para>An event that specifies a handler to be executed every  specified period of time. For example:</para>
 		
 <example id="timer"><title>Using timer.ms</title>		
 <programlisting>
 probe timer.ms(4000)
 {
-	exit()
+	printf("hello world\n")
 }
 </programlisting>
 </example>
 
 <para>
-	<xref linkend="timer"/> is an example of a probe that allows you to terminate the script after 4000 milliseconds. Note that you can also use the following timer events:
+	<xref linkend="timer"/> is an example of a probe that prints <command>hello world</command> every 4000 milliseconds. Note that you can also use the following timer events:
 </para>
 	
 <itemizedlist>
-<listitem><para><command>timer.s(<replaceable>[seconds]</replaceable>)</command></para></listitem>
+<listitem><para><command>timer.s(<replaceable>seconds</replaceable>)</command></para></listitem>
 
-<listitem><para><command>timer.us(<replaceable>[microseconds]</replaceable>)</command></para></listitem>
+<listitem><para><command>timer.us(<replaceable>microseconds</replaceable>)</command></para></listitem>
 
-<listitem><para><command>timer.ns(<replaceable>[nanoseconds]</replaceable>)</command></para></listitem>
+<listitem><para><command>timer.ns(<replaceable>nanoseconds</replaceable>)</command></para></listitem>
 
-<listitem><para><command>timer.hz(<replaceable>[hertz]</replaceable>)</command></para></listitem>
+<listitem><para><command>timer.hz(<replaceable>hertz</replaceable>)</command></para></listitem>
 
-<listitem><para><command>timer.jiffies(<replaceable>[jiffies]</replaceable>)</command></para></listitem>
+<listitem><para><command>timer.jiffies(<replaceable>jiffies</replaceable>)</command></para></listitem>
 
 </itemizedlist>
 <para>
-	When used in conjunction with another probe that traps a large quantity of data, timer events allow you to limit the information your script is collecting (and printing out).
+	When used in conjunction with another probe that collects information that updates periodically, timer events allows you to see how that information changes over time. 
 </para>	
+<!--
+<para>
+	When used in conjunction with another probe that collects a large quantity of data, timer events allow you to limit the information your script is collecting (and printing out). It is also useful for collecting periodically updating information.
+</para>	-->
 		
 	</listitem>	
 </varlistentry>
@@ -231,18 +241,18 @@ probe begin
 <formalpara id="printf">
 	<title>printf ( ) Statements</title>
 <para>
-	The <command>printf ()</command> statement is one of the simplest functions for printing data. <command>printf ()</command> can also be used to trap data using a wide variety of SystemTap handler functions using the following format:
+	The <command>printf ()</command> statement is one of the simplest functions for printing data. <command>printf ()</command> can also be used to display data using a wide variety of SystemTap functions in the following format:
 </para>
 
 </formalpara>
 
 
 <programlisting>
-printf ("<replaceable>[format string]</replaceable>\n", <replaceable>[argument]</replaceable>)
+printf ("<replaceable>format string</replaceable>\n", <replaceable>argument</replaceable>)
 </programlisting>
 
 <para>
-	The <replaceable>[format string]</replaceable> region specifies how <replaceable>[argument]</replaceable> should be displayed. The format string of <xref linkend="helloworld"/> simply instructs SystemTap to print <command>hello world</command>, and contains no arguments. 
+	The <replaceable>format string</replaceable> region specifies how <replaceable>argument</replaceable> should be displayed. The format string of <xref linkend="helloworld"/> simply instructs SystemTap to print <command>hello world</command>, and contains no arguments. 
 </para>
 
 <para>
@@ -288,13 +298,13 @@ hald(2360) open
 </screen>
 
 <formalpara>
-	<title>Handler Functions</title>	
-	<para>SystemTap supports a wide variety of handler functions that can be used as <command>printf ()</command> arguments. <xref linkend="syscall-open"/> uses the handler functions <command>execname()</command> (current process name) and <command>pid()</command> (current process ID).</para>
+	<title>SystemTap Functions</title>	
+	<para>SystemTap supports a wide variety of functions that can be used as <command>printf ()</command> arguments. <xref linkend="syscall-open"/> uses the SystemTap functions <command>execname()</command> (current process name) and <command>pid()</command> (current process ID).</para>
 </formalpara>		
 
-<remark>is "handler function" an appropriate term?</remark>
+<remark>is "handler function" an appropriate term? wcohen: use "SystemTap functions" to match up language in man pages</remark>
 
-	<para>The following is a list of commonly-used handler functions:</para>	
+	<para>The following is a list of commonly-used SystemTap functions:</para>	
 <variablelist>
 
 <varlistentry>
@@ -349,7 +359,7 @@ hald(2360) open
 <varlistentry>
 	<term>thread_indent()</term>
 	<listitem>
-		<para>This particular handler function is quite useful, providing you with a way to better organize your print results. When used with an indentation parameter (for example, <command>-1</command>), it allows the probe to internally store an "indentation counter" for each thread (identified by ID, as in <command>tid</command>). It then returns a string with some generic trace data along with an appropriate number of indentation spaces.</para>
+		<para>This particular function is quite useful, providing you with a way to better organize your print results. When used with an indentation parameter (for example, <command>-1</command>), it allows the probe to internally store an "indentation counter" for each thread (identified by ID, as in <command>tid</command>). It then returns a string with some generic trace data along with an appropriate number of indentation spaces.</para>
 		
 		<para>The generic data included in the returned string includes a timestamp (number of microseconds since the most recent initial indentation), a process name, and the thread ID. This allows you to identify what functions were called, who called them, and the duration of each function call.
 		</para>	
@@ -405,13 +415,13 @@ probe kernel.function("*@net/socket.c").return
 -->	
 </variablelist>		
 
-<para>For more information about supported handler functions, refer to <command>man stapfuncs</command>.</para>
+<para>For more information about supported SystemTap functions, refer to <command>man stapfuncs</command>.</para>
 
-<remark>will need a complete listing of supported handler functions? also, handler function descriptions seem ambiguous, please advise.</remark>
+<remark>will need a complete listing of supported handler functions? also, SystemTap function descriptions seem ambiguous, please advise.</remark>
 
 <!--	
 <para>
-	<replaceable>[variable]</replaceable> can be either <command>%s</command> for strings, or <command>%d</command> for numbers, depending on the <replaceable>[handler function]</replaceable> used. Each <command>printf ()</command> statement can contain multiple <replaceable>[variable]</replaceable>s, with each one corresponding to a matching <replaceable>[handler function]</replaceable>. Multiple <replaceable>[handler function]</replaceable>s are delimited by comma (<command>,</command>). 
+	<replaceable>variable</replaceable> can be either <command>%s</command> for strings, or <command>%d</command> for numbers, depending on the <replaceable>handler function</replaceable> used. Each <command>printf ()</command> statement can contain multiple <replaceable>variable</replaceable>s, with each one corresponding to a matching <replaceable>handler function</replaceable>. Multiple <replaceable>handler function</replaceable>s are delimited by comma (<command>,</command>). 
 </para>	
 
 	<command>printf ()</command> is a SystemTap-supported C statement, and can also trap data using a wide variety
diff --git a/doc/SystemTap_Beginners_Guide/en-US/SystemTap_Beginners_Guide.xml b/doc/SystemTap_Beginners_Guide/en-US/SystemTap_Beginners_Guide.xml
index 8b864720..ccc97915 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/SystemTap_Beginners_Guide.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/SystemTap_Beginners_Guide.xml
@@ -6,8 +6,9 @@
 	<xi:include href="Book_Info.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Preface.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Introduction.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
-	<xi:include href="Understanding_How_SystemTap_Works.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Using_SystemTap.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+	<!--<xi:include href="Installation.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />-->
+	<xi:include href="Understanding_How_SystemTap_Works.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_SystemTap_Scripts.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Errors.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Tips_Tricks.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Tips_Tricks.xml b/doc/SystemTap_Beginners_Guide/en-US/Tips_Tricks.xml
index 0377c499..0327962b 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Tips_Tricks.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Tips_Tricks.xml
@@ -4,11 +4,12 @@
 
 <chapter id="tips-tricks">
 	<title>Tips and Tricks</title>
-	<remark>This chapter covers miscellaneous tips/tricks</remark>
-	
-	<remark>This is a tentative section, and will only be included if content can be provided</remark>
+	<remark>- This chapter covers miscellaneous tips/tricks</remark>
 	
+	<remark>- This is a tentative section, and will only be included if content can be provided</remark>
 	
+	<remark>- add use of exit() here</remark>
 
+	<remark>- add "basic constructs" here; i.e. using if/else, while loops, for loops, examples; if done, consider moving up to before Useful_SystemTap_Scripts.xm, after Understanding_How_SystemTap_Works.xml</remark>
 </chapter>
 
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Understanding_How_SystemTap_Works.xml b/doc/SystemTap_Beginners_Guide/en-US/Understanding_How_SystemTap_Works.xml
index 2794901c..1fb8cfb4 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Understanding_How_SystemTap_Works.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Understanding_How_SystemTap_Works.xml
@@ -35,16 +35,20 @@
 	
 	<step><para>First, SystemTap checks the script against the existing tapset library (normally in <filename>/usr/share/systemtap/tapset/</filename> for any tapsets used.</para></step>
 	
-	<step><para>SystemTap then translates the script to C, running the system C compiler to create a kernel module from it.</para></step>
+	<step><para>SystemTap then translates the script to C, running the system C compiler to create a kernel module from it. The tools that perform this step are contained in the <filename>systemtap</filename> package (refer to <xref linkend="installproper"/> for more information).</para></step>
 	
-	<step><para>SystemTap loads the module, then enables all the probed events by "hooking" those events into the kernel.</para></step>
+	<step><para>SystemTap loads the module, then enables all the probes (events and handlers) in the script. The tools that enable this function are deployed by the <filename>systemtap-runtime</filename> package (refer to <xref linkend="installproper"/> for more information).</para></step>
 		
+<!--	<step><para>SystemTap loads the module, then enables all the probed events by "hooking" those events into the kernel.</para></step>
+		-->
 	<step><para>As the events occur, their corresponding handlers are executed.</para></step>
 	
-	<step><para>Once the SystemTap session is terminated, the hooked events are disconnected from the kernel; afterwards, the kernel module is unloaded.</para></step>    
+	<step><para>Once the SystemTap session is terminated, the probes are disconnected from the kernel; afterwards, the kernel module is unloaded.</para></step>
+	<!--
+	<step><para>Once the SystemTap session is terminated, the hooked events are disconnected from the kernel; afterwards, the kernel module is unloaded.</para></step>-->    
 </procedure>
 
-<para>This sequence is driven from a single command-line program: <command>stap</command>. This program is SystemTap's main front-end tool. For more information about <command>stap</command>, refer to <command>man stap</command> (once SystemTap is set up on your machine).</para>
+<para>This sequence is driven from a single command-line program: <command>stap</command>. This program is SystemTap's main front-end tool. For more information about <command>stap</command>, refer to <command>man stap</command> (once SystemTap is properly installed on your machine).</para>
 
 </section>
 <xi:include href="Scripts.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-disktop.xml b/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-disktop.xml
index a3e3d8a7..0c191cff 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-disktop.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-disktop.xml
@@ -3,7 +3,7 @@
 ]>
 
 
-	<section id="useful-disk-disktop">
+	<section id="disktop">
 		<title>Summarizing Disk Read/Write Traffic</title>
 		
 		
@@ -73,7 +73,7 @@ probe timer.ms(5000) {
 	}
 	/* print top ten I/O */
 	foreach ([process,cmd,userid,parent,action] in io_stat- limit 10)
-	  printf("%8d %8d %8d %25s %8s %4s %12d\n",userid,process,parent,cmd,device[process,cmd,userid,parent,action],action,io_stat[process,cmd,userid,parent,action])
+	  printf("%8d %8d %8d %25s %8s %4s %12d\n",userid,process,parent,cmd,device[process,cmd,userid,parent,action], action,io_stat[process,cmd,userid,parent,action])
 
 	/* clear data */
 	delete io_stat
@@ -115,18 +115,18 @@ probe end{
 	<title><xref linkend="scriptdisktop"/> Sample Output</title>
 <screen>
 [...]
-Mon Sep 29 03:38:28 2008 , Average:  19Kb/sec, Read:       7Kb, Write:     89Kb

+Mon Sep 29 03:38:28 2008 , Average:  19Kb/sec, Read: 7Kb, Write: 89Kb
 
-UID      PID     PPID                       CMD   DEVICE    T        BYTES

-0    26319    26294                   firefox     sda5    W        90229

-0     2758     2757           pam_timestamp_c     sda5    R         8064

-0     2885        1                     cupsd     sda5    W         1678

+UID      PID     PPID                       CMD   DEVICE    T    BYTES
+0    26319    26294                   firefox     sda5    W        90229
+0     2758     2757           pam_timestamp_c     sda5    R         8064
+0     2885        1                     cupsd     sda5    W         1678
 
-Mon Sep 29 03:38:38 2008 , Average:   1Kb/sec, Read:       7Kb, Write:      1Kb

+Mon Sep 29 03:38:38 2008 , Average:   1Kb/sec, Read: 7Kb, Write: 1Kb
 
-UID      PID     PPID                       CMD   DEVICE    T        BYTES

-0     2758     2757           pam_timestamp_c     sda5    R         8064

-0     2885        1                     cupsd     sda5    W         1678

+UID      PID     PPID                       CMD   DEVICE    T    BYTES
+0     2758     2757           pam_timestamp_c     sda5    R         8064
+0     2885        1                     cupsd     sda5    W         1678
 </screen>
 </example>
 
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-iotop.xml b/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-iotop.xml
new file mode 100644
index 00000000..fe249428
--- /dev/null
+++ b/doc/SystemTap_Beginners_Guide/en-US/Useful_Scripts-iotop.xml
@@ -0,0 +1,76 @@
+<?xml version='1.0'?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+]>
+
+	<section id="iotopsect">
+		<title>Periodically Print I/O Activity</title>
+
+	<para>This section describes how to monitor I/O activity on the system.</para>
+	
+<formalpara id="iotop">
+	<title>iotop.stp</title>
+<para>
+<programlisting>
+global reads, writes, total_io
+
+probe kernel.function("vfs_read") {
+    reads[execname()] += $count
+}
+
+probe kernel.function("vfs_write") {
+    writes[execname()] += $count
+}
+
+# print top 10 IO processes every 5 seconds
+probe timer.s(5) {
+    foreach (name in writes)
+        total_io[name] += writes[name]
+    foreach (name in reads)
+        total_io[name] += reads[name]
+    printf ("%16s\t%10s\t%10s\n", "Process", "KB Read", "KB Written")
+    foreach (name in total_io- limit 10)
+        printf("%16s\t%10d\t%10d\n", name,
+               reads[name]/1024, writes[name]/1024)
+    delete reads
+    delete writes
+    delete total_io
+    print("\n")
+}
+</programlisting>
+</para>
+</formalpara>
+
+<para><xref linkend="iotop"/> prints out the top ten executables generating I/O traffic every 5-second interval, in descending order. Its output contains the process name and the amount of data read or written by the process, in KB. For example:</para>
+
+<example id="iotopoutput">
+	<title><xref linkend="iotop"/> Sample Output</title>
+<screen>
+[...]	
+         Process           KB Read      KB Written
+            Xorg             50287               0
+         staprun              3328               0
+ multiload-apple              3039              23
+            sshd               208               1
+        floaters                14              62
+  NetworkManager                15               0
+ gnome-vfs-daemo                 8               0
+           cupsd                 3               3
+        sendmail                 4               0
+   mixer_applet2                 3               0
+
+         Process           KB Read      KB Written
+            Xorg             51886               0
+         staprun              3328               0
+ multiload-apple              3039              23
+            sshd              1344               4
+           snmpd                90               0
+        floaters                15              66
+  NetworkManager                23               0
+      irqbalance                16               0
+ pam_timestamp_c                 9               0
+        sendmail                 4               0
+</screen>
+</example>
+
+<para><xref linkend="iotopoutput"/> displays top I/O writes and reads within a random 10-second interval. Note that <xref linkend="iotop"/> is recursive, as it also detects I/O resulting from SystemTap activity &mdash; <xref linkend="iotopoutput"/> also displays reads done by <command>staprun</command>.</para>
+</section>
\ No newline at end of file
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Useful_SystemTap_Scripts.xml b/doc/SystemTap_Beginners_Guide/en-US/Useful_SystemTap_Scripts.xml
index 51fef276..03e515ff 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Useful_SystemTap_Scripts.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Useful_SystemTap_Scripts.xml
@@ -21,10 +21,14 @@
 	<xi:include href="Useful_Scripts-Disk.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-disktop.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-IO.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+	<xi:include href="Useful_Scripts-iotop.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-Kernel.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-Network.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-Signals.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-Syscalls.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="Useful_Scripts-Others.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
+<!--	
+	<xi:include href="" xmlns:xi="http://www.w3.org/2001/XInclude" />	
+-->
 </chapter>
 
diff --git a/doc/SystemTap_Beginners_Guide/en-US/Using_SystemTap.xml b/doc/SystemTap_Beginners_Guide/en-US/Using_SystemTap.xml
index 01e30fc7..ebf71276 100644
--- a/doc/SystemTap_Beginners_Guide/en-US/Using_SystemTap.xml
+++ b/doc/SystemTap_Beginners_Guide/en-US/Using_SystemTap.xml
@@ -11,99 +11,7 @@
 	<para>
 		This chapter instructs users how to install SystemTap, and provides an introduction on how to run SystemTap scripts.
 	</para>	
-	
-	<section id="using-setup">
-		<title>Setup and Installation</title>
-		<remark>
-			required packages, installation thru yum, repos (?); possibly, a script to install all required packages
-		</remark>
-		
-		<remark>
-			notes in ~/Desktop/SystemTap/aug21chatlog and ~/Desktop/SystemTap/noted_wcohenmeeting 
-		</remark>	
-		
-		<para>
-			To deploy SystemTap, you need to install the SystemTap packages along with the corresponding set of debug RPMs of your kernel. This means that if your system has multiple kernels installed, and you wish to use SystemTap on more than one kernel, you will need to install the debug RPMs for <emphasis>each</emphasis> of those kernels.
-		</para>
-<formalpara>
-	<title>Preparing For Installation</title>
-		<para>
-			To view what kernels and kernel versions are installed on your system, check the contents of <filename>/boot</filename>. Each installed kernel/kernel version has a corresponding <filename>vmlinuz-<replaceable>[kernel version]</replaceable></filename> there.
-		</para>
-</formalpara>		
-		<para>
-			To determine what kernel your system is currently using, use:
-		</para>
-		
-<screen>
-uname -r
-</screen>
-		
-		<para>
-			You will also need to configure <command>yum</command> to point to a repository that houses the necessary debug RPMs. One such repository is:
-		</para>
-		
-		<para><ulink url="ftp://ftp.redhat.com/pub/redhat/linux/enterprise/5Client/en/os/i386/Debuginfo/">ftp://ftp.redhat.com/pub/redhat/linux/enterprise/5Client/en/os/i386/Debuginfo/</ulink></para>
-		
-		<remark>find any other such repository, if only for RHEL</remark>	
-			
-			
-<procedure id="installproper">
-	<title>Deploying SystemTap</title>
-	
-<step>
-	<para>Once you've decided which kernels you need to use SystemTap with, install the following packages:</para>
-	
-	<itemizedlist>
-		<listitem><para><filename>systemtap</filename></para></listitem>
-		<listitem><para><filename>systemtap-runtime</filename></para></listitem>
-	</itemizedlist>
-	
-	<para>This will install the SystemTap suite of tools.</para> 
-</step>
-
-<step>
-	<para>Next, you'll need to download and install the necessary debug RPMs for your kernel. Most debugging RPMs for Red Hat Enterprise Linux 5 can be found at the following link:</para>
-	
-	<para>The necessary debugging RPMs are as follows:</para>
-	
-<itemizedlist>
-	<listitem><para><filename>kernel-debuginfo</filename></para></listitem>
-	<listitem><para><filename>kernel-debuginfo-common</filename></para></listitem>
-	<listitem><para><filename>kernel-devel</filename></para></listitem>
-</itemizedlist>
-
-<para>For example, if you wish to use SystemTap on kernel version <filename>2.6.18-53.el5</filename>, then you need to download the following debugging RPMs:</para>
-
-<example id="debuggingrpmlist">
-	<title>Sample List of Debugging RPMs</title>
-<itemizedlist>
-	<listitem><para><filename>kernel-debuginfo-2.6.18-53.1.13.el5.i686.rpm</filename></para></listitem>
-	<listitem><para><filename>kernel-debuginfo-common-2.6.18-53.1.13.el5.i686.rpm</filename></para></listitem>
-	<listitem><para><filename>kernel-devel-2.6.18-53.1.13.el5.i686.rpm</filename></para></listitem>
-</itemizedlist>
-</example>
-</step>
-
-<step>
-	<para>Install the debugging RPMs using <command>rpm -ivh <replaceable>[RPM]</replaceable></command> or <command>yum localinstall <replaceable>[RPM]</replaceable></command>.</para>
-</step>
-
-<step>
-	<para>
-		Restart the system, loading the appropriate kernel at the <command>grub</command> screen.
-	</para>	
-</step>	
-</procedure>
-
-
-<!--		
-<itemizedlist>
-	<listitem><para><filename>systemtap</filename></para></listitem>
-	<listitem><para><filename>systemtap-runtime</filename></para></listitem>
-</itemizedlist>	
-		-->		
-	</section>
+	<xi:include href="Installation.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />
 	<xi:include href="CrossInstrumenting.xml" xmlns:xi="http://www.w3.org/2001/XInclude" />	
 	<section id="using-usage">
 		<title>Usage</title>
@@ -118,9 +26,18 @@ uname -r
 		<remark>- Tapsets: short intro on usage</remark>
 		-->
 		<para>
-			SystemTap scripts are run through the command <command>stap</command>. <command>stap</command> can run SystemTap scripts from standard input or from file. Below is a list of common options available to you: 
+			SystemTap scripts are run through the command <command>stap</command>. <command>stap</command> can run SystemTap scripts from standard input or from file.
 		</para>
 		
+<important>
+	<title>Important</title>
+<para>Running SystemTap requires root privileges. As such, you need to either log in as root or configure <command>sudo</command> accordingly for specific users who need to run SystemTap (refer to <command>man sudo</command> or <command>man visudo</command> for more information.</para> 
+
+</important>
+		
+		
+		<para> Below is a list of commonly used <command>stap</command> options: </para>
+		
 <variablelist>
 
 <varlistentry>
@@ -133,23 +50,23 @@ uname -r
 </varlistentry>	
 	
 <varlistentry>
-	<term>-I <replaceable>[directory]</replaceable></term>
+	<term>-I <replaceable>directory</replaceable></term>
 	<listitem>
-		<para>Check the script against additional tapsets found in  <command><replaceable>[directory]</replaceable></command>. This is useful if you have additional tapsets that are not included in <filename>/usr/share/systemtap/tapset/</filename>; for more information about tapsets, refer to <xref linkend="understanding-tapsets"/>.</para>
+		<para>Check the script against additional tapsets found in  <command><replaceable>directory</replaceable></command>. This is useful if you have additional tapsets that are not included in <filename>/usr/share/systemtap/tapset/</filename>; for more information about tapsets, refer to <xref linkend="understanding-tapsets"/>.</para>
 	</listitem>	
 </varlistentry>	
 
 <varlistentry>
-	<term>-o <replaceable>[filename]</replaceable></term>
+	<term>-o <replaceable>filename</replaceable></term>
 	<listitem>
-		<para>Sends the standard output to file (<replaceable>[filename]</replaceable>).</para>
+		<para>Sends the standard output to file (<replaceable>filename</replaceable>).</para>
 	</listitem>	
 </varlistentry>	
 
 <varlistentry>
-	<term>-e "<replaceable>[script]</replaceable></term>
+	<term>-e "<replaceable>script</replaceable></term>
 	<listitem>
-		<para>Run given <command><replaceable>[script]</replaceable></command>.</para>
+		<para>Run given <command><replaceable>script</replaceable></command>.</para>
 	</listitem>
 </varlistentry>
 
diff --git a/elaborate.cxx b/elaborate.cxx
index 94bfd1c5..afdc796e 100644
--- a/elaborate.cxx
+++ b/elaborate.cxx
@@ -1234,7 +1234,8 @@ void add_global_var_display (systemtap_session& s)
 	  vardecl* idx_v[idx_count];
 	  // Create a foreach loop
 	  foreach_loop* fe = new foreach_loop;
-	  fe->sort_direction = 0;
+	  fe->sort_direction = -1; // imply decreasing sort on value
+	  fe->sort_column = 0;     // as in   foreach ([a,b,c] in array-) { }
 	  fe->limit = NULL;
 
 	  // Create indices for the foreach loop
diff --git a/runtime/ChangeLog b/runtime/ChangeLog
index 6672dbb5..e22bee26 100644
--- a/runtime/ChangeLog
+++ b/runtime/ChangeLog
@@ -1,3 +1,13 @@
+2008-10-02  Tim Moore  <timoore@redhat.com>
+
+	* addr-map.c (add_bad_addr_entry): Fix bugs in allocating a new
+	table and copying old entries into the new table.
+
+2008-09-30  Tim Moore  <timoore@redhat.com>
+
+	* addr-map.c (add_bad_addr_entry): Rewrite allocation of address
+	table to simplify locking and eliminate a race condition.
+
 2008-09-26  David Smith  <dsmith@redhat.com>
 
 	* task_finder.c (__STP_ATTACHED_TASK_EVENTS): Removed UTRACE_STOP,
diff --git a/runtime/addr-map.c b/runtime/addr-map.c
index 8231b57f..706da454 100644
--- a/runtime/addr-map.c
+++ b/runtime/addr-map.c
@@ -113,64 +113,72 @@ add_bad_addr_entry(unsigned long min_addr, unsigned long max_addr,
   struct addr_map_entry* max_entry = 0;
   struct addr_map_entry* new_entry = 0;
   size_t existing = 0;
-  
+
+  /* Loop allocating memory for a new entry in the map. */
   while (1)
     {
-      size_t old_size;
+      size_t old_size = 0;
       spin_lock(&addr_map_lock);
       old_map = blackmap;
-      if (!blackmap)
-        {
-          existing = 0;
-          old_size = 0;
-        }
-      else
+      if (old_map)
+        old_size = old_map->size;
+      /* Either this is the first time through the loop, or we
+         allocated a map previous time, but someone has come in and
+         added an entry while we were sleeping. */
+      if (!new_map || (new_map && new_map->size < old_size + 1))
         {
-          min_entry = lookup_addr_aux(min_addr, blackmap);
-          max_entry = lookup_addr_aux(max_addr, blackmap);
-          if (min_entry || max_entry)
+          spin_unlock(&addr_map_lock);
+          if (new_map)
             {
-              if (existing_min)
-                *existing_min = min_entry;
-              if (existing_max)
-                *existing_max = max_entry;
-              spin_unlock(&addr_map_lock);
-              return 1;
+              kfree(new_map);
+              new_map = 0;
             }
-          existing = upper_bound(min_addr, old_map);
-          old_size = old_map->size;
+          new_map = kmalloc(sizeof(*new_map)
+                            + sizeof(*new_entry) * (old_size + 1),
+                            GFP_KERNEL);
+          if (!new_map)
+            return -ENOMEM;
+          new_map->size = old_size + 1;
         }
-      spin_unlock(&addr_map_lock);
-      new_map = kmalloc(sizeof(*new_map)
-                       + sizeof(*new_entry) * (old_size + 1),
-                       GFP_KERNEL);
-      if (!new_map)
-        return -ENOMEM;
-      spin_lock(&addr_map_lock);
-      if (blackmap != old_map)
+      else
+        break;
+    }
+  if (!blackmap)
+    {
+      existing = 0;
+    }
+  else
+    {
+      min_entry = lookup_addr_aux(min_addr, blackmap);
+      max_entry = lookup_addr_aux(max_addr, blackmap);
+      if (min_entry || max_entry)
         {
-          kfree(new_map);
+          if (existing_min)
+            *existing_min = min_entry;
+          if (existing_max)
+            *existing_max = max_entry;
           spin_unlock(&addr_map_lock);
-          continue;
-        }
-      new_entry = &new_map->entries[existing];
-      new_entry->min = min_addr;
-      new_entry->max = max_addr;
-      if (old_map)
-        {
-          memcpy(&new_map->entries, old_map->entries,
-                 existing * sizeof(*new_entry));
-          if (old_map->size > existing)
-            memcpy(new_entry + 1, &old_map->entries[existing + 1],
-                   (old_map->size - existing) * sizeof(*new_entry));
+          kfree(new_map);
+          return 1;
         }
-      new_map->size = blackmap->size + 1;
-      blackmap = new_map;
-      spin_unlock(&addr_map_lock);
-      if (old_map)
-        kfree(old_map);
-      return 0;
+      existing = upper_bound(min_addr, old_map);
+    }
+  new_entry = &new_map->entries[existing];
+  new_entry->min = min_addr;
+  new_entry->max = max_addr;
+  if (old_map)
+    {
+      memcpy(&new_map->entries, old_map->entries,
+             existing * sizeof(*new_entry));
+      if (old_map->size > existing)
+        memcpy(new_entry + 1, &old_map->entries[existing],
+               (old_map->size - existing) * sizeof(*new_entry));
     }
+  blackmap = new_map;
+  spin_unlock(&addr_map_lock);
+  if (old_map)
+    kfree(old_map);
+  return 0;
 }
 
 void
diff --git a/tapsets.cxx b/tapsets.cxx
index b1475997..a5a62c7a 100644
--- a/tapsets.cxx
+++ b/tapsets.cxx
@@ -2241,7 +2241,8 @@ struct dwflpp
 	{ 
 	  die = dwarf_formref_die (&attr_mem, &vardie);
           stringstream alternatives;
-          print_members(die,alternatives); 
+          if (die != NULL)
+	    print_members(die,alternatives); 
 	  throw semantic_error("unable to find local '" + local + "'"
                                + " near pc " + lex_cast_hex<string>(pc)
                                + (alternatives.str() == "" ? "" : (" (alternatives:" + alternatives.str () + ")")));
@@ -2317,7 +2318,8 @@ struct dwflpp
 	{ 
 	  die = dwarf_formref_die (&attr_mem, vardie);
           stringstream alternatives;
-          print_members(die,alternatives); 
+          if (die != NULL)
+	    print_members(die,alternatives); 
 	  throw semantic_error("unable to find return value"
                                " near pc " + lex_cast_hex<string>(pc)
                                + (alternatives.str() == "" ? "" : (" (alternatives:" + alternatives.str () + ")")));
diff --git a/testsuite/ChangeLog b/testsuite/ChangeLog
index f3fad077..041df499 100644
--- a/testsuite/ChangeLog
+++ b/testsuite/ChangeLog
@@ -1,3 +1,12 @@
+2008-10-03  Mark Wielaard  <mjw@redhat.com>
+
+	* systemtap.base/global_end.exp: Adjust expected foreach ordering.
+
+2008-10-01  Mark Wielaard  <mjw@redhat.com>
+
+	* semok/thirtythree.stp: Use page->mapping instead of page->inuse
+	as annonymous struct value (also available in older kernels).
+
 2008-09-26  Frank Ch. Eigler  <fche@elastic.org>
 
 	PR 6916.
diff --git a/testsuite/semok/thirtythree.stp b/testsuite/semok/thirtythree.stp
index d5171f66..90070370 100755
--- a/testsuite/semok/thirtythree.stp
+++ b/testsuite/semok/thirtythree.stp
@@ -1,5 +1,5 @@
 #! stap -p2
 # Per bz3016, this should get through the semantic pass without warnings.
 probe kernel.function("do_mpage_readpage") {
-       printf("\n page ->inuse %u",$page->inuse)
+       printf("\n page->mapping %p",$page->mapping)
 }
diff --git a/testsuite/systemtap.base/global_end.exp b/testsuite/systemtap.base/global_end.exp
index cd5c6f83..b6b9fd30 100644
--- a/testsuite/systemtap.base/global_end.exp
+++ b/testsuite/systemtap.base/global_end.exp
@@ -9,13 +9,13 @@ set ok 0
 expect {
     -timeout 180
     -re {one,0x1.*one,0x2.*two,0x1.*two,0x2} { incr ok; exp_continue }
-    -re {alpha."one".1.=0x1} { incr ok; exp_continue }
-    -re {alpha."one".2.=0x2} { incr ok; exp_continue }
-    -re {alpha."two".1.=0x3} { incr ok; exp_continue }
     -re {alpha."two".2.=0x4} { incr ok; exp_continue }
+    -re {alpha."two".1.=0x3} { incr ok; exp_continue }
+    -re {alpha."one".2.=0x2} { incr ok; exp_continue }
+    -re {alpha."one".1.=0x1} { incr ok; exp_continue }
     -re {gamma="abcdefghijklmnopqrstuvwxyz"} { incr ok; exp_continue }
-    -re {iota."one".="eleven"} { incr ok; exp_continue }
     -re {iota."two".="twelve"} { incr ok; exp_continue }
+    -re {iota."one".="eleven"} { incr ok; exp_continue }
     -re {epsilon."one",1. @count=0x4 @min=0x1 @max=0x4 @sum=0xa @avg=0x2} { incr ok; exp_continue }
     -re {epsilon."two",2. @count=0x4 @min=0xa @max=0x28 @sum=0x64 @avg=0x19} { incr ok; exp_continue }
     -re {phi @count=0x4 @min=0x1 @max=0x4 @sum=0xa @avg=0x2} { incr ok; exp_continue }
diff --git a/testsuite/systemtap.base/optim_arridx.exp b/testsuite/systemtap.base/optim_arridx.exp
index bef4d2b4..1f3f4371 100644
--- a/testsuite/systemtap.base/optim_arridx.exp
+++ b/testsuite/systemtap.base/optim_arridx.exp
@@ -58,7 +58,7 @@ end /* <- end */
   # locals
   idx0:long
 {
-foreach ([idx0] in arr3) printf("arr3[%#d]=%#x\\n", idx0, arr3[idx0])
+foreach ([idx0] in arr3-) printf("arr3[%#d]=%#x\\n", idx0, arr3[idx0])
 }
 end /* <- end */
 {