20 files changed, 499 insertions, 143 deletions

diff --git a/ChangeLog b/ChangeLog
index 11ef1787..18f20a08 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,38 @@
+2008-07-29  Dave Brolley  <brolley@redhat.com>
+
+	* Makefile.am (bin_SCRIPTS): add stap-find-servers, stap-start-server,
+	stap-find-or-start-server, stap-stop-server.
+	(EXTRA_DIST): Likewise.
+	(check): Ensure that a compatible systemtap server is running before
+	running the tests, if requested. If we start a sterver, stop it after
+	running the tests.
+	(installcheck): Likewise.
+	* stap-client (parse_options): Specify stdin as 'scripts/-' on the
+	generated command line.
+	(create_request): Use $script_file instead of '-'.
+	(unpack_response): Existence of the systemtap temp directory is
+	optional.
+	(find_and_connect_to_server): Use stap-find-servers and choose_server.
+	(choose_server): Rewritten from match_server. Examine multiple
+	servers.
+	(connect_to_server): New function.
+	(maybe_call_staprun): Check for existence of a module.
+	Use staprun_PATH.
+	(staprun_PATH): New function.
+	(fatal): Call disconnect_from_server.
+	(server_fatal): Likewise.
+	* stap-server: Catch SIGTERM and SIGINT.
+	(create_response): Check for the existence of $tmpdir_stap.
+	(terminate): New function.
+	* stap-serverd: Catch SIGTERM and SIGINT.
+	(listen): Run 'nc | stap-server' in the background and wait for them
+	to finish.
+	(terminate): Renamed from handle_sigint. Kill avahi-publish-service
+	and nc.
+	* systemtap.spec: Add stap-find-servers, stap-start-server,
+	stap-find-or-start-server, stap-stop-server.
+	* Makefile.in: Regenerated.
+
 2008-07-24  Josh Stone  <joshua.i.stone@intel.com>
 
 	* buildrun.cxx (compile_pass): Remove STAPCONF_MODULE_NSECTIONS,
diff --git a/Makefile.am b/Makefile.am
index 3f619584..eaddfd59 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -12,7 +12,7 @@ AM_CXXFLAGS = -Wall -Werror
 
 dist_man_MANS = stap.1 stapprobes.5 stapfuncs.5 stapvars.5 stapex.5 staprun.8 man/stapprobes.iosched.5 man/stapprobes.netdev.5 man/stapprobes.nfs.5 man/stapprobes.nfsd.5 man/stapprobes.pagefault.5 man/stapprobes.process.5 man/stapprobes.rpc.5 man/stapprobes.scsi.5 man/stapprobes.signal.5 man/stapprobes.socket.5 man/stapprobes.tcp.5 man/stapprobes.udp.5
 bin_PROGRAMS = stap staprun
-bin_SCRIPTS = stap-client stap-server stap-serverd
+bin_SCRIPTS = stap-client stap-server stap-serverd stap-find-servers stap-start-server stap-find-or-start-server stap-stop-server
 stap_SOURCES = main.cxx \
 	parse.cxx staptree.cxx elaborate.cxx translate.cxx \
 	tapsets.cxx buildrun.cxx loc2c.c hash.cxx mdfour.c \
@@ -155,7 +155,7 @@ EXTRA_DIST = auto_free.h buildrun.h elaborate.h loc2c.h session.h \
 	     testsuite systemtap.spec runtime tapset \
 	     dwarf_wrappers.h \
 	     git_version.h git_version.sh \
-	     stap-client stap-server stap-serverd \
+	     stap-client stap-server stap-serverd stap-find-servers stap-start-server stap-find-or-start-server stap-stop-server \
 	     systemtap.spec
 
 EXAMPLE_SOURCE_DIR = $(srcdir)/testsuite/systemtap.examples
@@ -250,18 +250,54 @@ SUBDIRS = testsuite doc
 
 check:
 	SRCDIR=`cd $(srcdir); pwd`; \
-	$(MAKE) -C testsuite check SYSTEMTAP_RUNTIME=$$SRCDIR/runtime SYSTEMTAP_TAPSET=$$SRCDIR/tapset LD_LIBRARY_PATH=$(PWD)/lib-elfutils:$(PWD)/lib-elfutils/systemtap SYSTEMTAP_PATH=$(PWD) RUNTESTFLAGS="$(RUNTESTFLAGS)"
+	BUILDDIR=`cd $(builddir); pwd`; \
+	need_server=0; \
+	(echo "X$(EXTRA_TOOL_OPTS)" | grep -q server) && need_server=1; \
+	if test $$need_server = 1; then \
+	  echo "Testing using a systemtap server"; \
+	  need_server=0; \
+	  server_pid=`stap-find-or-start-server` || need_server=1; \
+	  if test $$need_server = 1; then \
+	      echo "Cannot find or start a systemtap server"; \
+	      exit -1; \
+	  fi; \
+	  client_path="$$BUILDDIR/testsuite/net"; \
+	  mkdir -p $$BUILDDIR/testsuite/net; \
+	  cp -p $$SRCDIR/stap-client $$BUILDDIR/testsuite/net/stap; \
+	fi; \
+	$(MAKE) -C testsuite check SYSTEMTAP_RUNTIME=$$SRCDIR/runtime SYSTEMTAP_TAPSET=$$SRCDIR/tapset LD_LIBRARY_PATH=$(PWD)/lib-elfutils:$(PWD)/lib-elfutils/systemtap SYSTEMTAP_PATH="$$client_path:$(PWD)" RUNTESTFLAGS="$(RUNTESTFLAGS)"; \
+	if test "X$$server_pid" != "X"; then \
+	  stap-stop-server $$server_pid; \
+	fi
 
 installcheck:
+	BUILDDIR=`cd $(builddir); pwd`; \
 	if test \! -e $(DESTDIR)$(bindir)/stap; then \
 	  echo $(DESTDIR)$(bindir)/stap doesn\'t exist, run make install; \
 	  exit -1; \
-	fi
+	fi; \
 	if test $(builddir)/stap -nt $(DESTDIR)$(bindir)/stap; then \
 	  echo "$(DESTDIR)$(bindir)/stap is not recent, run make install"; \
 	  exit -1; \
+	fi; \
+	need_server=0; \
+	(echo "X$(EXTRA_TOOL_OPTS)" | grep -q server) && need_server=1; \
+	if test $$need_server = 1; then \
+	  echo "Testing using a systemtap server"; \
+	  need_server=0; \
+	  server_pid=`stap-find-or-start-server` || need_server=1; \
+	  if test $$need_server = 1; then \
+	      echo "Cannot find or start a systemtap server"; \
+	      exit -1; \
+	  fi; \
+	  client_path=":$$BUILDDIR/testsuite/net"; \
+	  mkdir -p $$BUILDDIR/testsuite/net; \
+	  cp -p $(DESTDIR)$(bindir)/stap-client $$BUILDDIR/testsuite/net/stap; \
+	fi; \
+	$(MAKE) -C testsuite installcheck RUNTESTFLAGS="$(RUNTESTFLAGS)" EXTRA_SYSTEMTAP_PATH="$(EXTRA_SYSTEMTAP_PATH)$$client_path"; \
+	if test "X$$server_pid" != "X"; then \
+	  stap-stop-server $$server_pid; \
 	fi
-	$(MAKE) -C testsuite installcheck RUNTESTFLAGS="$(RUNTESTFLAGS)"
 
 rpm:	systemtap.spec dist
 	rpmbuild --define "_sourcedir $(PWD)/" -ba systemtap.spec
diff --git a/Makefile.in b/Makefile.in
index cc8d4fd3..e3b6f146 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -275,7 +275,7 @@ AM_CPPFLAGS = -DBINDIR='"$(bindir)"' -DPKGDATADIR='"${pkgdatadir}"' -DPKGLIBDIR=
 AM_CFLAGS = -D_GNU_SOURCE -fexceptions -Wall -Werror -Wunused -Wformat=2 -W
 AM_CXXFLAGS = -Wall -Werror
 dist_man_MANS = stap.1 stapprobes.5 stapfuncs.5 stapvars.5 stapex.5 staprun.8 man/stapprobes.iosched.5 man/stapprobes.netdev.5 man/stapprobes.nfs.5 man/stapprobes.nfsd.5 man/stapprobes.pagefault.5 man/stapprobes.process.5 man/stapprobes.rpc.5 man/stapprobes.scsi.5 man/stapprobes.signal.5 man/stapprobes.socket.5 man/stapprobes.tcp.5 man/stapprobes.udp.5
-bin_SCRIPTS = stap-client stap-server stap-serverd
+bin_SCRIPTS = stap-client stap-server stap-serverd stap-find-servers stap-start-server stap-find-or-start-server stap-stop-server
 stap_SOURCES = main.cxx \
 	parse.cxx staptree.cxx elaborate.cxx translate.cxx \
 	tapsets.cxx buildrun.cxx loc2c.c hash.cxx mdfour.c \
@@ -334,7 +334,7 @@ EXTRA_DIST = auto_free.h buildrun.h elaborate.h loc2c.h session.h \
 	     testsuite systemtap.spec runtime tapset \
 	     dwarf_wrappers.h \
 	     git_version.h git_version.sh \
-	     stap-client stap-server stap-serverd \
+	     stap-client stap-server stap-serverd stap-find-servers stap-start-server stap-find-or-start-server stap-stop-server \
 	     systemtap.spec
 
 EXAMPLE_SOURCE_DIR = $(srcdir)/testsuite/systemtap.examples
@@ -1636,18 +1636,54 @@ uninstall-local:
 
 check:
 	SRCDIR=`cd $(srcdir); pwd`; \
-	$(MAKE) -C testsuite check SYSTEMTAP_RUNTIME=$$SRCDIR/runtime SYSTEMTAP_TAPSET=$$SRCDIR/tapset LD_LIBRARY_PATH=$(PWD)/lib-elfutils:$(PWD)/lib-elfutils/systemtap SYSTEMTAP_PATH=$(PWD) RUNTESTFLAGS="$(RUNTESTFLAGS)"
+	BUILDDIR=`cd $(builddir); pwd`; \
+	need_server=0; \
+	(echo "X$(EXTRA_TOOL_OPTS)" | grep -q server) && need_server=1; \
+	if test $$need_server = 1; then \
+	  echo "Testing using a systemtap server"; \
+	  need_server=0; \
+	  server_pid=`stap-find-or-start-server` || need_server=1; \
+	  if test $$need_server = 1; then \
+	      echo "Cannot find or start a systemtap server"; \
+	      exit -1; \
+	  fi; \
+	  client_path="$$BUILDDIR/testsuite/net"; \
+	  mkdir -p $$BUILDDIR/testsuite/net; \
+	  cp -p $$SRCDIR/stap-client $$BUILDDIR/testsuite/net/stap; \
+	fi; \
+	$(MAKE) -C testsuite check SYSTEMTAP_RUNTIME=$$SRCDIR/runtime SYSTEMTAP_TAPSET=$$SRCDIR/tapset LD_LIBRARY_PATH=$(PWD)/lib-elfutils:$(PWD)/lib-elfutils/systemtap SYSTEMTAP_PATH="$$client_path:$(PWD)" RUNTESTFLAGS="$(RUNTESTFLAGS)"; \
+	if test "X$$server_pid" != "X"; then \
+	  stap-stop-server $$server_pid; \
+	fi
 
 installcheck:
+	BUILDDIR=`cd $(builddir); pwd`; \
 	if test \! -e $(DESTDIR)$(bindir)/stap; then \
 	  echo $(DESTDIR)$(bindir)/stap doesn\'t exist, run make install; \
 	  exit -1; \
-	fi
+	fi; \
 	if test $(builddir)/stap -nt $(DESTDIR)$(bindir)/stap; then \
 	  echo "$(DESTDIR)$(bindir)/stap is not recent, run make install"; \
 	  exit -1; \
+	fi; \
+	need_server=0; \
+	(echo "X$(EXTRA_TOOL_OPTS)" | grep -q server) && need_server=1; \
+	if test $$need_server = 1; then \
+	  echo "Testing using a systemtap server"; \
+	  need_server=0; \
+	  server_pid=`stap-find-or-start-server` || need_server=1; \
+	  if test $$need_server = 1; then \
+	      echo "Cannot find or start a systemtap server"; \
+	      exit -1; \
+	  fi; \
+	  client_path=":$$BUILDDIR/testsuite/net"; \
+	  mkdir -p $$BUILDDIR/testsuite/net; \
+	  cp -p $(DESTDIR)$(bindir)/stap-client $$BUILDDIR/testsuite/net/stap; \
+	fi; \
+	$(MAKE) -C testsuite installcheck RUNTESTFLAGS="$(RUNTESTFLAGS)" EXTRA_SYSTEMTAP_PATH="$(EXTRA_SYSTEMTAP_PATH)$$client_path"; \
+	if test "X$$server_pid" != "X"; then \
+	  stap-stop-server $$server_pid; \
 	fi
-	$(MAKE) -C testsuite installcheck RUNTESTFLAGS="$(RUNTESTFLAGS)"
 
 rpm:	systemtap.spec dist
 	rpmbuild --define "_sourcedir $(PWD)/" -ba systemtap.spec
diff --git a/stap-client b/stap-client
index 8c3607ae..033ef4b3 100755
--- a/stap-client
+++ b/stap-client
@@ -169,8 +169,13 @@ function parse_options {
 
     # If the script file was given and it's not '-', then replace it with its
     # client-temp-name in the command string.
-    if test "X$script_file" != "X" -a "$script_file" != "-"; then
-	local local_name=`generate_client_temp_name $script_file`
+    if test "X$script_file" != "X"; then
+	local local_name
+	if test "$script_file" != "-"; then
+	    local_name=`generate_client_temp_name $script_file`
+	else
+	    local_name=$script_file
+	fi
 	cmdline=`echo $cmdline | sed s,$script_file,script/$local_name,`
     fi
 }
@@ -257,7 +262,7 @@ function process_R {
 # Include the given file or directory in the client's temporary
 # tree to be sent to the server.
 function include_file_or_directory {
-    # Add a symbolic link of the named directory to our temporary directory
+    # Add a symbolic link of the named file or directory to our temporary directory
     local local_name=`generate_client_temp_name $2`
     mkdir -p $tmpdir_client/$1/`dirname $local_name` || \
 	fatal "ERROR: could not create $tmpdir_client/$1/`dirname $local_name`"
@@ -292,7 +297,7 @@ function create_request {
 	if test "$script_file" = "-"; then
 	    mkdir -p $tmpdir_client/script || \
 		fatal "ERROR: cannot create temporary diectory " $tmpdir_client/script
-	    cat > $tmpdir_client/script/-
+	    cat > $tmpdir_client/script/$script_file
 	else
 	    include_file_or_directory script $script_file > /dev/null
 	fi
@@ -343,9 +348,6 @@ function package_request {
 #   client -> "request:"
 #   server -> "ready:"
 #   client -> $tar_client
-#
-# $tmpdir_client is provided on the request so that the server knows what
-# the tar file will expand to.
 function send_request {
     echo "request:" >&3
     # Get the server's response.
@@ -404,7 +406,7 @@ function unpack_response {
     tar -xzf $tar_server || \
 	fatal "ERROR: Unpacking of server response, $tar_server, failed"
 
-    # Identify the server's request tree. The tar file should have expanded
+    # Identify the server's response tree. The tar file should have expanded
     # into a single directory named to match $tmpdir_prefix_server.??????
     # which should now be the only item in the current directory.
     test "`ls | wc -l`" = 1 || \
@@ -419,34 +421,39 @@ function unpack_response {
     # Check the contents of the expanded directory. It should contain:
     # 1) a file called stdout
     # 2) a file called stderr
-    # 3) a directory named to match stap??????
-    test "`ls $tmpdir_server | wc -l`" = 3 || \
-	fatal "ERROR: Wrong number of files after expansion of server's tar file"
+    # 3) optionally a directory named to match stap??????
+    local num_files=`ls $tmpdir_server | wc -l`
+    test $num_files = 3 -o $num_files = 2 || \
+	fatal "ERROR: Wrong number of files in server's temp directory"
     test -f $tmpdir_server/stdout || \
 	fatal "ERROR: `pwd`/$tmpdir_server/stdout does not exist or is not a regular file"
     test -f $tmpdir_server/stderr || \
 	fatal "ERROR: `pwd`/$tmpdir_server/stderr does not exist or is not a regular file"
 
+    # See if there is a systemtap temp directory
     tmpdir_stap=`ls $tmpdir_server | grep stap`
     tmpdir_stap=`expr "$tmpdir_stap" : "\\\(stap......\\\)"`
-    test "X$tmpdir_stap" = "X" && \
-	fatal "ERROR: `pwd`/$tmpdir_server/stap?????? does not exist"
-    test -d $tmpdir_server/$tmpdir_stap || \
-	fatal "ERROR: `pwd`/$tmpdir_server/$tmpdir_stap is not a directory"
-
-    # Move the systemtap temp directory to a local temp location, if -k
-    # was specified.
-    if test $keep_temps = 1; then
-	local local_tmpdir_stap=`mktemp -dt stapXXXXXX` || \
-	    fatal "ERROR: cannot create temporary directory " $local_tmpdir_stap
-	mv $tmpdir_server/$tmpdir_stap/* $local_tmpdir_stap 2>/dev/null
-	rm -fr $tmpdir_server/$tmpdir_stap
-
-	# Correct the name of the temp directory in the server's stderr output
-	sed -i "s,^Keeping temporary directory.*,Keeping temporary directory \"$local_tmpdir_stap\"," $tmpdir_server/stderr
-	tmpdir_stap=$local_tmpdir_stap
-    else
-	tmpdir_stap=`pwd`/$tmpdir_stap
+    if test "X$tmpdir_stap" != "X"; then
+	test -d $tmpdir_server/$tmpdir_stap || \
+	    fatal "ERROR: `pwd`/$tmpdir_server/$tmpdir_stap is not a directory"
+
+        # Move the systemtap temp directory to a local temp location, if -k
+        # was specified.
+	if test $keep_temps = 1; then
+	    local local_tmpdir_stap=`mktemp -dt stapXXXXXX` || \
+		fatal "ERROR: cannot create temporary directory " $local_tmpdir_stap
+	    mv $tmpdir_server/$tmpdir_stap/* $local_tmpdir_stap 2>/dev/null
+	    rm -fr $tmpdir_server/$tmpdir_stap
+
+	    # Correct the name of the temp directory in the server's stderr output
+	    sed -i "s,^Keeping temporary directory.*,Keeping temporary directory \"$local_tmpdir_stap\"," $tmpdir_server/stderr
+	    tmpdir_stap=$local_tmpdir_stap
+	else
+	    tmpdir_stap=`pwd`/$tmpdir_stap
+	fi
+
+        # Make sure we own the systemtap temp directory if we are root.
+	test $EUID = 0 && chown $EUID:$EUID $tmpdir_stap
     fi
 
     # Move the contents of the server's tmpdir down one level to the
@@ -454,95 +461,57 @@ function unpack_response {
     mv $tmpdir_server/* . 2>/dev/null
     rm -fr $tmpdir_server
     tmpdir_server=`pwd`
-
-    # Make sure we own the systemtap temp directory if we are root.
-    test $EUID = 0 && chown $EUID:$EUID $tmpdir_stap
 }
 
 # function: find_and_connect_to_server
 #
-# Find and establish connection with a compatibale stap server.
+# Find and establish connection with a compatible stap server.
 function find_and_connect_to_server {
-    # Find a server
-    server=`avahi-browse $avahi_service_tag --terminate -r 2>/dev/null | match_server`
-    port=`expr "$server" : '[^/]*/\(.*\)'`
-    server=`expr "$server" : '\([^/]*\)/.*'`
-
-    test "X$server" != "X" || \
-	fatal "ERROR: cannot find a server"
-
-    test "X$port" != "X" || \
-	fatal "ERROR: server port not provided"
-
-    # Open a connection to the server
-    if ! exec 3<> /dev/tcp/$server/$port; then
-	fatal "ERROR: cannot connect to server at /dev/tcp/$server/$port"
-    fi
+    cd $tmpdir_client
+    # Use a temp file here instead of a pipeline so that the side effects
+    # of choose_server are seen by the rest of this script.
+    stap-find-servers > servers
+    choose_server < servers
+    rm -fr servers
 }
 
-# function: match_server
+# function: choose_server
 #
-# Find a suitable server using the avahi-browse output provided on stdin.
-function match_server {
-    local server_ip
-
-    # Loop over the avahi service descriptors.
-    while read
+# Examine each line from stdin and attempt to connect to each server
+# specified until successful.
+function choose_server {
+    local num_servers=0
+    local name
+    while read name server port remain
     do
-	# Examine the next service descriptor
-	# Is it a stap server?
-	(echo $REPLY | grep -q "^=.*_stap") || continue
-	
-	# Get the details of the service
-	local service_tag equal data
-	while read service_tag equal service_data
-	do
-	    case $service_tag in
-		'=' )
-		    break ;;
-		hostname )
-		    server_name=`expr "$service_data" : '\[\([^]]*\)\]'`
-		    ;;
-		address )
-		    # Sometimes (seems random), avahi-resolve-host-name resolves a local server to its
-		    # hardware address rather its ip address. Keep trying until we get
-		    # an ip address.
-		    server_ip=`expr "$service_data" : '\[\([^]]*\)\]'`
-		    local attempt
-		    for ((attempt=0; $attempt < 5; ++attempt))
-		    do
-			server_ip=`expr "$server_ip" : '^\([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\)$'`
-			if test "X$server_ip" != "X"; then
-			    break
-			fi
-		        # Resolve the server.domain to an ip address.
-			server_ip=`avahi-resolve-host-name $hostname`
-			server_ip=`expr "$server_ip" : '.*	\(.*\)$'`
-		    done
-		    ;;
-		port )
-		    port=`expr "$service_data" : '\[\([^]]*\)\]'`
-		    ;;
-		txt )
-		    sysinfo_server=`expr "$service_data" : '\[\"\([^]]*\)\"\]'`
-		    ;;
-		* )
-		    break ;;
-	    esac
-	done
+	num_servers=$(($num_servers + 1))
+
+	if test "X$server" = "X"; then
+	    fatal "ERROR: server ip address not provided"
+	fi
 
-	# It is a stap server, but is it compatible?
-	if test "$sysinfo_server" != "`client_sysinfo`"; then
-	    server_ip=
-	    continue
+	if test "X$port" = "X"; then
+	    fatal "ERROR: server port not provided"
 	fi
 
-	if test "X$server_ip" != "X"; then
-	    break
+	if connect_to_server $server $port; then
+	    return 0
 	fi
     done
 
-    echo $server_ip/$port
+    if test num_servers = 0; then
+	fatal "ERROR: cannot find a server"
+    fi
+
+    fatal "ERROR: unable to connect to a server"
+}
+
+# function: connect_to_server IP PORT
+#
+# Establish connection with the given server
+function connect_to_server {
+    # Open a connection to the server
+    exec 3<> /dev/tcp/$1/$2
 }
 
 # function: disconnect_from_server
@@ -568,15 +537,40 @@ function stream_output {
 # Call staprun using the module returned from the server, if requested.
 function maybe_call_staprun {
     if test $p_phase = 5; then
+        # Can't call staprun without a module.
+	test "X$tmpdir_stap" = "X" && return
+
+	local mod_name=`ls $tmpdir_stap | grep '.ko$'`
+	test "X$mod_name" != "X" || \
+	    fatal "ERROR: no module to run in $tmpdir_stap"
+
+	# We have a module. Try to run it
 	for ((--v_level; $v_level > 0; --v_level))
 	do
 	    staprun_opts="$staprun_opts -v"
 	done
-	staprun $staprun_opts \
+	PATH=`staprun_PATH` staprun $staprun_opts \
 	    $tmpdir_stap/`ls $tmpdir_stap | grep '.ko$'`
     fi
 }
 
+# function: staprun_PATH
+#
+# Compute a PATH suitable for running staprun.
+function staprun_PATH {
+    # staprun may invoke 'stap'. So we can use the current PATH if we were
+    # not invoked as 'stap' or we are not the first 'stap' on the PATH.
+    local first_stap=`which stap 2>/dev/null`
+    if test `which $0 2>/dev/null` != $first_stap; then
+	echo "$PATH"
+	return
+    fi
+
+    # Otherwise, remove the PATH component where we live from the PATH
+    local PATH_component=`dirname $first_stap`
+    echo $PATH | sed "s,$PATH_component,,g"
+}
+
 # function: check_server_error SERVER_RESPONSE
 #
 # Check the given server response for an error message.
@@ -591,6 +585,7 @@ function check_server_error {
 # Prints its arguments to stderr and exits
 function fatal {
     echo $0: "$@" >&2
+    disconnect_from_server
     cleanup
     exit 1
 }
@@ -602,6 +597,7 @@ function fatal {
 function server_fatal {
     echo $0: "$@" >&2
     cat <&3 >&2
+    disconnect_from_server
     cleanup
     exit 1
 }
diff --git a/stap-find-or-start-server b/stap-find-or-start-server
new file mode 100755
index 00000000..0ea0ef43
--- /dev/null
+++ b/stap-find-or-start-server
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Find or start a systemtap server
+#
+# Copyright (C) 2008 Red Hat Inc.
+#
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+# This script attempts to find a systemtap server. If one is found, it
+# simply exits with 0.
+#
+# Otherwise, it attempts to start a server. If succesful, it echoes the
+# process id and exits with 0.
+#
+# Otherwise, it exits with 1
+
+# Is there a server available?
+stap-find-servers >/dev/null 2>&1 && exit 0
+
+# No server available, try to start one.
+pid=`stap-start-server`
+if test $? = 0; then
+  echo $pid
+  # Make sure the server is started
+  for ((attempt=0; $attempt < 5; ++attempt))
+  do
+    stap-find-servers >/dev/null 2>&1 && exit 0
+    sleep 1
+  done
+fi
+
+# Could not find or start a server
+exit 1
diff --git a/stap-find-servers b/stap-find-servers
new file mode 100755
index 00000000..9e7b633d
--- /dev/null
+++ b/stap-find-servers
@@ -0,0 +1,128 @@
+#!/bin/bash
+
+# Find compile servers for systemtap
+#
+# Copyright (C) 2008 Red Hat Inc.
+#
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+# This script uses avahi to find systemtap compile servers on the local
+# network. Information about each server found is printed to stdout.
+
+#-----------------------------------------------------------------------------
+# Helper functions.
+#-----------------------------------------------------------------------------
+# function: configuration
+function configuration {
+    avahi_service_tag=_stap._tcp
+}
+
+# function: initialization
+function initialization {
+    rc=1 # not found yet
+    if test "X$1" = "X--all"; then
+	find_all=1
+    else
+	find_all=0
+    fi
+}
+
+# function: find_and_connect_to_server
+#
+# Find and establish connection with a compatibale stap server.
+function find_servers {
+    # Find a server
+    avahi-browse $avahi_service_tag --terminate -r 2>/dev/null | match_server
+    rc=$?
+}
+
+# function: match_server
+#
+# Find a suitable server using the avahi-browse output provided on stdin.
+function match_server {
+    local server_ip
+    local server_name
+    local server_sysinfo
+    local server_port
+    local rc=1 # not found yet
+
+    # Loop over the avahi service descriptors.
+    while read
+    do
+	# Examine the next service descriptor
+	# Is it a stap server?
+	(echo $REPLY | grep -q "^=.*_stap") || continue
+	
+	# Get the details of the service
+	local service_tag equal data
+	while read service_tag equal service_data
+	do
+	    case $service_tag in
+		'=' )
+		    break ;;
+		hostname )
+		    server_name=`expr "$service_data" : '\[\([^]]*\)\]'`
+		    ;;
+		address )
+		    # Sometimes (seems random), avahi-resolve-host-name resolves a local server to its
+		    # hardware address rather its ip address. Keep trying until we get
+		    # an ip address.
+		    server_ip=`expr "$service_data" : '\[\([^]]*\)\]'`
+		    local attempt
+		    for ((attempt=0; $attempt < 5; ++attempt))
+		    do
+			server_ip=`expr "$server_ip" : '^\([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*\)$'`
+			if test "X$server_ip" != "X"; then
+			    break
+			fi
+		        # Resolve the server.domain to an ip address.
+			server_ip=`avahi-resolve-host-name $hostname`
+			server_ip=`expr "$server_ip" : '.*	\(.*\)$'`
+		    done
+		    ;;
+		port )
+		    server_port=`expr "$service_data" : '\[\([^]]*\)\]'`
+		    ;;
+		txt )
+		    server_sysinfo=`expr "$service_data" : '\[\"\([^]]*\)\"\]'`
+		    ;;
+		* )
+		    break ;;
+	    esac
+	done
+
+	# It is a stap server, but is it compatible?
+	if test $find_all = 0 -a "$server_sysinfo" != "`client_sysinfo`"; then
+	    continue
+	fi
+
+	# It's compatible, or we're finding all servers. Print a summary line
+	echo $server_name $server_ip $server_port "'$server_sysinfo'"
+	rc=0
+    done
+
+    exit $rc
+}
+
+# function client_sysinfo
+#
+# Generate the client's sysinfo and echo it to stdout
+function client_sysinfo {
+    if test "X$sysinfo_client" = "X"; then
+	# Add some info from uname
+	sysinfo_client="`uname -rvm`"
+    fi
+    echo $sysinfo_client
+}
+
+#-----------------------------------------------------------------------------
+# Beginning of main line execution.
+#-----------------------------------------------------------------------------
+configuration
+initialization "$@"
+find_servers
+
+exit $rc
diff --git a/stap-server b/stap-server
index 16ffe8ee..7df324b4 100755
--- a/stap-server
+++ b/stap-server
@@ -13,6 +13,9 @@
 # contained in the unpacked tree to build the requested systemtap kernel module.
 # This module is then written to stdout.
 
+# Catch ctrl-c and other termination signals
+trap 'terminate' SIGTERM SIGINT
+
 #-----------------------------------------------------------------------------
 # Helper functions.
 #-----------------------------------------------------------------------------
@@ -312,6 +315,7 @@ function call_stap {
     else
 	server_p_phase=$p_phase
     fi
+
     eval stap $cmdline -k -p $server_p_phase \
 	>>  $tmpdir_server/stdout \
 	2>> $tmpdir_server/stderr
@@ -328,19 +332,21 @@ function create_response {
     tmpdir_line=`cat stderr | grep "Keeping temp"`
     tmpdir_stap=`expr "$tmpdir_line" : '.*"\(.*\)".*'`
 
-    # Remove the message about keeping th<e stap temp directory from stderr, unless
+    # Remove the message about keeping the stap temp directory from stderr, unless
     # the user did request to keep it.
-    if test $keep_temps != 1; then
-	sed -i "/^Keeping temp/d" stderr
+    if test "X$tmpdir_stap" != "X"; then
+	if test $keep_temps != 1; then
+	    sed -i "/^Keeping temp/d" stderr
+	fi
+
+        # Add the contents of the stap temp directory to the server output directory
+	ln -s $tmpdir_stap `basename $tmpdir_stap`
     fi
 
     # If the user specified -p5, remove the name of the kernel module from stdout.
     if test $p_phase = 5; then
 	sed -i '/\.ko$/d' stdout
     fi
-
-    # Add the contents of the stap temp directory to the server output directory
-    ln -s $tmpdir_stap `basename $tmpdir_stap`
 }
 
 # function: package_response
@@ -404,6 +410,15 @@ function cleanup {
     fi
 }
 
+# function: terminate
+#
+# Terminate gracefully.
+function terminate {
+    # Clean up
+    cleanup
+    exit
+}
+
 #-----------------------------------------------------------------------------
 # Beginning of main line execution.
 #-----------------------------------------------------------------------------
diff --git a/stap-serverd b/stap-serverd
index eaaeda00..af4b2717 100755
--- a/stap-serverd
+++ b/stap-serverd
@@ -13,8 +13,8 @@
 # incoming connections. When a connection is detected, the stap-server script
 # is run to handle the request.
 
-# Catch ctrl-c
-trap 'handle_sigint' SIGINT
+# Catch ctrl-c and other termination signals
+trap 'terminate' SIGTERM SIGINT
 
 #-----------------------------------------------------------------------------
 # Helper functions.
@@ -61,7 +61,10 @@ function listen {
     # Loop forever accepting requests
     while true
     do
-	nc -l $port < $fifo_name | stap-server $((port + 1)) > $fifo_name 2>&1
+	# Run this in the background and wait for it. This way any signals
+	# received (i.e. SIGTERM) will be processed.
+	nc -l $port < $fifo_name | stap-server $((port + 1)) > $fifo_name 2>&1 &
+	wait %nc
     done
 }
 
@@ -74,13 +77,24 @@ function fatal {
     exit 1
 }
 
-# function: handle_sigint
+# function: terminate
 #
-# Terminate gracefully when SIGINT is received.
-function handle_sigint {
-    echo "$0: received SIGINT. Exiting."
+# Terminate gracefully.
+function terminate {
+    echo "$0: Exiting"
+
+    # Kill the running 'avahi-publish-service' job
+    kill -s SIGTERM %avahi-publish-service 2> /dev/null
+    wait %avahi-publish-service 2> /dev/null
+
+    # Kill any running 'nc -l' job.
+    kill -s SIGTERM "%nc -l" 2> /dev/null
+    wait "%nc - l" 2> /dev/null
+
+    # Clean up
     cd `dirname $tmpdir`
     rm -fr $tmpdir
+
     exit
 }
 
diff --git a/stap-start-server b/stap-start-server
new file mode 100755
index 00000000..d143e269
--- /dev/null
+++ b/stap-start-server
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Start a systemtap server
+#
+# Copyright (C) 2008 Red Hat Inc.
+#
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+# This script attempts to start a systemtap server and echoes the
+# process id, if successful.
+
+# start the server
+stap-serverd </dev/null >/dev/null 2>&1 &
+server_pid=$!
+
+# Exit if the server did not start ok
+(ps | grep -q $server_pid) || exit 1
+
+# The server started ok. Echo its process id.
+echo $server_pid
+exit 0
diff --git a/stap-stop-server b/stap-stop-server
new file mode 100755
index 00000000..5afcf705
--- /dev/null
+++ b/stap-stop-server
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Start a systemtap server
+#
+# Copyright (C) 2008 Red Hat Inc.
+#
+# This file is part of systemtap, and is free software.  You can
+# redistribute it and/or modify it under the terms of the GNU General
+# Public License (GPL); either version 2, or (at your option) any
+# later version.
+
+# This script attempts to stop a systemtap server with the
+# given pid.
+
+# Get the process id.
+pid=$1
+if test "X$pid" = "X"; then
+   echo "Usage: $0 PROCESS_ID" >&2
+   exit 1
+fi
+
+# Verify that it is a systemtap server
+(ps -a | grep stap-serverd | grep -q $pid)
+if test $? != 0; then
+   echo "$pid is not a systemtap server"
+   exit 1
+fi
+
+# Try to kill the server
+kill -s SIGTERM $pid
diff --git a/systemtap.spec b/systemtap.spec
index deec1d28..503022d1 100644
--- a/systemtap.spec
+++ b/systemtap.spec
@@ -178,6 +178,10 @@ exit 0
 %{_bindir}/stap
 %{_bindir}/stap-server
 %{_bindir}/stap-serverd
+%{_bindir}/stap-find-servers
+%{_bindir}/stap-start-server
+%{_bindir}/stap-find-or-start-server
+%{_bindir}/stap-stop-server
 %{_mandir}/man1/*
 %{_mandir}/man5/*
 
diff --git a/testsuite/ChangeLog b/testsuite/ChangeLog
index 14243b80..4d3cab7c 100644
--- a/testsuite/ChangeLog
+++ b/testsuite/ChangeLog
@@ -1,3 +1,14 @@
+2008-07-29  Dave Brolley  <brolley@redhat.com>
+
+	* Makefile.am (SYSTEMTAP_PATH): Add $(EXTRA_SYSTEMTAP_PATH).
+	* lib/stap_compile.exp: Revert previous change.
+	* lib/stap_run.exp: Likewise.
+	* lib/stap_run2.exp: Likewise.
+	* lib/stap_run_binary.exp: Likewise.
+	* lib/stap_run_exact.exp: Likewise.
+	* lib/systemtap.exp (stap_exec): Removed.
+	* Makefile.in: Regenerated.
+
 2008-07-14  Dave Brolley  <brolley@redhat.com>
 
 	* Makefile.am (TOOL_OPTS): New variable.
diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am
index 5f3044cd..03c70cb6 100644
--- a/testsuite/Makefile.am
+++ b/testsuite/Makefile.am
@@ -40,7 +40,7 @@ SYSTEMTAP_RUNTIME=$(DESTDIR)$(pkgdatadir)/runtime
 SYSTEMTAP_TAPSET=$(DESTDIR)$(pkgdatadir)/tapset
 LD_LIBRARY_PATH=$(DESTDIR)$(libdir)/systemtap
 CRASH_LIBDIR=$(DESTDIR)$(libdir)/systemtap
-SYSTEMTAP_PATH=$(DESTDIR)$(bindir)
+SYSTEMTAP_PATH=$(EXTRA_SYSTEMTAP_PATH):$(DESTDIR)$(bindir)
 
 RUNTESTDEFAULTFLAGS = --tool $$tool --tool_opts \'$(TOOL_OPTS)\' --srcdir $$srcdir
 EXPECT = expect
diff --git a/testsuite/Makefile.in b/testsuite/Makefile.in
index 8fe7a21f..cc23aed4 100644
--- a/testsuite/Makefile.in
+++ b/testsuite/Makefile.in
@@ -149,7 +149,7 @@ SYSTEMTAP_RUNTIME = $(DESTDIR)$(pkgdatadir)/runtime
 SYSTEMTAP_TAPSET = $(DESTDIR)$(pkgdatadir)/tapset
 LD_LIBRARY_PATH = $(DESTDIR)$(libdir)/systemtap
 CRASH_LIBDIR = $(DESTDIR)$(libdir)/systemtap
-SYSTEMTAP_PATH = $(DESTDIR)$(bindir)
+SYSTEMTAP_PATH = $(EXTRA_SYSTEMTAP_PATH):$(DESTDIR)$(bindir)
 RUNTESTDEFAULTFLAGS = --tool $$tool --tool_opts \'$(TOOL_OPTS)\' --srcdir $$srcdir
 EXPECT = expect
 RUNTEST = "env SYSTEMTAP_RUNTIME=$(SYSTEMTAP_RUNTIME) SYSTEMTAP_TAPSET=$(SYSTEMTAP_TAPSET) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH) CRASH_LIBDIR=$(CRASH_LIBDIR) PATH=$(SYSTEMTAP_PATH):$$PATH $(srcdir)/execrc runtest"
diff --git a/testsuite/lib/stap_compile.exp b/testsuite/lib/stap_compile.exp
index 8f82018f..35124a44 100644
--- a/testsuite/lib/stap_compile.exp
+++ b/testsuite/lib/stap_compile.exp
@@ -4,7 +4,7 @@
 # - script is the script to compile
 # Additional arguments are passed to stap as-is.
 proc stap_compile { TEST_NAME compile script args } {
-    set cmd [concat [stap_exec] {-v -p4 -e} $script $args]
+    set cmd [concat stap {-v -p4 -e} $script $args]
 
     verbose -log "running $cmd" 
     eval spawn $cmd
diff --git a/testsuite/lib/stap_run.exp b/testsuite/lib/stap_run.exp
index 5f67d773..c0027e95 100644
--- a/testsuite/lib/stap_run.exp
+++ b/testsuite/lib/stap_run.exp
@@ -26,7 +26,7 @@ proc stap_run { TEST_NAME {LOAD_GEN_FUNCTION ""} {OUTPUT_CHECK_STRING ""} args }
 
     if {[info procs installtest_p] != "" && ![installtest_p]} { untested $TEST_NAME; return }
 
-    set cmd [concat [stap_exec] -v $args]
+    set cmd [concat stap -v $args]
     if [file readable $test_file_name] {
         lappend cmd $test_file_name
     }
diff --git a/testsuite/lib/stap_run2.exp b/testsuite/lib/stap_run2.exp
index 1d9dc0b3..9849aefb 100644
--- a/testsuite/lib/stap_run2.exp
+++ b/testsuite/lib/stap_run2.exp
@@ -15,7 +15,7 @@ proc stap_run2 { TEST_NAME args } {
     
     if {[info procs installtest_p] != "" && ![installtest_p]} { untested $TEST_NAME; return }
     
-    set cmd [concat [stap_exec] $args $test_file_name]
+    set cmd [concat stap $args $test_file_name]
     catch {eval exec $cmd} res
 
     set n 0
diff --git a/testsuite/lib/stap_run_binary.exp b/testsuite/lib/stap_run_binary.exp
index a5e1195a..1d31d817 100644
--- a/testsuite/lib/stap_run_binary.exp
+++ b/testsuite/lib/stap_run_binary.exp
@@ -13,7 +13,7 @@ proc stap_run_binary { TEST_NAME} {
   if {[info procs installtest_p] != "" && ![installtest_p]} {untested $TEST_NAME; return}
 
   set hex_args {-ve 8/1 "%02x " "\n"}
-  set res [exec [stap_exec] $test_file_name | hexdump $hex_args] 
+  set res [exec stap $test_file_name | hexdump $hex_args] 
 
   if {[string compare $res $::result_string] == 0} {
       pass "$TEST_NAME"
diff --git a/testsuite/lib/stap_run_exact.exp b/testsuite/lib/stap_run_exact.exp
index 23c22ec0..6a473798 100644
--- a/testsuite/lib/stap_run_exact.exp
+++ b/testsuite/lib/stap_run_exact.exp
@@ -12,7 +12,7 @@
 proc stap_run_exact { TEST_NAME test_file_name args } {
     if {[info procs installtest_p] != "" && ![installtest_p]} { untested $TEST_NAME; return }
     
-    set cmd [concat [stap_exec] $args $test_file_name]
+    set cmd [concat stap $args $test_file_name]
     catch {eval exec $cmd} res
 
     set n 0
diff --git a/testsuite/lib/systemtap.exp b/testsuite/lib/systemtap.exp
index e74bd13c..db5c1587 100644
--- a/testsuite/lib/systemtap.exp
+++ b/testsuite/lib/systemtap.exp
@@ -16,15 +16,6 @@ proc use_server_p {} {
 }
 
 
-proc stap_exec {} {
-    if {[info procs use_server_p] != "" && [use_server_p]} then {
-	return "stap-client"
-    } else {
-	return "stap"
-    }
-}
-
-
 proc print_systemtap_version {} {
     set version [exec /bin/uname -r]
     set location "/boot/vmlinux-$version"