priocntl workaround for Solaris 9 pty-close bug

Implement gross hack to use priocntl to work around the Solaris 9
pty-close bug.  Run expect at a higher class "FX" priority than
spawned processes, which run at a lower class "FX" priority.  "make
check" needs to start from a process which has FX priority >= 30 and
FX priority limit >= 30.  Thanks to Bill Sommerfeld for the hints.

ticket: new

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@16069 dc483132-0cff-0310-8789-dd5450dbe970

22 files changed, 181 insertions, 34 deletions

diff --git a/src/ChangeLog b/src/ChangeLog
index 047a8aa2e1..531ba7e65f 100644
--- a/src/ChangeLog
+++ b/src/ChangeLog
@@ -1,3 +1,10 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* aclocal.m4 (KRB5_AC_PRIOCNTL_HACK): Set PRIOCNTL_HACK=1 on
+	Solaris 9.
+
+	* configure.in: Remove exitsleep.
+
 2003-12-27  Ken Raeburn  <raeburn@mit.edu>
 
 	* configure.in: Use AC_HELP_STRING for kdc-replay-cache option
diff --git a/src/aclocal.m4 b/src/aclocal.m4
index 646871764b..05e0fb8b53 100644
--- a/src/aclocal.m4
+++ b/src/aclocal.m4
@@ -1492,3 +1492,16 @@ AC_MSG_RESULT($krb5_cv_need_bind_8_compat)
 test $krb5_cv_need_bind_8_compat = yes && AC_DEFINE(BIND_8_COMPAT,1,[Define if OS has bind 9])
 ])
 dnl
+dnl KRB5_AC_PRIOCNTL_HACK
+dnl
+dnl
+AC_DEFUN([KRB5_AC_PRIOCNTL_HACK],
+[case $krb5_cv_host in
+*-*-solaris2.9*)
+	PRIOCNTL_HACK=1
+	;;
+*)
+	PRIOCNTL_HACK=0
+	;;
+esac
+AC_SUBST(PRIOCNTL_HACK)])
diff --git a/src/config/ChangeLog b/src/config/ChangeLog
index 16c1fa9cbc..778d89e3da 100644
--- a/src/config/ChangeLog
+++ b/src/config/ChangeLog
@@ -1,5 +1,7 @@
 2004-02-12  Tom Yu  <tlyu@mit.edu>
 
+	* shlib.conf (*-*-solaris-*): Remove exitsleep.
+
 	* pre.in (FAKEDEST, FAKEPREFIX, FAKELIBDIR): Support variables for
 	the _RLD_ROOT hack.
 
diff --git a/src/config/shlib.conf b/src/config/shlib.conf
index 8442ab8346..ed41bd8fce 100644
--- a/src/config/shlib.conf
+++ b/src/config/shlib.conf
@@ -255,18 +255,6 @@ mips-*-netbsd*)
 	CC_LINK_SHARED='$(PURE) $(CC) $(PROG_LIBPATH) $(RPATH_FLAG)$(PROG_RPATH) $(CFLAGS) $(LDFLAGS)'
 	CC_LINK_STATIC='$(PURE) $(CC) $(PROG_LIBPATH) $(CFLAGS) $(LDFLAGS)'
 	RUN_ENV='LD_LIBRARY_PATH=`echo $(PROG_LIBPATH) | sed -e "s/-L//g" -e "s/ /:/g"`; export LD_LIBRARY_PATH;'
-	case $krb5_cv_host in
-	*-*-solaris2.9*)
-		case $krb5_cv_prog_cc in
-		# XXX need better test for 64-bit compilation
-		*xarch=v9*)
-			RUN_ENV="$RUN_ENV"' LD_PRELOAD_64=$(BUILDTOP)/util/exitsleep; export LD_PRELOAD_64;'
-		;;
-		*)
-			RUN_ENV="$RUN_ENV"' LD_PRELOAD_32=$(BUILDTOP)/util/exitsleep; export LD_PRELOAD_32;'
-		;;
-		esac
-	esac
 	;;
 
 *-*-sunos*)
diff --git a/src/configure.in b/src/configure.in
index d683469223..0d95002607 100644
--- a/src/configure.in
+++ b/src/configure.in
@@ -72,21 +72,6 @@ AC_CONFIG_SUBDIRS(lib/kdb lib/gssapi lib/rpc lib/kadm5)
 if test -n "$KRB4_LIB"; then
   AC_CONFIG_SUBDIRS(krb524)
 fi
-case $krb5_cv_host in
-*-*-solaris2.9*)
-	if test "$krb5_cv_prog_gcc" = yes; then
-		DL_COMPILE='$(CC) -fpic -G -nostdlib'
-		DL_COMPILE_TAIL='-lc -ldl'
-	else
-		DL_COMPILE='$(CC) -Kpic -G'
-		DL_COMPILE_TAIL='-ldl'
-	fi
-	EXITSLEEP_TARG=exitsleep
-	;;
-esac
-AC_SUBST(DL_COMPILE)
-AC_SUBST(DL_COMPILE_TAIL)
-AC_SUBST(EXITSLEEP_TARG)
 
 AC_CONFIG_SUBDIRS(kdc kadmin slave clients appl tests)
 AC_CONFIG_FILES(krb5-config, [chmod +x krb5-config])
diff --git a/src/lib/kadm5/ChangeLog b/src/lib/kadm5/ChangeLog
index cff723bbad..0eb995f0c5 100644
--- a/src/lib/kadm5/ChangeLog
+++ b/src/lib/kadm5/ChangeLog
@@ -1,3 +1,7 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* configure.in: Invoke PRIOCNTL_HACK.
+
 2003-06-03  Tom Yu  <tlyu@mit.edu>
 
 	* alt_prof.c (krb5_read_realm_params): Don't bother reading in
diff --git a/src/lib/kadm5/configure.in b/src/lib/kadm5/configure.in
index 8a00e266b0..915c9507ff 100644
--- a/src/lib/kadm5/configure.in
+++ b/src/lib/kadm5/configure.in
@@ -19,5 +19,6 @@ dnl
 KRB5_BUILD_LIBOBJS
 KRB5_BUILD_LIBRARY_WITH_DEPS
 KRB5_BUILD_PROGRAM
+KRB5_AC_PRIOCNTL_HACK
 dnl
 V5_AC_OUTPUT_MAKEFILE(. clnt srv unit-test)
diff --git a/src/lib/kadm5/unit-test/ChangeLog b/src/lib/kadm5/unit-test/ChangeLog
index d1cd62047b..39aedf631f 100644
--- a/src/lib/kadm5/unit-test/ChangeLog
+++ b/src/lib/kadm5/unit-test/ChangeLog
@@ -1,3 +1,11 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* config/unix.exp (PRIOCNTL_HACK): Wrap "spawn" to do priocntl
+	things to work around Solaris 9 pty-close bug.
+
+	* Makefile.in (unit-test-client-body, unit-test-server-body): Add
+	PRIOCNTL_HACK.
+
 2003-10-16  Tom Yu  <tlyu@mit.edu>
 
 	* api.1/lock.exp: Work around a race condition in the Solaris 9
diff --git a/src/lib/kadm5/unit-test/Makefile.in b/src/lib/kadm5/unit-test/Makefile.in
index d38362bbad..382ac14d13 100644
--- a/src/lib/kadm5/unit-test/Makefile.in
+++ b/src/lib/kadm5/unit-test/Makefile.in
@@ -115,14 +115,16 @@ unit-test-client-body: site.exp test-noauth test-destroy test-handle-client
 	$(ENV_SETUP) $(RUNTEST) --tool api RPC=1 API=$(CLNTTCL) \
 		KINIT=$(BUILDTOP)/clients/kinit/kinit \
 		KDESTROY=$(BUILDTOP)/clients/kdestroy/kdestroy \
-		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local $(RUNTESTFLAGS)
+		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local \
+		PRIOCNTL_HACK=@PRIOCNTL_HACK@ $(RUNTESTFLAGS)
 	-mv api.log capi.log
 	-mv api.sum capi.sum
 
 unit-test-server-body: site.exp test-handle-server lock-test 
 	$(ENV_SETUP) $(RUNTEST) --tool api RPC=0 API=$(SRVTCL) \
 		LOCKTEST=./lock-test \
-		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local $(RUNTESTFLAGS)
+		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local \
+		PRIOCNTL_HACK=@PRIOCNTL_HACK@ $(RUNTESTFLAGS)
 	-mv api.log sapi.log
 	-mv api.sum sapi.sum
 
diff --git a/src/lib/kadm5/unit-test/config/unix.exp b/src/lib/kadm5/unit-test/config/unix.exp
index 0472789ea1..efc07693f2 100644
--- a/src/lib/kadm5/unit-test/config/unix.exp
+++ b/src/lib/kadm5/unit-test/config/unix.exp
@@ -14,6 +14,44 @@ if {[info exists exp_version_4]} {
 	set wait_status_index 3
 }
 
+# Hack around Solaris 9 kernel race condition that causes last output
+# from a pty to get dropped.
+if { $PRIOCNTL_HACK } {
+    catch {exec priocntl -s -c FX -m 30 -p 30 -i pid [getpid]}
+    rename spawn oldspawn
+    proc spawn { args } {
+	upvar 1 spawn_id spawn_id
+	set newargs {}
+	set inflags 1
+	set eatnext 0
+	foreach arg $args {
+	    if { $arg eq "-ignore" ||
+		 $arg eq "-open" ||
+		 $arg eq "-leaveopen" } {
+		lappend newargs $arg
+		set eatnext 1
+		continue
+	    }
+	    if [string match "-*" $arg] {
+		lappend newargs $arg
+		continue
+	    }
+	    if { $eatnext } {
+		set eatnext 0
+		lappend newargs $arg
+		continue
+	    }
+	    if { $inflags } {
+		set inflags 0
+		set newargs [concat $newargs {priocntl -e -c FX -p 0}]
+	    }
+	    lappend newargs $arg
+	}
+	set pid [eval oldspawn $newargs]
+	return $pid
+    }
+}
+
 # Variables for keeping track of api process state
 set api_pid "0"
 
diff --git a/src/lib/rpc/unit-test/ChangeLog b/src/lib/rpc/unit-test/ChangeLog
index 1d6c648b76..c87dade474 100644
--- a/src/lib/rpc/unit-test/ChangeLog
+++ b/src/lib/rpc/unit-test/ChangeLog
@@ -1,3 +1,10 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* configure.in: Invoke KRB5_AC_PRIOCNTL_HACK.
+
+	* config/unix.exp (PRIOCNTL_HACK): Wrap "spawn" to do priocntl
+	things to work around Solaris 9 pty-close bug.
+
 2003-12-02  Ken Raeburn  <raeburn@mit.edu>
 
 	* lib/helpers.exp (expect_kadm_ok, eof_client): Accept and ignore
diff --git a/src/lib/rpc/unit-test/Makefile.in b/src/lib/rpc/unit-test/Makefile.in
index a9ed5c3d7c..a4c2fc52d4 100644
--- a/src/lib/rpc/unit-test/Makefile.in
+++ b/src/lib/rpc/unit-test/Makefile.in
@@ -54,6 +54,7 @@ unit-test-body:
 		$(RUNTEST) SERVER=./server CLIENT=./client \
 		KINIT=$(BUILDTOP)/clients/kinit/kinit \
 		KDESTROY=$(BUILDTOP)/clients/kdestroy/kdestroy \
+		PRIOCNTL_HACK=@PRIOCNTL_HACK@ \
 		PASS="$(PASS)" --tool rpc_test $(RUNTESTFLAGS) ; \
 	then \
 		echo Cleaning up... ; \
diff --git a/src/lib/rpc/unit-test/config/unix.exp b/src/lib/rpc/unit-test/config/unix.exp
index 49ae4d1653..8b7df9f163 100644
--- a/src/lib/rpc/unit-test/config/unix.exp
+++ b/src/lib/rpc/unit-test/config/unix.exp
@@ -9,6 +9,44 @@ set kdestroy $KDESTROY
 
 set hostname [exec hostname]
 
+# Hack around Solaris 9 kernel race condition that causes last output
+# from a pty to get dropped.
+if { $PRIOCNTL_HACK } {
+    catch {exec priocntl -s -c FX -m 30 -p 30 -i pid [getpid]}
+    rename spawn oldspawn
+    proc spawn { args } {
+	upvar 1 spawn_id spawn_id
+	set newargs {}
+	set inflags 1
+	set eatnext 0
+	foreach arg $args {
+	    if { $arg eq "-ignore" ||
+		 $arg eq "-open" ||
+		 $arg eq "-leaveopen" } {
+		lappend newargs $arg
+		set eatnext 1
+		continue
+	    }
+	    if [string match "-*" $arg] {
+		lappend newargs $arg
+		continue
+	    }
+	    if { $eatnext } {
+		set eatnext 0
+		lappend newargs $arg
+		continue
+	    }
+	    if { $inflags } {
+		set inflags 0
+		set newargs [concat $newargs {priocntl -e -c FX -p 0}]
+	    }
+	    lappend newargs $arg
+	}
+	set pid [eval oldspawn $newargs]
+	return $pid
+    }
+}
+
 # this will initialize the database and keytab
 load_lib "helpers.exp"
 
diff --git a/src/lib/rpc/unit-test/configure.in b/src/lib/rpc/unit-test/configure.in
index 68ac8d1086..d06cb6fb87 100644
--- a/src/lib/rpc/unit-test/configure.in
+++ b/src/lib/rpc/unit-test/configure.in
@@ -25,4 +25,5 @@ changequote([, ])
 AC_SUBST(PASS)
 dnl
 CHECK_SIGNALS
+KRB5_AC_PRIOCNTL_HACK
 V5_AC_OUTPUT_MAKEFILE
diff --git a/src/tests/ChangeLog b/src/tests/ChangeLog
index a4bf487a36..91dfb26f92 100644
--- a/src/tests/ChangeLog
+++ b/src/tests/ChangeLog
@@ -1,3 +1,7 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* configure.in: Invoke KRB5_AC_PRIOCNTL_HACK.
+
 2003-06-04  Tom Yu  <tlyu@mit.edu>
 
 	* Makefile.in (kdb_check): Remove uses of "dump -old", etc., since
diff --git a/src/tests/configure.in b/src/tests/configure.in
index 205eb1dc41..cc81450342 100644
--- a/src/tests/configure.in
+++ b/src/tests/configure.in
@@ -26,4 +26,5 @@ else
 	KRB4_DEJAGNU_TEST="KRBIV=1"
 fi
 AC_SUBST(KRB4_DEJAGNU_TEST)
+KRB5_AC_PRIOCNTL_HACK
 V5_AC_OUTPUT_MAKEFILE(. resolve asn.1 create hammer verify gssapi dejagnu)
diff --git a/src/tests/dejagnu/ChangeLog b/src/tests/dejagnu/ChangeLog
index 51affdd6ed..560919156f 100644
--- a/src/tests/dejagnu/ChangeLog
+++ b/src/tests/dejagnu/ChangeLog
@@ -1,3 +1,7 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* Makefile.in (check-runtest-yes): Add PRIOCNTL_HACK.
+
 2003-01-05  Sam Hartman  <hartmans@mit.edu>
 
 	* t_inetd.c: Remove declaration of errno
diff --git a/src/tests/dejagnu/Makefile.in b/src/tests/dejagnu/Makefile.in
index e245299334..15e82c2633 100644
--- a/src/tests/dejagnu/Makefile.in
+++ b/src/tests/dejagnu/Makefile.in
@@ -22,7 +22,7 @@ check-runtest-no::
 	@echo "+++"
 
 check-runtest-yes:: t_inetd site.exp
-	$(RUNTEST) --tool krb --srcdir $(srcdir) $(KRB4_RUNTESTFLAGS) $(RUNTESTFLAGS)
+	$(RUNTEST) --tool krb --srcdir $(srcdir) $(KRB4_RUNTESTFLAGS) PRIOCNTL_HACK=@PRIOCNTL_HACK@ $(RUNTESTFLAGS)
 
 t_inetd:: t_inetd.o $(KRB5_BASE_DEPLIBS)
 	$(CC_LINK) -o t_inetd t_inetd.o $(KRB5_BASE_LIBS)
@@ -44,6 +44,7 @@ runenv.vals: runenv.vars
 site.exp: runenv.vals
 	echo "set runvarlist [list `cat runenv.vals | tr '\n' ' '`]" | \
 		sed -e 's%=\.%='`pwd`'/.%g' > site.exp
+
 # +++ Dependency line eater +++
 # 
 # Makefile dependencies follow.  This must be the last section in
diff --git a/src/tests/dejagnu/config/ChangeLog b/src/tests/dejagnu/config/ChangeLog
index 9396e5eb90..db49f55787 100644
--- a/src/tests/dejagnu/config/ChangeLog
+++ b/src/tests/dejagnu/config/ChangeLog
@@ -1,3 +1,8 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* default.exp (PRIOCNTL_HACK): Wrap "spawn" to do priocntl things
+	to work around Solaris 9 pty-close bug.
+
 2003-12-13  Ken Raeburn  <raeburn@mit.edu>
 
 	* default.exp (passes): Add an AES-only pass.
diff --git a/src/tests/dejagnu/config/default.exp b/src/tests/dejagnu/config/default.exp
index 215671d340..68b551d69d 100644
--- a/src/tests/dejagnu/config/default.exp
+++ b/src/tests/dejagnu/config/default.exp
@@ -54,6 +54,44 @@ if 0 {
     }
 }
 
+# Hack around Solaris 9 kernel race condition that causes last output
+# from a pty to get dropped.
+if { $PRIOCNTL_HACK } {
+    catch {exec priocntl -s -c FX -m 30 -p 30 -i pid [getpid]}
+    rename spawn oldspawn
+    proc spawn { args } {
+	upvar 1 spawn_id spawn_id
+	set newargs {}
+	set inflags 1
+	set eatnext 0
+	foreach arg $args {
+	    if { $arg eq "-ignore" ||
+		 $arg eq "-open" ||
+		 $arg eq "-leaveopen" } {
+		lappend newargs $arg
+		set eatnext 1
+		continue
+	    }
+	    if [string match "-*" $arg] {
+		lappend newargs $arg
+		continue
+	    }
+	    if { $eatnext } {
+		set eatnext 0
+		lappend newargs $arg
+		continue
+	    }
+	    if { $inflags } {
+		set inflags 0
+		set newargs [concat $newargs {priocntl -e -c FX -p 0}]
+	    }
+	    lappend newargs $arg
+	}
+	set pid [eval oldspawn $newargs]
+	return $pid
+    }
+}
+
 # The des.des3-tgt.no-kdc-des3 pass will fail if the KDC doesn't
 # constrain ticket key enctypes to those in permitted_enctypes.  It
 # does this by not putting des3 in the permitted_enctypes, while
diff --git a/src/util/ChangeLog b/src/util/ChangeLog
index 54b9cdbbd3..0c1e44f211 100644
--- a/src/util/ChangeLog
+++ b/src/util/ChangeLog
@@ -1,5 +1,7 @@
 2004-02-12  Tom Yu  <tlyu@mit.edu>
 
+	* Makefile.in (all-recurse): Remove exitsleep.
+	
 	* Makefile.in (all-unix, clean-unix): Add new rule to build fake
 	root directory for _RLD_ROOT hacks.
 
diff --git a/src/util/Makefile.in b/src/util/Makefile.in
index dd106bbc28..dc5d9ddce9 100644
--- a/src/util/Makefile.in
+++ b/src/util/Makefile.in
@@ -28,7 +28,7 @@ SLIBSH=sed -e 's|@''CC''@|$(CC)|g' -e 's,@''HOST_TYPE''@,$(HOST_TYPE),g' -e 's,@
 DL_COMPILE=@DL_COMPILE@
 DL_COMPILE_TAIL=@DL_COMPILE_TAIL@
 
-all-recurse: libupdate makeshlib @EXITSLEEP_TARG@
+all-recurse: libupdate makeshlib
 all-unix:: $(FAKEPREFIX)/lib
 all-mac::
 
@@ -71,9 +71,6 @@ makeshlib: $(srcdir)/makeshlib.sh Makefile
 	 $(RM) $@ $@.tmp
 	 $(SLIBSH) $(srcdir)/makeshlib.sh >$@.tmp&&chmod a+x $@.tmp&&mv $@.tmp $@
 
-exitsleep: $(srcdir)/exitsleep.c
-	$(DL_COMPILE) -oexitsleep $(srcdir)/exitsleep.c $(DL_COMPILE_TAIL)
-
 clean::
 	$(RM) libupdate makeshlib