priocntl workaround for Solaris 9 pty-close bug

Implement gross hack to use priocntl to work around the Solaris 9
pty-close bug.  Run expect at a higher class "FX" priority than
spawned processes, which run at a lower class "FX" priority.  "make
check" needs to start from a process which has FX priority >= 30 and
FX priority limit >= 30.  Thanks to Bill Sommerfeld for the hints.

ticket: new

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@16069 dc483132-0cff-0310-8789-dd5450dbe970

9 files changed, 102 insertions, 2 deletions

diff --git a/src/lib/kadm5/ChangeLog b/src/lib/kadm5/ChangeLog
index cff723bbad..0eb995f0c5 100644
--- a/src/lib/kadm5/ChangeLog
+++ b/src/lib/kadm5/ChangeLog
@@ -1,3 +1,7 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* configure.in: Invoke PRIOCNTL_HACK.
+
 2003-06-03  Tom Yu  <tlyu@mit.edu>
 
 	* alt_prof.c (krb5_read_realm_params): Don't bother reading in
diff --git a/src/lib/kadm5/configure.in b/src/lib/kadm5/configure.in
index 8a00e266b0..915c9507ff 100644
--- a/src/lib/kadm5/configure.in
+++ b/src/lib/kadm5/configure.in
@@ -19,5 +19,6 @@ dnl
 KRB5_BUILD_LIBOBJS
 KRB5_BUILD_LIBRARY_WITH_DEPS
 KRB5_BUILD_PROGRAM
+KRB5_AC_PRIOCNTL_HACK
 dnl
 V5_AC_OUTPUT_MAKEFILE(. clnt srv unit-test)
diff --git a/src/lib/kadm5/unit-test/ChangeLog b/src/lib/kadm5/unit-test/ChangeLog
index d1cd62047b..39aedf631f 100644
--- a/src/lib/kadm5/unit-test/ChangeLog
+++ b/src/lib/kadm5/unit-test/ChangeLog
@@ -1,3 +1,11 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* config/unix.exp (PRIOCNTL_HACK): Wrap "spawn" to do priocntl
+	things to work around Solaris 9 pty-close bug.
+
+	* Makefile.in (unit-test-client-body, unit-test-server-body): Add
+	PRIOCNTL_HACK.
+
 2003-10-16  Tom Yu  <tlyu@mit.edu>
 
 	* api.1/lock.exp: Work around a race condition in the Solaris 9
diff --git a/src/lib/kadm5/unit-test/Makefile.in b/src/lib/kadm5/unit-test/Makefile.in
index d38362bbad..382ac14d13 100644
--- a/src/lib/kadm5/unit-test/Makefile.in
+++ b/src/lib/kadm5/unit-test/Makefile.in
@@ -115,14 +115,16 @@ unit-test-client-body: site.exp test-noauth test-destroy test-handle-client
 	$(ENV_SETUP) $(RUNTEST) --tool api RPC=1 API=$(CLNTTCL) \
 		KINIT=$(BUILDTOP)/clients/kinit/kinit \
 		KDESTROY=$(BUILDTOP)/clients/kdestroy/kdestroy \
-		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local $(RUNTESTFLAGS)
+		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local \
+		PRIOCNTL_HACK=@PRIOCNTL_HACK@ $(RUNTESTFLAGS)
 	-mv api.log capi.log
 	-mv api.sum capi.sum
 
 unit-test-server-body: site.exp test-handle-server lock-test 
 	$(ENV_SETUP) $(RUNTEST) --tool api RPC=0 API=$(SRVTCL) \
 		LOCKTEST=./lock-test \
-		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local $(RUNTESTFLAGS)
+		KADMIN_LOCAL=$(BUILDTOP)/kadmin/cli/kadmin.local \
+		PRIOCNTL_HACK=@PRIOCNTL_HACK@ $(RUNTESTFLAGS)
 	-mv api.log sapi.log
 	-mv api.sum sapi.sum
 
diff --git a/src/lib/kadm5/unit-test/config/unix.exp b/src/lib/kadm5/unit-test/config/unix.exp
index 0472789ea1..efc07693f2 100644
--- a/src/lib/kadm5/unit-test/config/unix.exp
+++ b/src/lib/kadm5/unit-test/config/unix.exp
@@ -14,6 +14,44 @@ if {[info exists exp_version_4]} {
 	set wait_status_index 3
 }
 
+# Hack around Solaris 9 kernel race condition that causes last output
+# from a pty to get dropped.
+if { $PRIOCNTL_HACK } {
+    catch {exec priocntl -s -c FX -m 30 -p 30 -i pid [getpid]}
+    rename spawn oldspawn
+    proc spawn { args } {
+	upvar 1 spawn_id spawn_id
+	set newargs {}
+	set inflags 1
+	set eatnext 0
+	foreach arg $args {
+	    if { $arg eq "-ignore" ||
+		 $arg eq "-open" ||
+		 $arg eq "-leaveopen" } {
+		lappend newargs $arg
+		set eatnext 1
+		continue
+	    }
+	    if [string match "-*" $arg] {
+		lappend newargs $arg
+		continue
+	    }
+	    if { $eatnext } {
+		set eatnext 0
+		lappend newargs $arg
+		continue
+	    }
+	    if { $inflags } {
+		set inflags 0
+		set newargs [concat $newargs {priocntl -e -c FX -p 0}]
+	    }
+	    lappend newargs $arg
+	}
+	set pid [eval oldspawn $newargs]
+	return $pid
+    }
+}
+
 # Variables for keeping track of api process state
 set api_pid "0"
 
diff --git a/src/lib/rpc/unit-test/ChangeLog b/src/lib/rpc/unit-test/ChangeLog
index 1d6c648b76..c87dade474 100644
--- a/src/lib/rpc/unit-test/ChangeLog
+++ b/src/lib/rpc/unit-test/ChangeLog
@@ -1,3 +1,10 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* configure.in: Invoke KRB5_AC_PRIOCNTL_HACK.
+
+	* config/unix.exp (PRIOCNTL_HACK): Wrap "spawn" to do priocntl
+	things to work around Solaris 9 pty-close bug.
+
 2003-12-02  Ken Raeburn  <raeburn@mit.edu>
 
 	* lib/helpers.exp (expect_kadm_ok, eof_client): Accept and ignore
diff --git a/src/lib/rpc/unit-test/Makefile.in b/src/lib/rpc/unit-test/Makefile.in
index a9ed5c3d7c..a4c2fc52d4 100644
--- a/src/lib/rpc/unit-test/Makefile.in
+++ b/src/lib/rpc/unit-test/Makefile.in
@@ -54,6 +54,7 @@ unit-test-body:
 		$(RUNTEST) SERVER=./server CLIENT=./client \
 		KINIT=$(BUILDTOP)/clients/kinit/kinit \
 		KDESTROY=$(BUILDTOP)/clients/kdestroy/kdestroy \
+		PRIOCNTL_HACK=@PRIOCNTL_HACK@ \
 		PASS="$(PASS)" --tool rpc_test $(RUNTESTFLAGS) ; \
 	then \
 		echo Cleaning up... ; \
diff --git a/src/lib/rpc/unit-test/config/unix.exp b/src/lib/rpc/unit-test/config/unix.exp
index 49ae4d1653..8b7df9f163 100644
--- a/src/lib/rpc/unit-test/config/unix.exp
+++ b/src/lib/rpc/unit-test/config/unix.exp
@@ -9,6 +9,44 @@ set kdestroy $KDESTROY
 
 set hostname [exec hostname]
 
+# Hack around Solaris 9 kernel race condition that causes last output
+# from a pty to get dropped.
+if { $PRIOCNTL_HACK } {
+    catch {exec priocntl -s -c FX -m 30 -p 30 -i pid [getpid]}
+    rename spawn oldspawn
+    proc spawn { args } {
+	upvar 1 spawn_id spawn_id
+	set newargs {}
+	set inflags 1
+	set eatnext 0
+	foreach arg $args {
+	    if { $arg eq "-ignore" ||
+		 $arg eq "-open" ||
+		 $arg eq "-leaveopen" } {
+		lappend newargs $arg
+		set eatnext 1
+		continue
+	    }
+	    if [string match "-*" $arg] {
+		lappend newargs $arg
+		continue
+	    }
+	    if { $eatnext } {
+		set eatnext 0
+		lappend newargs $arg
+		continue
+	    }
+	    if { $inflags } {
+		set inflags 0
+		set newargs [concat $newargs {priocntl -e -c FX -p 0}]
+	    }
+	    lappend newargs $arg
+	}
+	set pid [eval oldspawn $newargs]
+	return $pid
+    }
+}
+
 # this will initialize the database and keytab
 load_lib "helpers.exp"
 
diff --git a/src/lib/rpc/unit-test/configure.in b/src/lib/rpc/unit-test/configure.in
index 68ac8d1086..d06cb6fb87 100644
--- a/src/lib/rpc/unit-test/configure.in
+++ b/src/lib/rpc/unit-test/configure.in
@@ -25,4 +25,5 @@ changequote([, ])
 AC_SUBST(PASS)
 dnl
 CHECK_SIGNALS
+KRB5_AC_PRIOCNTL_HACK
 V5_AC_OUTPUT_MAKEFILE