priocntl workaround for Solaris 9 pty-close bug

Implement gross hack to use priocntl to work around the Solaris 9
pty-close bug.  Run expect at a higher class "FX" priority than
spawned processes, which run at a lower class "FX" priority.  "make
check" needs to start from a process which has FX priority >= 30 and
FX priority limit >= 30.  Thanks to Bill Sommerfeld for the hints.

ticket: new

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@16069 dc483132-0cff-0310-8789-dd5450dbe970

6 files changed, 54 insertions, 1 deletions

diff --git a/src/tests/ChangeLog b/src/tests/ChangeLog
index a4bf487a36..91dfb26f92 100644
--- a/src/tests/ChangeLog
+++ b/src/tests/ChangeLog
@@ -1,3 +1,7 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* configure.in: Invoke KRB5_AC_PRIOCNTL_HACK.
+
 2003-06-04  Tom Yu  <tlyu@mit.edu>
 
 	* Makefile.in (kdb_check): Remove uses of "dump -old", etc., since
diff --git a/src/tests/configure.in b/src/tests/configure.in
index 205eb1dc41..cc81450342 100644
--- a/src/tests/configure.in
+++ b/src/tests/configure.in
@@ -26,4 +26,5 @@ else
 	KRB4_DEJAGNU_TEST="KRBIV=1"
 fi
 AC_SUBST(KRB4_DEJAGNU_TEST)
+KRB5_AC_PRIOCNTL_HACK
 V5_AC_OUTPUT_MAKEFILE(. resolve asn.1 create hammer verify gssapi dejagnu)
diff --git a/src/tests/dejagnu/ChangeLog b/src/tests/dejagnu/ChangeLog
index 51affdd6ed..560919156f 100644
--- a/src/tests/dejagnu/ChangeLog
+++ b/src/tests/dejagnu/ChangeLog
@@ -1,3 +1,7 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* Makefile.in (check-runtest-yes): Add PRIOCNTL_HACK.
+
 2003-01-05  Sam Hartman  <hartmans@mit.edu>
 
 	* t_inetd.c: Remove declaration of errno
diff --git a/src/tests/dejagnu/Makefile.in b/src/tests/dejagnu/Makefile.in
index e245299334..15e82c2633 100644
--- a/src/tests/dejagnu/Makefile.in
+++ b/src/tests/dejagnu/Makefile.in
@@ -22,7 +22,7 @@ check-runtest-no::
 	@echo "+++"
 
 check-runtest-yes:: t_inetd site.exp
-	$(RUNTEST) --tool krb --srcdir $(srcdir) $(KRB4_RUNTESTFLAGS) $(RUNTESTFLAGS)
+	$(RUNTEST) --tool krb --srcdir $(srcdir) $(KRB4_RUNTESTFLAGS) PRIOCNTL_HACK=@PRIOCNTL_HACK@ $(RUNTESTFLAGS)
 
 t_inetd:: t_inetd.o $(KRB5_BASE_DEPLIBS)
 	$(CC_LINK) -o t_inetd t_inetd.o $(KRB5_BASE_LIBS)
@@ -44,6 +44,7 @@ runenv.vals: runenv.vars
 site.exp: runenv.vals
 	echo "set runvarlist [list `cat runenv.vals | tr '\n' ' '`]" | \
 		sed -e 's%=\.%='`pwd`'/.%g' > site.exp
+
 # +++ Dependency line eater +++
 # 
 # Makefile dependencies follow.  This must be the last section in
diff --git a/src/tests/dejagnu/config/ChangeLog b/src/tests/dejagnu/config/ChangeLog
index 9396e5eb90..db49f55787 100644
--- a/src/tests/dejagnu/config/ChangeLog
+++ b/src/tests/dejagnu/config/ChangeLog
@@ -1,3 +1,8 @@
+2004-02-12  Tom Yu  <tlyu@mit.edu>
+
+	* default.exp (PRIOCNTL_HACK): Wrap "spawn" to do priocntl things
+	to work around Solaris 9 pty-close bug.
+
 2003-12-13  Ken Raeburn  <raeburn@mit.edu>
 
 	* default.exp (passes): Add an AES-only pass.
diff --git a/src/tests/dejagnu/config/default.exp b/src/tests/dejagnu/config/default.exp
index 215671d340..68b551d69d 100644
--- a/src/tests/dejagnu/config/default.exp
+++ b/src/tests/dejagnu/config/default.exp
@@ -54,6 +54,44 @@ if 0 {
     }
 }
 
+# Hack around Solaris 9 kernel race condition that causes last output
+# from a pty to get dropped.
+if { $PRIOCNTL_HACK } {
+    catch {exec priocntl -s -c FX -m 30 -p 30 -i pid [getpid]}
+    rename spawn oldspawn
+    proc spawn { args } {
+	upvar 1 spawn_id spawn_id
+	set newargs {}
+	set inflags 1
+	set eatnext 0
+	foreach arg $args {
+	    if { $arg eq "-ignore" ||
+		 $arg eq "-open" ||
+		 $arg eq "-leaveopen" } {
+		lappend newargs $arg
+		set eatnext 1
+		continue
+	    }
+	    if [string match "-*" $arg] {
+		lappend newargs $arg
+		continue
+	    }
+	    if { $eatnext } {
+		set eatnext 0
+		lappend newargs $arg
+		continue
+	    }
+	    if { $inflags } {
+		set inflags 0
+		set newargs [concat $newargs {priocntl -e -c FX -p 0}]
+	    }
+	    lappend newargs $arg
+	}
+	set pid [eval oldspawn $newargs]
+	return $pid
+    }
+}
+
 # The des.des3-tgt.no-kdc-des3 pass will fail if the KDC doesn't
 # constrain ticket key enctypes to those in permitted_enctypes.  It
 # does this by not putting des3 in the permitted_enctypes, while