Changes to help streams work correctly on AIX 4.1.4 and HPUX 9.

Adapted loosly from 1837 in krb5-bugs, although I ended up not using
any of Doug's code.

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@7708 dc483132-0cff-0310-8789-dd5450dbe970

6 files changed, 93 insertions, 29 deletions

diff --git a/src/util/pty/ChangeLog b/src/util/pty/ChangeLog
index fb88e790b..10791a915 100644
--- a/src/util/pty/ChangeLog
+++ b/src/util/pty/ChangeLog
@@ -1,3 +1,23 @@
+Sun Mar 24 19:59:14 1996  Sam Hartman  <hartmans@tertius.mit.edu>
+
+	* configure.in : Do streams handling by deciding what modules to
+ 	push for each system.
+
+	* init_slave.c (pty_initialize_slave): Better abstraction for
+ 	dealing with what modules get pushed on what streams system.
+  	There is a list of modules controlled on a module-by-module basis
+ 	by configure.in, and the modules included in that list are pushed.
+
+	* void_assoc.c: Duplicate comment from open_ctty.c explaining that
+ 	it's OK to call void_association twice, and giving the caviats
+ 	about setsid.
+
+	* open_ctty.c (pty_open_ctty): Remove test to make sure /dev/tty
+ 	worked, so we can push the streams for the HP.
+
+	* open_slave.c (pty_open_slave): Test to see if /dev/tty works
+ 	only after calling pty_initialize_slave.
+
 Sat Mar 23 15:24:38 1996  Sam Hartman  <hartmans@tertius.mit.edu>
 
 	* configure.in : Remove shadow passwords check because nothing in
diff --git a/src/util/pty/configure.in b/src/util/pty/configure.in
index 3d5a649aa..f2673a48b 100644
--- a/src/util/pty/configure.in
+++ b/src/util/pty/configure.in
@@ -12,6 +12,13 @@ LOGINLIBS=
 dnl
 dnl Make our operating system-specific security checks and definitions for
 dnl login.
+dnl  In addition, the following code decides what streams modules will
+dnl be pushed onto a pty.In particular, if HAVE_STREAMS is defined and
+dnl HAVE_LINE_PUSH is not defined, modules may be pushed by inserting
+dnl An appropriate generic ifdef for each module in init_slave.c and
+dnl  AC_DEFINES for the operating systems that need the modules.
+dnl  Each OS that supports streams has a different idea of what you want to
+dnl push.
 dnl
 case $krb5_cv_host in
 *-*-ultrix*)
@@ -30,6 +37,15 @@ alpha-dec-osf*)
 		LOGINLIBS="$LOGINLIBS -lsecurity"
 	)
 	;;
+*-*-solaris*)
+     AC_DEFINE(PUSH_PTEM)
+     AC_DEFINE(PUSH_LDTERM)
+     AC_DEFINE(PUSH_TTCOMPAT)
+     ;;
+*-*-hpux*)
+    AC_DEFINE(PUSH_PTEM)
+     AC_DEFINE(PUSH_LDTERM)
+     ;;
 esac
 dnl
 AC_SUBST(LOGINLIBS)
diff --git a/src/util/pty/init_slave.c b/src/util/pty/init_slave.c
index d10c9c852..10f0efb31 100644
--- a/src/util/pty/init_slave.c
+++ b/src/util/pty/init_slave.c
@@ -1,7 +1,7 @@
 /*
  * pty_open_slave: open slave side of terminal, clearing for use.
  *
- * Copyright 1995 by the Massachusetts Institute of Technology.
+ * Copyright 1995, 1996 by the Massachusetts Institute of Technology.
  * 
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
@@ -20,6 +20,26 @@
 #include "libpty.h"
 #include "pty-int.h"
 
+/* * The following is an array of modules that should be pushed on the
+ *  stream.  See configure.in for caviats and notes about when this
+ *  array is used and not used.
+ */
+#if defined(HAVE_STREAMS)&&(!defined(HAVE_LINE_PUSH))
+static char *push_list[] = {
+#ifdef PUSH_PTEM
+  "ptem",
+#endif
+#ifdef PUSH_LDTERM
+  "ldterm",
+#endif
+#ifdef PUSH_TTCOMPAT
+"ttcompat",
+#endif
+  0};
+#endif /*HAVE_STREAMS but not HAVE_LINE_PUSH*/
+
+ 
+
 long pty_initialize_slave (fd)
     int fd;
 {
@@ -47,15 +67,13 @@ long pty_initialize_slave (fd)
 	    return PTY_OPEN_SLAVE_LINE_PUSHFAIL;
 	}
 #else /*No line_push */
-#ifdef sun 
-    if (ioctl(fd, I_PUSH, "ptem") < 0)
-	return PTY_OPEN_SLAVE_PUSH_FAIL;
-    if (ioctl(fd, I_PUSH, "ldterm") < 0)
-	return PTY_OPEN_SLAVE_PUSH_FAIL;
-    if (ioctl(fd, I_PUSH, "ttcompat") < 0)
-	return PTY_OPEN_SLAVE_PUSH_FAIL;
+    {
+       char **module = &push_list[0];
+      while (*module)
+		if (ioctl(fd, I_PUSH, *(module++)) < 0)
+		  	return PTY_OPEN_SLAVE_PUSH_FAIL;
+    }
 
-#endif /*SUN*/
 #endif /*LINE_PUSH*/
 #endif /*HAVE_STREAMS*/
 
diff --git a/src/util/pty/open_ctty.c b/src/util/pty/open_ctty.c
index b4b9ba97c..00a7eac80 100644
--- a/src/util/pty/open_ctty.c
+++ b/src/util/pty/open_ctty.c
@@ -1,7 +1,7 @@
 /*
  * pty_open_ctty: Open and establish controlling terminal.
  *
- * Copyright 1995 by the Massachusetts Institute of Technology.
+ * Copyright 1995, 1996 by the Massachusetts Institute of Technology.
  *
  * Permission to use, copy, modify, and distribute this software and
  * its documentation for any purpose and without fee is hereby
@@ -37,7 +37,7 @@ pty_open_ctty (slave, fd)
     const char * slave;
     int *fd;
 {
-    int testfd, retval;
+    int retval;
 
 /* First, dissociate from previous terminal */
     if ( (retval = ptyint_void_association()) != 0 )
@@ -54,21 +54,9 @@ pty_open_ctty (slave, fd)
 #ifdef ultrix
     setpgrp(0, getpid());
 #endif
-#ifdef ultrix
-    setpgrp(0, getpid());
-#endif
 
 #ifdef TIOCSCTTY
     ioctl(*fd, TIOCSCTTY, 0); /* Don't check return.*/
 #endif /* TIOCSTTY */
-
-    testfd = open("/dev/tty", O_RDWR|O_NDELAY);
-    if ( testfd < 0 )
-	{
-	    close(*fd);
-	    *fd = -1;
-	    return PTY_OPEN_SLAVE_NOCTTY;
-	}
-    close(testfd);
     return 0;
 }
diff --git a/src/util/pty/open_slave.c b/src/util/pty/open_slave.c
index b34f5c4bb..05b83b809 100644
--- a/src/util/pty/open_slave.c
+++ b/src/util/pty/open_slave.c
@@ -1,7 +1,7 @@
 /*
  * pty_open_slave: open slave side of terminal, clearing for use.
  *
- * Copyright 1995 by the Massachusetts Institute of Technology.
+ * Copyright 1995, 1996 by the Massachusetts Institute of Technology.
  *
  * 
  * Permission to use, copy, modify, and distribute this software and
@@ -26,7 +26,7 @@ long pty_open_slave ( slave, fd)
     const char *slave;
     int *fd;
 {
-    int vfd;
+    int vfd, testfd;
     long retval;
 #ifdef POSIX_SIGNALS
     struct sigaction sa;
@@ -78,7 +78,17 @@ long pty_open_slave ( slave, fd)
     if  (( retval = pty_open_ctty ( slave, fd))  != 0 ) {
 	return PTY_OPEN_SLAVE_OPENFAIL;
     }
-    return pty_initialize_slave (*fd);
-}
-
+    retval =  pty_initialize_slave (*fd);
 
+    if (retval)
+      return retval;
+    testfd = open("/dev/tty", O_RDWR|O_NDELAY);
+    if ( testfd < 0 )
+      {
+	close(*fd);
+	*fd = -1;
+	return PTY_OPEN_SLAVE_NOCTTY;
+      }
+    close(testfd);
+    return 0;
+}
diff --git a/src/util/pty/void_assoc.c b/src/util/pty/void_assoc.c
index c1a3acdca..9e7c64368 100644
--- a/src/util/pty/void_assoc.c
+++ b/src/util/pty/void_assoc.c
@@ -1,7 +1,7 @@
 /*
  * ptyint_void_association(): Void association with controlling terminal
  *
- * Copyright 1995 by the Massachusetts Institute of Technology.
+ * Copyright 1995, 1996 by the Massachusetts Institute of Technology.
  *
  * 
  * Permission to use, copy, modify, and distribute this software and
@@ -20,6 +20,18 @@
 #include <com_err.h>
 #include "libpty.h"
 #include "pty-int.h"
+/* 
+ * This routine will be called twice.  It's not particularly important
+ * that the setsid() or TIOCSTTY ioctls succeed (they may not the
+ * second time), but rather that we have a controlling terminal at the
+ * end.  It is assumed that vhangup doesn't exist and confuse the
+ * process's notion of controlling terminal on any system without
+ * TIOCNOTTY.  That is, either vhangup() leaves the controlling
+ * terminal in tact, breaks the association completely, or the system
+ * provides TIOCNOTTY to get things back into a reasonable state.  In
+ * practice, vhangup() either breaks the association completely or
+ * doesn't effect controlling terminals, so this condition is met.
+ */
 
 long ptyint_void_association()
 {