Fixed thread leaks during shutdown.

Various codes have been modified to properly stop threads during
shutdown. A new ID attribute has been added to the LDAP connection
factory classes to help identify leaking threads.

https://fedorahosted.org/pki/ticket/1327

16 files changed, 47 insertions, 35 deletions

diff --git a/base/server/cms/src/com/netscape/cms/authentication/DirBasedAuthentication.java b/base/server/cms/src/com/netscape/cms/authentication/DirBasedAuthentication.java
index 78aa399b4..a8a95284d 100644
--- a/base/server/cms/src/com/netscape/cms/authentication/DirBasedAuthentication.java
+++ b/base/server/cms/src/com/netscape/cms/authentication/DirBasedAuthentication.java
@@ -277,7 +277,7 @@ public abstract class DirBasedAuthentication
             mGroupUserIDName = mLdapConfig.getString(PROP_GROUP_USERID_NAME, "cn");
             CMS.debug("DirBasedAuthentication: mGroupUserIDName="+ mGroupUserIDName);
         }
-        mConnFactory = CMS.getLdapAnonConnFactory();
+        mConnFactory = CMS.getLdapAnonConnFactory("DirBasedAuthentication");
         mConnFactory.init(mLdapConfig);
 
         /* initialize dn pattern */
diff --git a/base/server/cms/src/com/netscape/cms/authentication/PortalEnroll.java b/base/server/cms/src/com/netscape/cms/authentication/PortalEnroll.java
index c94cec617..46ce65f76 100644
--- a/base/server/cms/src/com/netscape/cms/authentication/PortalEnroll.java
+++ b/base/server/cms/src/com/netscape/cms/authentication/PortalEnroll.java
@@ -152,7 +152,7 @@ public class PortalEnroll extends DirBasedAuthentication {
             throw new EPropertyNotFound(CMS.getUserMessage("CMS_BASE_GET_PROPERTY_FAILED", "objectclass"));
 
         /* Get connect parameter */
-        mLdapFactory = CMS.getLdapBoundConnFactory();
+        mLdapFactory = CMS.getLdapBoundConnFactory("PortalEnroll");
         mLdapFactory.init(mLdapConfig);
         mLdapConn = mLdapFactory.getConn();
 
diff --git a/base/server/cms/src/com/netscape/cms/authentication/UidPwdPinDirAuthentication.java b/base/server/cms/src/com/netscape/cms/authentication/UidPwdPinDirAuthentication.java
index 9e551966c..ed2074053 100644
--- a/base/server/cms/src/com/netscape/cms/authentication/UidPwdPinDirAuthentication.java
+++ b/base/server/cms/src/com/netscape/cms/authentication/UidPwdPinDirAuthentication.java
@@ -157,7 +157,7 @@ public class UidPwdPinDirAuthentication extends DirBasedAuthentication
 
         if (mRemovePin) {
             removePinLdapConfigStore = config.getSubStore("ldap");
-            removePinLdapFactory = CMS.getLdapBoundConnFactory();
+            removePinLdapFactory = CMS.getLdapBoundConnFactory("UidPwdPinDirAuthentication");
             removePinLdapFactory.init(removePinLdapConfigStore);
             removePinLdapConnection = removePinLdapFactory.getConn();
         }
diff --git a/base/server/cms/src/com/netscape/cms/authorization/DirAclAuthz.java b/base/server/cms/src/com/netscape/cms/authorization/DirAclAuthz.java
index d0677875c..bdf3f5e9c 100644
--- a/base/server/cms/src/com/netscape/cms/authorization/DirAclAuthz.java
+++ b/base/server/cms/src/com/netscape/cms/authorization/DirAclAuthz.java
@@ -143,7 +143,7 @@ public class DirAclAuthz extends AAclAuthz
                 return;
         }
 
-        mLdapConnFactory = CMS.getLdapBoundConnFactory();
+        mLdapConnFactory = CMS.getLdapBoundConnFactory("DirAclAuthz");
         mLdapConnFactory.init(ldapConfig);
 
         // retrieve aclResources from the LDAP server and load
diff --git a/base/server/cms/src/com/netscape/cms/listeners/PinRemovalListener.java b/base/server/cms/src/com/netscape/cms/listeners/PinRemovalListener.java
index 21d69e29d..dbe03ce6f 100644
--- a/base/server/cms/src/com/netscape/cms/listeners/PinRemovalListener.java
+++ b/base/server/cms/src/com/netscape/cms/listeners/PinRemovalListener.java
@@ -95,7 +95,7 @@ public class PinRemovalListener implements IRequestListener {
         mConfig = config;
 
         mLdapConfig = mConfig.getSubStore(PROP_LDAP);
-        mConnFactory = CMS.getLdapBoundConnFactory();
+        mConnFactory = CMS.getLdapBoundConnFactory("PinRemovalListener");
         mConnFactory.init(mLdapConfig);
         mRemovePinLdapConnection = mConnFactory.getConn();
 
diff --git a/base/server/cms/src/com/netscape/cms/logging/LogFile.java b/base/server/cms/src/com/netscape/cms/logging/LogFile.java
index c465a4e53..9d19eddc6 100644
--- a/base/server/cms/src/com/netscape/cms/logging/LogFile.java
+++ b/base/server/cms/src/com/netscape/cms/logging/LogFile.java
@@ -519,6 +519,8 @@ public class LogFile implements ILogEventListener, IExtendedPluginInfo {
         if (fileName == null)
             throw new ELogException(CMS.getUserMessage("CMS_LOG_INVALID_FILE_NAME", "null"));
 
+        CMS.debug("Creating " + getClass().getSimpleName() + "(" + fileName + ")");
+
         //If we want to reuse the old log files
         //mFileName = fileName + "." + mLogFileDateFormat.format(mDate);
         mFileName = fileName;
@@ -863,9 +865,10 @@ public class LogFile implements ILogEventListener, IExtendedPluginInfo {
      * </ul>
      */
     public synchronized void shutdown() {
-        String auditMessage = null;
 
-        CMS.debug("LogFile:In log shutdown");
+        CMS.debug("Destroying LogFile(" + mFileName + ")");
+
+        String auditMessage = null;
 
         setFlushInterval(0);
 
@@ -891,10 +894,13 @@ public class LogFile implements ILogEventListener, IExtendedPluginInfo {
     public synchronized void setFlushInterval(int flushInterval) {
         mFlushInterval = flushInterval * 1000;
 
-        if ((mFlushThread == null) && (mFlushInterval > 0)) {
+        if (mFlushThread == null && mFlushInterval > 0) {
             mFlushThread = new FlushThread();
             mFlushThread.setDaemon(true);
             mFlushThread.start();
+
+        } else if (mFlushThread != null && mFlushInterval == 0) {
+            mFlushThread.interrupt();
         }
 
         this.notify();
@@ -921,9 +927,7 @@ public class LogFile implements ILogEventListener, IExtendedPluginInfo {
                     try {
                         LogFile.this.wait(mFlushInterval);
                     } catch (InterruptedException e) {
-                        // This shouldn't happen very often
-                        ConsoleError.send(new
-                                SystemEvent(CMS.getUserMessage("CMS_LOG_THREAD_INTERRUPT", "flush")));
+                        // shutdown
                     }
                 }
 
diff --git a/base/server/cms/src/com/netscape/cms/logging/RollingLogFile.java b/base/server/cms/src/com/netscape/cms/logging/RollingLogFile.java
index 0a7a9f098..32568dac0 100644
--- a/base/server/cms/src/com/netscape/cms/logging/RollingLogFile.java
+++ b/base/server/cms/src/com/netscape/cms/logging/RollingLogFile.java
@@ -139,6 +139,7 @@ public class RollingLogFile extends LogFile {
      * Shutdown this log file.
      */
     public synchronized void shutdown() {
+        CMS.debug("Destroying RollingLogFile(" + mFileName + ")");
         setRolloverTime("0");
         setExpirationTime("0");
         super.shutdown();
@@ -157,6 +158,9 @@ public class RollingLogFile extends LogFile {
             mRolloverThread = new RolloverThread();
             mRolloverThread.setDaemon(true);
             mRolloverThread.start();
+
+        } else if (mRolloverThread != null && mRolloverInterval == 0) {
+            mRolloverThread.interrupt();
         }
 
         this.notify();
@@ -190,6 +194,10 @@ public class RollingLogFile extends LogFile {
                     mExpirationThread.setDaemon(true);
                     mExpirationThread.start();
                 }
+
+            } else if (mExpirationThread != null && mExpirationTime == 0) {
+                mExpirationThread.interrupt();
+
             } else {
                 mExpLock.notify();
             }
@@ -401,9 +409,7 @@ public class RollingLogFile extends LogFile {
                     try {
                         RollingLogFile.this.wait(mRolloverInterval);
                     } catch (InterruptedException e) {
-                        // This shouldn't happen very often
-                        CMS.getLogger().getLogQueue().log(new
-                                SystemEvent(CMS.getUserMessage("CMS_LOG_THREAD_INTERRUPT", "rollover")));
+                        // shutdown
                     }
                 }
 
@@ -475,11 +481,13 @@ public class RollingLogFile extends LogFile {
                         try {
                             mExpLock.wait(sleepTime);
                         } catch (InterruptedException e) {
-                            // This shouldn't happen very often
-                            ConsoleError.send(new
-                                    SystemEvent(CMS.getUserMessage("CMS_LOG_THREAD_INTERRUPT", "expiration")));
+                            // shutdown
                         }
                     }
+
+                    if (mExpirationTime == 0) {
+                        break;
+                    }
                 }
             }
             mExpirationThread = null;
diff --git a/base/server/cms/src/com/netscape/cms/policy/constraints/AttributePresentConstraints.java b/base/server/cms/src/com/netscape/cms/policy/constraints/AttributePresentConstraints.java
index e12b8220f..fb56c67c4 100644
--- a/base/server/cms/src/com/netscape/cms/policy/constraints/AttributePresentConstraints.java
+++ b/base/server/cms/src/com/netscape/cms/policy/constraints/AttributePresentConstraints.java
@@ -257,7 +257,7 @@ public class AttributePresentConstraints extends APolicyRule
 
         mLdapConfig = mConfig.getSubStore(PROP_LDAP);
 
-        mConnFactory = CMS.getLdapBoundConnFactory();
+        mConnFactory = CMS.getLdapBoundConnFactory("AttributePresentConstraints");
         mConnFactory.init(mLdapConfig);
         mCheckAttrLdapConnection = mConnFactory.getConn();
 
diff --git a/base/server/cms/src/com/netscape/cms/profile/def/nsNKeySubjectNameDefault.java b/base/server/cms/src/com/netscape/cms/profile/def/nsNKeySubjectNameDefault.java
index c90b76c5d..49e951ae3 100644
--- a/base/server/cms/src/com/netscape/cms/profile/def/nsNKeySubjectNameDefault.java
+++ b/base/server/cms/src/com/netscape/cms/profile/def/nsNKeySubjectNameDefault.java
@@ -264,7 +264,7 @@ public class nsNKeySubjectNameDefault extends EnrollDefault {
             mParamsConfig = mInstConfig.getSubStore(PROP_PARAMS);
             mLdapConfig = mParamsConfig.getSubStore(PROP_LDAP);
             mBaseDN = mParamsConfig.getString(CONFIG_LDAP_BASEDN, null);
-            mConnFactory = CMS.getLdapAnonConnFactory();
+            mConnFactory = CMS.getLdapAnonConnFactory("nsNKeySubjectNameDefault");
             mConnFactory.init(mLdapConfig);
 
             /* initialize dn pattern */
diff --git a/base/server/cms/src/com/netscape/cms/profile/def/nsTokenUserKeySubjectNameDefault.java b/base/server/cms/src/com/netscape/cms/profile/def/nsTokenUserKeySubjectNameDefault.java
index 86a2a6961..0a8d9db90 100644
--- a/base/server/cms/src/com/netscape/cms/profile/def/nsTokenUserKeySubjectNameDefault.java
+++ b/base/server/cms/src/com/netscape/cms/profile/def/nsTokenUserKeySubjectNameDefault.java
@@ -287,7 +287,7 @@ public class nsTokenUserKeySubjectNameDefault extends EnrollDefault {
                 return;
 
             mBaseDN = mParamsConfig.getString(CONFIG_LDAP_BASEDN, null);
-            mConnFactory = CMS.getLdapAnonConnFactory();
+            mConnFactory = CMS.getLdapAnonConnFactory("nsTokenUserKeySubjectNameDefault");
             mConnFactory.init(mLdapConfig);
 
             /* initialize dn pattern */
diff --git a/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCaCertPublisher.java b/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCaCertPublisher.java
index 06050480e..f740ef3c4 100644
--- a/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCaCertPublisher.java
+++ b/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCaCertPublisher.java
@@ -186,7 +186,7 @@ public class LdapCaCertPublisher
                 String mgr_dn = mConfig.getString("bindDN", null);
                 String mgr_pwd = mConfig.getString("bindPWD", null);
 
-                altConn = CMS.getBoundConnection(host, portVal,
+                altConn = CMS.getBoundConnection("LdapCaCertPublisher", host, portVal,
                         version,
                         sslSocket, mgr_dn, mgr_pwd);
                 conn = altConn;
diff --git a/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCrlPublisher.java b/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCrlPublisher.java
index 9173f3cdf..80ffa3c4b 100644
--- a/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCrlPublisher.java
+++ b/base/server/cms/src/com/netscape/cms/publish/publishers/LdapCrlPublisher.java
@@ -177,7 +177,7 @@ public class LdapCrlPublisher implements ILdapPublisher, IExtendedPluginInfo {
                 String mgr_dn = mConfig.getString("bindDN", null);
                 String mgr_pwd = mConfig.getString("bindPWD", null);
 
-                altConn = CMS.getBoundConnection(host, portVal,
+                altConn = CMS.getBoundConnection("LdapCrlPublisher", host, portVal,
                         version,
                         sslSocket, mgr_dn, mgr_pwd);
                 conn = altConn;
diff --git a/base/server/cms/src/com/netscape/cms/publish/publishers/LdapUserCertPublisher.java b/base/server/cms/src/com/netscape/cms/publish/publishers/LdapUserCertPublisher.java
index 020c25552..a01cf80d1 100644
--- a/base/server/cms/src/com/netscape/cms/publish/publishers/LdapUserCertPublisher.java
+++ b/base/server/cms/src/com/netscape/cms/publish/publishers/LdapUserCertPublisher.java
@@ -141,7 +141,7 @@ public class LdapUserCertPublisher implements ILdapPublisher, IExtendedPluginInf
                 String mgr_dn = mConfig.getString("bindDN", null);
                 String mgr_pwd = mConfig.getString("bindPWD", null);
 
-                altConn = CMS.getBoundConnection(host, portVal,
+                altConn = CMS.getBoundConnection("LdapUserCertPublisher", host, portVal,
                         version,
                         sslSocket, mgr_dn, mgr_pwd);
                 conn = altConn;
diff --git a/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java b/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
index ce9e3bf49..ee88865e6 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/csadmin/ConfigurationUtils.java
@@ -1350,7 +1350,7 @@ public class ConfigurationUtils {
         boolean setupReplication = cs.getBoolean("preop.database.setupReplication", true);
 
         IConfigStore dbCfg = cs.getSubStore("internaldb");
-        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory();
+        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
         dbFactory.init(dbCfg);
         LDAPConnection conn = dbFactory.getConn();
 
@@ -1807,7 +1807,7 @@ public class ConfigurationUtils {
         IConfigStore cs = CMS.getConfigStore();
 
         IConfigStore dbCfg = cs.getSubStore("internaldb");
-        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory();
+        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
         dbFactory.init(dbCfg);
         LDAPConnection conn = dbFactory.getConn();
 
@@ -1826,7 +1826,7 @@ public class ConfigurationUtils {
         IConfigStore cs = CMS.getConfigStore();
 
         IConfigStore dbCfg = cs.getSubStore("internaldb");
-        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory();
+        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
         dbFactory.init(dbCfg);
         LDAPConnection conn = dbFactory.getConn();
 
@@ -1897,7 +1897,7 @@ public class ConfigurationUtils {
         ILdapConnFactory masterFactory = null;
         try {
             IConfigStore masterCfg = cs.getSubStore("preop.internaldb.master");
-            masterFactory = CMS.getLdapBoundConnFactory();
+            masterFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
             masterFactory.init(masterCfg);
             masterConn = masterFactory.getConn();
         } catch (Exception e) {
@@ -1912,7 +1912,7 @@ public class ConfigurationUtils {
         ILdapConnFactory replicaFactory = null;
         try {
             IConfigStore replicaCfg = cs.getSubStore("internaldb");
-            replicaFactory = CMS.getLdapBoundConnFactory();
+            replicaFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
             replicaFactory.init(replicaCfg);
             replicaConn = replicaFactory.getConn();
         } catch (Exception e) {
@@ -3616,7 +3616,7 @@ public class ConfigurationUtils {
             SAXException, ParserConfigurationException {
         IConfigStore cs = CMS.getConfigStore();
         IConfigStore dbCfg = cs.getSubStore("internaldb");
-        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory();
+        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
         dbFactory.init(dbCfg);
         LDAPConnection conn = dbFactory.getConn();
         LDAPEntry entry = null;
@@ -4243,7 +4243,7 @@ public class ConfigurationUtils {
         IConfigStore cs = CMS.getConfigStore();
         String userbasedn = "ou=people, " + cs.getString("internaldb.basedn");
         IConfigStore dbCfg = cs.getSubStore("internaldb");
-        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory();
+        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
         dbFactory.init(dbCfg);
         LDAPConnection conn = dbFactory.getConn();
 
@@ -4309,7 +4309,7 @@ public class ConfigurationUtils {
 
         // update global next range entries
         IConfigStore dbCfg = cs.getSubStore("internaldb");
-        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory();
+        ILdapConnFactory dbFactory = CMS.getLdapBoundConnFactory("ConfigurationUtils");
         dbFactory.init(dbCfg);
         LDAPConnection conn = dbFactory.getConn();
 
diff --git a/base/server/cms/src/com/netscape/cms/servlet/csadmin/SecurityDomainProcessor.java b/base/server/cms/src/com/netscape/cms/servlet/csadmin/SecurityDomainProcessor.java
index de83d20bf..08b11c605 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/csadmin/SecurityDomainProcessor.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/csadmin/SecurityDomainProcessor.java
@@ -154,7 +154,7 @@ public class SecurityDomainProcessor extends CAProcessor {
             String filter = "objectclass=pkiSecurityGroup";
 
             IConfigStore ldapConfig = cs.getSubStore("internaldb");
-            connFactory = CMS.getLdapBoundConnFactory();
+            connFactory = CMS.getLdapBoundConnFactory("SecurityDomainProcessor");
             connFactory.init(ldapConfig);
             conn = connFactory.getConn();
 
diff --git a/base/server/cms/src/com/netscape/cms/servlet/csadmin/UpdateDomainXML.java b/base/server/cms/src/com/netscape/cms/servlet/csadmin/UpdateDomainXML.java
index 9aa150475..1a23823b2 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/csadmin/UpdateDomainXML.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/csadmin/UpdateDomainXML.java
@@ -91,7 +91,7 @@ public class UpdateDomainXML extends CMSServlet {
 
         try {
             IConfigStore ldapConfig = cs.getSubStore("internaldb");
-            connFactory = CMS.getLdapBoundConnFactory();
+            connFactory = CMS.getLdapBoundConnFactory("UpdateDomainXML");
             connFactory.init(ldapConfig);
             conn = connFactory.getConn();
             conn.delete(dn);
@@ -125,7 +125,7 @@ public class UpdateDomainXML extends CMSServlet {
 
         try {
             IConfigStore ldapConfig = cs.getSubStore("internaldb");
-            connFactory = CMS.getLdapBoundConnFactory();
+            connFactory = CMS.getLdapBoundConnFactory("UpdateDomainXML");
             connFactory.init(ldapConfig);
             conn = connFactory.getConn();
             conn.modify(dn, mod);
@@ -160,7 +160,7 @@ public class UpdateDomainXML extends CMSServlet {
 
         try {
             IConfigStore ldapConfig = cs.getSubStore("internaldb");
-            connFactory = CMS.getLdapBoundConnFactory();
+            connFactory = CMS.getLdapBoundConnFactory("UpdateDomainXML");
             connFactory.init(ldapConfig);
             conn = connFactory.getConn();
             conn.add(entry);