Fix Bug 503838 - rhcs71-80 external publishing ldap connection pools not reliable - improve connections or discovery.

git-svn-id: svn+ssh://svn.fedorahosted.org/svn/pki/trunk@1263 c9f7a03b-bd48-0410-a16d-cbbf54688b0b

4 files changed, 70 insertions, 9 deletions

diff --git a/pki/base/common/src/com/netscape/cmscore/ldap/LdapConnModule.java b/pki/base/common/src/com/netscape/cmscore/ldap/LdapConnModule.java
index ff540cbe8..273a21c96 100644
--- a/pki/base/common/src/com/netscape/cmscore/ldap/LdapConnModule.java
+++ b/pki/base/common/src/com/netscape/cmscore/ldap/LdapConnModule.java
@@ -97,6 +97,8 @@ public class LdapConnModule implements ILdapConnModule {
         int minConns = mConfig.getInteger(ILdapBoundConnFactory.PROP_MINCONNS, 3);
         int maxConns = mConfig.getInteger(ILdapBoundConnFactory.PROP_MAXCONNS, 15);
         // must get authInfo from the config, don't default to internaldb!!!
+
+        CMS.debug("Creating LdapBoundConnFactory for LdapConnModule."); 
         mLdapConnFactory =
              new LdapBoundConnFactory(minConns, maxConns, (LdapConnInfo)connInfo, authInfo);
 
diff --git a/pki/base/common/src/com/netscape/cmscore/ldap/LdapRequestListener.java b/pki/base/common/src/com/netscape/cmscore/ldap/LdapRequestListener.java
index 4388eeb6c..d467bd957 100644
--- a/pki/base/common/src/com/netscape/cmscore/ldap/LdapRequestListener.java
+++ b/pki/base/common/src/com/netscape/cmscore/ldap/LdapRequestListener.java
@@ -178,7 +178,7 @@ class LdapEnrollmentListener implements IRequestListener {
 
     public void accept(IRequest r) {
         CMS.debug(
-            "handling publishing for enrollment request id " +
+            "LdapRequestListener handling publishing for enrollment request id " +
             r.getRequestId());
 
         String profileId = r.getExtDataInString("profileId");
@@ -231,7 +231,7 @@ class LdapEnrollmentListener implements IRequestListener {
 		
                 results[i] = IRequest.RES_SUCCESS;
                 CMS.debug(
-                    "Published cert serial no 0x" +
+                    "acceptX509: Published cert serial no 0x" +
                     xcert.getSerialNumber().toString(16));
                 //mProcessor.setPublishedFlag(xcert.getSerialNumber(), true);
             } catch (ELdapException e) {
diff --git a/pki/base/common/src/com/netscape/cmscore/ldap/PublisherProcessor.java b/pki/base/common/src/com/netscape/cmscore/ldap/PublisherProcessor.java
index ee26478b8..c3cbeb182 100644
--- a/pki/base/common/src/com/netscape/cmscore/ldap/PublisherProcessor.java
+++ b/pki/base/common/src/com/netscape/cmscore/ldap/PublisherProcessor.java
@@ -831,6 +831,8 @@ public class PublisherProcessor implements
         if (!enabled())
             return;
 
+        CMS.debug("PublishProcessor::publishCACert");
+
             // get mapper and publisher for cert type.
         Enumeration rules = getRules(PROP_LOCAL_CA);
 
@@ -874,6 +876,7 @@ public class PublisherProcessor implements
             } catch (Exception e) {
                 // continue publishing even publisher has errors
                 //log(ILogger.LL_WARN, e.toString());
+                CMS.debug("PublisherProcessor::publishCACert returned error: " + e.toString());
                 error = true;
                 errorRule = errorRule + " " + rule.getInstanceName() +
                         " error:" + e.toString();
@@ -1010,6 +1013,8 @@ public class PublisherProcessor implements
                 error = true;
                 errorRule = errorRule + " " + rule.getInstanceName() +
                         " error:" + e.toString();
+
+                CMS.debug("PublisherProcessor::publishXCertPair: error: " + e.toString());
             }
         }
     }
@@ -1023,6 +1028,7 @@ public class PublisherProcessor implements
         boolean error = false;
         String errorRule = "";
 
+        CMS.debug("In  PublisherProcessor::publishCert");
         if (!enabled())
             return;
 
@@ -1031,8 +1037,10 @@ public class PublisherProcessor implements
 
          // Bugscape  #52306  -  Remove superfluous log messages on failure
         if (rules == null || !rules.hasMoreElements()) {
+                  CMS.debug("Publishing: can't find publishing rule,bailing.");
                   return;
         }
+
         while (rules.hasMoreElements()) {
             LdapRule rule = (LdapRule) rules.nextElement();
 
@@ -1062,6 +1070,7 @@ public class PublisherProcessor implements
         if (!error) {
             setPublishedFlag(cert.getSerialNumber(), true);
         } else {
+            CMS.debug("PublishProcessor::publishCert : " + CMS.getUserMessage("CMS_LDAP_PUBLISH_FAILED",errorRule));
             throw new ELdapException(CMS.getUserMessage("CMS_LDAP_PUBLISH_FAILED", errorRule));
         }
     }
@@ -1208,6 +1217,8 @@ public class PublisherProcessor implements
                         "Error publishing CRL to " + dn + ": " + e);
                     error = true;
                     errorRule = errorRule + " " + rule.getInstanceName();
+
+                    CMS.debug("PublisherProcessor::publishCRL: error: " + e.toString());
                 }
             }
         }catch (ELdapException e) {
@@ -1268,6 +1279,7 @@ public class PublisherProcessor implements
                         "Error publishing CRL to " + dn + ": " + e.toString());
                     error = true;
                     errorRule = errorRule + " " + rule.getInstanceName();
+                    CMS.debug("PublisherProcessor::publishCRL: error: " + e.toString()); 
                 } 
             }
         } catch (ELdapException e) {
@@ -1287,7 +1299,7 @@ public class PublisherProcessor implements
         IRequest r, Object obj) throws ELdapException {
         if (!enabled())
             return;
-CMS.debug("PublisherProcessor: in publishNow()");
+        CMS.debug("PublisherProcessor: in publishNow()");
         LDAPConnection conn = null;
 
         try {
@@ -1297,7 +1309,11 @@ CMS.debug("PublisherProcessor: in publishNow()");
                 LdapCertMapResult result = null;
 
                 if (mLdapConnModule != null) {
-                    conn = mLdapConnModule.getConn();
+                    try {
+                        conn = mLdapConnModule.getConn();
+                    } catch(ELdapException e) {
+                        throw e;
+                   }
                 }
                 try {
                     if ((mapper instanceof com.netscape.cms.publish.mappers.LdapCertSubjMap) &&
@@ -1325,7 +1341,7 @@ CMS.debug("PublisherProcessor: in publishNow()");
                     publisher.publish(conn, (String)dirdn, cert);
                 }
             } catch (Throwable e1) {
-                CMS.debug("Error publishing: publisher=" + publisher + " error=" + e1.toString());
+                CMS.debug("PublisherProcessor::publishNow : publisher=" + publisher + " error=" + e1.toString());
                 throw e1;
             }
             log(ILogger.LL_INFO, "published certificate serial number: 0x" + 
diff --git a/pki/base/common/src/com/netscape/cmscore/ldapconn/LdapBoundConnFactory.java b/pki/base/common/src/com/netscape/cmscore/ldapconn/LdapBoundConnFactory.java
index f5709222e..31e3dca1c 100644
--- a/pki/base/common/src/com/netscape/cmscore/ldapconn/LdapBoundConnFactory.java
+++ b/pki/base/common/src/com/netscape/cmscore/ldapconn/LdapBoundConnFactory.java
@@ -230,7 +230,7 @@ public class LdapBoundConnFactory implements ILdapBoundConnFactory {
      * makes the minumum number of connections
      */
     private void makeMinimum() throws ELdapException {
-        if (mMasterConn == null) 
+        if (mMasterConn == null || mMasterConn.isConnected() == false) 
             return;
         int increment;
 
@@ -307,9 +307,22 @@ public class LdapBoundConnFactory implements ILdapBoundConnFactory {
         throws ELdapException {
         boolean waited = false;
 
-        if (mMasterConn == null) {
-            makeConnection(true);
+        CMS.debug("In LdapBoundConnFactory::getConn()"); 
+        if(mMasterConn != null) 
+            CMS.debug("masterConn is connected: " + mMasterConn.isConnected());
+        else
+            CMS.debug("masterConn is null.");
+
+        if (mMasterConn == null || !mMasterConn.isConnected()) {
+            try {
+                  makeConnection(true);
+            }  catch (ELdapException e) {
+                mMasterConn = null;
+                CMS.debug("Can't create master connection in LdapBoundConnFactory::getConn! " + e.toString());
+                throw e;
+            }
         }
+
         if (mNumConns == 0) 
             makeMinimum();
         if (mNumConns == 0) {
@@ -333,7 +346,37 @@ public class LdapBoundConnFactory implements ILdapBoundConnFactory {
         mNumConns--;
         LDAPConnection conn = mConns[mNumConns];
 
-        mConns[mNumConns] = null;
+        boolean isConnected = false;
+        if(conn != null) {
+            isConnected = conn.isConnected();
+        }
+
+        CMS.debug("getConn: conn is connected " + isConnected);
+
+        //If masterConn is still alive, lets try to bring this one
+        //back to life
+
+        if((isConnected == false) && (mMasterConn != null) 
+            && (mMasterConn.isConnected() == true))   {
+            CMS.debug("Attempt to bring back down connection.");
+
+            if(doCloning == true) {
+                mConns[mNumConns] = (BoundConnection) mMasterConn.clone();
+            }
+            else {
+                try {
+                   mConns[mNumConns] = (BoundConnection) makeNewConnection(true);
+                }
+                catch (ELdapException e) {
+                   mConns[mNumConns] = null;
+                }
+           }
+           conn = mConns[mNumConns];
+
+           CMS.debug("Re-animated connection: " + conn);
+       }
+
+       mConns[mNumConns] = null;
 
         if (waited) {
             log(ILogger.LL_WARN,