Ticket#1028 phase2: TPS rewrite: provide externalReg functionality

This patch is the 2nd phase of the externalReg feature, it makes the
following improvements:
* added feature: recovery by keyid (v.s. by cert)
* fixed some auditing message errors
* added some missing ldapStringAttributes needed for delegation to work
properly
* added missing externalReg required config parameters
* made corrections to some externalReg related parameters to allow
delegation to work properly
* added handle of some error cases
* made sure externalReg enrollment does not go half-way (once fails,
bails out)

tested:
* enrollment of the three default TPS profiles (tokenTypes)
* format of the tokens enrolled with the three default tps profiles
* delegation enrollments
* cuid match check

next phase:
* cert/key retention (allow preserving existing certs/keys on the token)

note:
* some of the activity log and cert status related issues that are not
specifically relating to externalReg will be addressed in other more
relevant tickets.

11 files changed, 305 insertions, 177 deletions

diff --git a/base/kra/src/com/netscape/kra/TokenKeyRecoveryService.java b/base/kra/src/com/netscape/kra/TokenKeyRecoveryService.java
index b59a8b942..79110f52b 100644
--- a/base/kra/src/com/netscape/kra/TokenKeyRecoveryService.java
+++ b/base/kra/src/com/netscape/kra/TokenKeyRecoveryService.java
@@ -22,6 +22,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.FilterOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.math.BigInteger;
 import java.security.PublicKey;
 import java.security.SecureRandom;
 import java.util.Hashtable;
@@ -287,8 +288,10 @@ public class TokenKeyRecoveryService implements IService {
 
         // retrieve based on Certificate
         String cert_s = request.getExtDataInString(ATTR_USER_CERT);
-        if (cert_s == null) {
-            CMS.debug("TokenKeyRecoveryService: not receive cert");
+        String keyid_s = request.getExtDataInString(IRequest.NETKEY_ATTR_KEYID);
+        /* have to have at least one */
+        if ((cert_s == null) && (keyid_s == null)) {
+            CMS.debug("TokenKeyRecoveryService: not receive cert or keyid");
             request.setExtData(IRequest.RESULT, Integer.valueOf(3));
             auditMessage = CMS.getLogMessage(
                         LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
@@ -301,13 +304,29 @@ public class TokenKeyRecoveryService implements IService {
             return false;
         }
 
-        String cert = normalizeCertStr(cert_s);
+        String cert = null;
+        BigInteger keyid = null;
         java.security.cert.X509Certificate x509cert = null;
-        try {
-            x509cert = Cert.mapCert(cert);
-            if (x509cert == null) {
-                CMS.debug("cert mapping failed");
-                request.setExtData(IRequest.RESULT, Integer.valueOf(5));
+        if (keyid_s == null) {
+            cert = normalizeCertStr(cert_s);
+            try {
+                x509cert = Cert.mapCert(cert);
+                if (x509cert == null) {
+                    CMS.debug("cert mapping failed");
+                    request.setExtData(IRequest.RESULT, Integer.valueOf(5));
+                    auditMessage = CMS.getLogMessage(
+                            LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
+                            auditSubjectID,
+                            ILogger.FAILURE,
+                            auditRecoveryID,
+                            agentId);
+
+                    audit(auditMessage);
+                    return false;
+                }
+            } catch (IOException e) {
+                CMS.debug("TokenKeyRecoveryService: mapCert failed");
+                request.setExtData(IRequest.RESULT, Integer.valueOf(6));
                 auditMessage = CMS.getLogMessage(
                         LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
                         auditSubjectID,
@@ -318,18 +337,8 @@ public class TokenKeyRecoveryService implements IService {
                 audit(auditMessage);
                 return false;
             }
-        } catch (IOException e) {
-            CMS.debug("TokenKeyRecoveryService: mapCert failed");
-            request.setExtData(IRequest.RESULT, Integer.valueOf(6));
-            auditMessage = CMS.getLogMessage(
-                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
-                        auditSubjectID,
-                        ILogger.FAILURE,
-                        auditRecoveryID,
-                        agentId);
-
-            audit(auditMessage);
-            return false;
+        } else {
+            keyid = new BigInteger(keyid_s);
         }
 
         try {
@@ -346,7 +355,14 @@ public class TokenKeyRecoveryService implements IService {
             KeyRecord keyRecord = null;
             CMS.debug("KRA reading key record");
             try {
-                keyRecord = (KeyRecord) mStorage.readKeyRecord(cert);
+                if (keyid != null) {
+                    CMS.debug("TokenKeyRecoveryService: recover by keyid");
+                    keyRecord = (KeyRecord) mStorage.readKeyRecord(keyid);
+                } else {
+                    CMS.debug("TokenKeyRecoveryService: recover by cert");
+                    keyRecord = (KeyRecord) mStorage.readKeyRecord(cert);
+                }
+
                 if (keyRecord != null)
                     CMS.debug("read key record");
                 else {
@@ -389,25 +405,15 @@ public class TokenKeyRecoveryService implements IService {
             }
 
             // see if the certificate matches the key
-            byte pubData[] = keyRecord.getPublicKeyData();
-            byte inputPubData[] = x509cert.getPublicKey().getEncoded();
-
-            if (inputPubData.length != pubData.length) {
-                mKRA.log(ILogger.LL_FAILURE, CMS.getLogMessage("CMSCORE_KRA_PUBLIC_KEY_LEN"));
-                auditMessage = CMS.getLogMessage(
-                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
-                        auditSubjectID,
-                        ILogger.FAILURE,
-                        auditRecoveryID,
-                        agentId);
-
-                audit(auditMessage);
-                throw new EKRAException(
-                        CMS.getUserMessage("CMS_KRA_PUBLIC_KEY_NOT_MATCHED"));
-            }
-
-            for (int i = 0; i < pubData.length; i++) {
-                if (pubData[i] != inputPubData[i]) {
+            byte pubData[] = null;
+            pubData = keyRecord.getPublicKeyData();
+            // but if search by keyid, did not come with a cert
+            // so can't check
+            if (keyid == null) {
+                // see if the certificate matches the key
+                byte inputPubData[] = x509cert.getPublicKey().getEncoded();
+
+                if (inputPubData.length != pubData.length) {
                     mKRA.log(ILogger.LL_FAILURE, CMS.getLogMessage("CMSCORE_KRA_PUBLIC_KEY_LEN"));
                     auditMessage = CMS.getLogMessage(
                             LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
@@ -420,7 +426,23 @@ public class TokenKeyRecoveryService implements IService {
                     throw new EKRAException(
                             CMS.getUserMessage("CMS_KRA_PUBLIC_KEY_NOT_MATCHED"));
                 }
-            }
+
+                for (int i = 0; i < pubData.length; i++) {
+                    if (pubData[i] != inputPubData[i]) {
+                        mKRA.log(ILogger.LL_FAILURE, CMS.getLogMessage("CMSCORE_KRA_PUBLIC_KEY_LEN"));
+                        auditMessage = CMS.getLogMessage(
+                                LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_PROCESSED,
+                                auditSubjectID,
+                                ILogger.FAILURE,
+                                auditRecoveryID,
+                                agentId);
+
+                        audit(auditMessage);
+                        throw new EKRAException(
+                                CMS.getUserMessage("CMS_KRA_PUBLIC_KEY_NOT_MATCHED"));
+                    }
+                }
+            } // else, searched by keyid, can't check
 
             Type keyType = PrivateKey.RSA;
             byte wrapped[];
diff --git a/base/server/cms/src/com/netscape/cms/servlet/tks/TokenServlet.java b/base/server/cms/src/com/netscape/cms/servlet/tks/TokenServlet.java
index 1cc1c89d9..83b8bef92 100644
--- a/base/server/cms/src/com/netscape/cms/servlet/tks/TokenServlet.java
+++ b/base/server/cms/src/com/netscape/cms/servlet/tks/TokenServlet.java
@@ -93,10 +93,10 @@ public class TokenServlet extends CMSServlet {
             "LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_5"; // AC: KDF SPEC CHANGE:  Need to log both KDD and CUID.
 
     private final static String LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS =
-            "LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS_7";
+            "LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS_12";
 
     private final static String LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_FAILURE =
-            "LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_FAILURE_8";
+            "LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_FAILURE_13";
 
     private final static String LOGGING_SIGNED_AUDIT_COMPUTE_RANDOM_DATA_REQUEST =
             "LOGGING_SIGNED_AUDIT_COMPUTE_RANDOM_DATA_REQUEST_2";
diff --git a/base/server/cmsbundle/src/LogMessages.properties b/base/server/cmsbundle/src/LogMessages.properties
index ef3872c8d..10d9ae5ca 100644
--- a/base/server/cmsbundle/src/LogMessages.properties
+++ b/base/server/cmsbundle/src/LogMessages.properties
@@ -2262,7 +2262,6 @@ LOGGING_SIGNED_AUDIT_COMPUTE_RANDOM_DATA_REQUEST_PROCESSED_FAILURE_4=<type=COMPU
 # - used for TPS to TKS to get a sessoin key for secure channel setup
 # SubjectID must be the CUID of the token establishing the secure channel
 # AgentID must be the trusted agent id used to make the request
-LOGGING_SIGNED_AUDIT_COMPUTE_SESSION_KEY_REQUEST_3=<type=COMPUTE_SESSION_KEY_REQUEST>:[AuditEvent=COMPUTE_SESSION_KEY_REQUEST][SubjectID={0}][Outcome={1}][AgentID={2}] TKS Compute session key request
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the 
 ##   CUID.  Renamed to "CUID_encoded" and "KDD_encoded" to reflect fact that
 ##   encoded parameters are being logged.
@@ -2283,7 +2282,6 @@ LOGGING_SIGNED_AUDIT_COMPUTE_SESSION_KEY_REQUEST_4=<type=COMPUTE_SESSION_KEY_REQ
 # SelectedToken is the cryptographic token performing key operations
 # KeyNickName is the number keyset ex: #01#01
 #
-LOGGING_SIGNED_AUDIT_COMPUTE_SESSION_KEY_REQUEST_PROCESSED_SUCCESS_8=<type=COMPUTE_SESSION_KEY_REQUEST_PROCESSED_SUCCESS>:[AuditEvent=COMPUTE_SESSION_KEY_REQUEST_PROCESSED_SUCCESS][SubjectID={0}][Outcome={1}][status={2}][AgentID={3}][IsCryptoValidate={4}][IsServerSideKeygen={5}][SelectedToken={6}][KeyNickName={7}] TKS Compute session key request processed successfully
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the
 ##   CUID.  Renamed to "CUID_decoded" and "KDD_decoded" to reflect fact
 ##   that decoded parameters are now logged.
@@ -2311,7 +2309,6 @@ LOGGING_SIGNED_AUDIT_COMPUTE_SESSION_KEY_REQUEST_PROCESSED_SUCCESS_13=<type=COMP
 # SelectedToken is the cryptographic token performing key operations
 # KeyNickName is the numeric keyset ex: #01#01
 # Error gives the error message
-LOGGING_SIGNED_AUDIT_COMPUTE_SESSION_KEY_REQUEST_PROCESSED_FAILURE_9=<type=COMPUTE_SESSION_KEY_REQUEST_PROCESSED_FAILURE>:[AuditEvent=COMPUTE_SESSION_KEY_REQUEST_PROCESSED_FAILURE][SubjectID={0}][Outcome={1}][status={2}][AgentID={3}][IsCryptoValidate={4}][IsServerSideKeygen={5}][SelectedToken={7}][KeyNickName={7}][Error={8}] TKS Compute session key request failed 
 #
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the CUID.  Renamed to "CUID_decoded" and "KDD_decoded" to reflect fact that decoded parameters are now logged.
 ##                       Also added TKSKeyset, KeyInfo_KeyVersion, NistSP800_108KdfOnKeyVersion, NistSP800_108KdfUseCuidAsKdd
@@ -2331,7 +2328,6 @@ LOGGING_SIGNED_AUDIT_COMPUTE_SESSION_KEY_REQUEST_PROCESSED_FAILURE_14=<type=COMP
 # status is 0 for success, non-zero for various errors
 # oldMasterKeyName is the old master key name
 # newMasterKeyName is the new master key name
-LOGGING_SIGNED_AUDIT_DIVERSIFY_KEY_REQUEST_5=<type=DIVERSIFY_KEY_REQUEST>:[AuditEvent=DIVERSIFY_KEY_REQUEST][SubjectID={0}][Outcome={1}][AgentID={2}][oldMasterKeyName={3}][newMasterKeyName={4}] TKS Key Change Over request
 #
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the CUID.  Renamed to "CUID_encoded" and "KDD_encoded" to reflect fact that encoded parameters are being logged.
 # CUID_encoded must be the special-encoded CUID of the token establishing the secure channel
@@ -2347,7 +2343,6 @@ LOGGING_SIGNED_AUDIT_DIVERSIFY_KEY_REQUEST_6=<type=DIVERSIFY_KEY_REQUEST>:[Audit
 # status is 0 for success, non-zero for various errors
 # oldMasterKeyName is the old master key name
 # newMasterKeyName is the new master key name
-LOGGING_SIGNED_AUDIT_DIVERSIFY_KEY_REQUEST_PROCESSED_SUCCESS_6=<type=DIVERSIFY_KEY_REQUEST_PROCESSED_SUCCESS>:[AuditEvent=DIVERSIFY_KEY_REQUEST_PROCESSED_SUCCESS][SubjectID={0}][Outcome={1}][status={2}][AgentID={3}][oldMasterKeyName={4}][newMasterKeyName={5}] TKS Key Change Over request processed successfully
 #
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the CUID.  Renamed to "CUID_decoded" and "KDD_decoded" to reflect fact that decoded parameters are now logged.
 ##                       Also added TKSKeyset, OldKeyInfo_KeyVersion, NewKeyInfo_KeyVersion, NistSP800_108KdfOnKeyVersion, NistSP800_108KdfUseCuidAsKdd
@@ -2371,7 +2366,6 @@ LOGGING_SIGNED_AUDIT_DIVERSIFY_KEY_REQUEST_PROCESSED_SUCCESS_12=<type=DIVERSIFY_
 # oldMasterKeyName is the old master key name
 # newMasterKeyName is the new master key name
 # Error gives the error message
-LOGGING_SIGNED_AUDIT_DIVERSIFY_KEY_REQUEST_PROCESSED_FAILURE_7=<type=DIVERSIFY_KEY_REQUEST_PROCESSED_FAILURE>:[AuditEvent=DIVERSIFY_KEY_REQUEST_PROCESSED_FAILURE][SubjectID={0}][Outcome={1}][status={2}][AgentID={3}][oldMasterKeyName={4}][newMasterKeyName={5}][Error={6}] TKS Key Change Over request failed 
 #
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the CUID.  Renamed to "CUID_decoded" and "KDD_decoded" to reflect fact that decoded parameters are now logged.
 ##                       Also added TKSKeyset, OldKeyInfo_KeyVersion, NewKeyInfo_KeyVersion, NistSP800_108KdfOnKeyVersion, NistSP800_108KdfUseCuidAsKdd
@@ -2409,7 +2403,6 @@ LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_5=<type=ENCRYPT_DATA_REQUEST>:[AuditEv
 # isRandom tells if the data is randomly generated on TKS
 # SelectedToken is the cryptographic token performing key operations
 # KeyNickName is the numeric keyset ex: #01#01
-LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS_7=<type=ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS>:[AuditEvent=ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS][SubjectID={0}][Outcome={1}][status={2}][AgentID={3}][isRandom={4}][SelectedToken={5}][KeyNickName={6}] TKS encrypt data request processed successfully
 #
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the CUID.  Renamed to "CUID_decoded" and "KDD_decoded" to reflect fact that decoded parameters are now logged.
 ##                       Also added TKSKeyset, KeyInfo_KeyVersion, NistSP800_108KdfOnKeyVersion, NistSP800_108KdfUseCuidAsKdd
@@ -2433,7 +2426,6 @@ LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_SUCCESS_12=<type=ENCRYPT_DAT
 # SelectedToken is the cryptographic token performing key operations
 # KeyNickName is the numeric keyset ex: #01#01
 # Error gives the error message
-LOGGING_SIGNED_AUDIT_ENCRYPT_DATA_REQUEST_PROCESSED_FAILURE_8=<type=ENCRYPT_DATA_REQUEST_PROCESSED_FAILURE>:[AuditEvent=ENCRYPT_DATA_REQUEST_PROCESSED_FAILURE][SubjectID={0}][Outcome={1}][status={2}][AgentID={3}][isRandom={4}][SelectedToken={5}][KeyNickName={6}][Error={7}] TKS encrypt data request failed 
 #
 ## AC: KDF SPEC CHANGE - Need to log both the KDD and CUID, not just the CUID.  Renamed to "CUID_decoded" and "KDD_decoded" to reflect fact that decoded parameters are now logged.
 ##                       Also added TKSKeyset, KeyInfo_KeyVersion, NistSP800_108KdfOnKeyVersion, NistSP800_108KdfUseCuidAsKdd
diff --git a/base/tps/shared/conf/CS.cfg.in b/base/tps/shared/conf/CS.cfg.in
index b899e7d21..e583ac097 100644
--- a/base/tps/shared/conf/CS.cfg.in
+++ b/base/tps/shared/conf/CS.cfg.in
@@ -56,7 +56,7 @@ auths.instance.ldap1.ldapStringAttributes._001=# For isExternalReg
 auths.instance.ldap1.ldapStringAttributes._002=#   attributes will be available as
 auths.instance.ldap1.ldapStringAttributes._003=#       $<attribute>$
 auths.instance.ldap1.ldapStringAttributes._004=#   attributes example:
-auths.instance.ldap1.ldapStringAttributes._005=#mail,cn,uid,exec-edipi,firstname,lastname,exec-pcc,exec-mail,certsToAdd,tokenCUID,tokenType
+auths.instance.ldap1.ldapStringAttributes._005=#mail,cn,uid,edipi,pcc,firstname,lastname,exec-edipi,exec-pcc,exec-mail,certsToAdd,tokenCUID,tokenType
 auths.instance.ldap1.attributes._006=################################# #############
 auths.instance.ldap1.ldapStringAttributes=mail,cn,uid
 auths.instance.ldap1.ldap.basedn=[LDAP_ROOT]
@@ -78,7 +78,7 @@ auths.instance.SSLclientCertAuth.pluginName=SSLclientCertAuth
 auths.revocationChecking.bufferSize=50
 authType=pwd
 authz._000=##
-authz._001=## new authorizatioin
+authz._001=## new authorization
 authz._002=##
 authz.evaluateOrder=deny,allow
 authz.impl._000=##
@@ -273,7 +273,7 @@ op.enroll.delegateIEtoken.keyGen.authentication.ca.profileId=caTokenUserDelegate
 op.enroll.delegateIEtoken.keyGen.authentication.certAttrId=c3
 op.enroll.delegateIEtoken.keyGen.authentication.certId=C3
 op.enroll.delegateIEtoken.keyGen.authentication.cuid_label=$cuid$
-op.enroll.delegateIEtoken.keyGen.authentication.dnpattern=cn=$auth.firstname$.$auth.lastname$.$auth.exec-edipi$,e=$auth.mail$,o=TMS Org
+op.enroll.delegateIEtoken.keyGen.authentication.dnpattern=cn=$auth.firstname$.$auth.lastname$.$auth.edipi$,e=$auth.mail$,o=TMS Org
 op.enroll.delegateIEtoken.keyGen.authentication.keySize=1024
 op.enroll.delegateIEtoken.keyGen.authentication.keyUsage=0
 op.enroll.delegateIEtoken.keyGen.authentication.keyUser=0
@@ -316,6 +316,7 @@ op.enroll.delegateIEtoken.keyGen.authentication.recovery.keyCompromise.scheme=Ge
 op.enroll.delegateIEtoken.keyGen.authentication.recovery.onHold.revokeCert=false
 op.enroll.delegateIEtoken.keyGen.authentication.recovery.onHold.revokeCert.reason=6
 op.enroll.delegateIEtoken.keyGen.authentication.recovery.onHold.scheme=GenerateNewKey
+op.enroll.delegateIEtoken.keyGen.encryption.ca.conn=ca1
 op.enroll.delegateIEtoken.keyGen.encryption.private.keyCapabilities.decrypt=true
 op.enroll.delegateIEtoken.keyGen.encryption.private.keyCapabilities.derive=false
 op.enroll.delegateIEtoken.keyGen.encryption.private.keyCapabilities.encrypt=false
@@ -440,7 +441,7 @@ op.enroll.delegateISEtoken.keyGen.authentication.ca.profileId=caTokenUserDelegat
 op.enroll.delegateISEtoken.keyGen.authentication.certAttrId=c3
 op.enroll.delegateISEtoken.keyGen.authentication.certId=C3
 op.enroll.delegateISEtoken.keyGen.authentication.cuid_label=$cuid$
-op.enroll.delegateISEtoken.keyGen.authentication.dnpattern=cn=$auth.firstname$.$auth.lastname$.$auth.exec-edipi$,e=$auth.mail$,o=TMS Org
+op.enroll.delegateISEtoken.keyGen.authentication.dnpattern=cn=$auth.firstname$.$auth.lastname$.$auth.edipi$,e=$auth.mail$,o=TMS Org
 op.enroll.delegateISEtoken.keyGen.authentication.keySize=1024
 op.enroll.delegateISEtoken.keyGen.authentication.keyUsage=0
 op.enroll.delegateISEtoken.keyGen.authentication.keyUser=0
@@ -654,6 +655,27 @@ op.enroll.delegateISEtoken.update.applet.encryption=true
 op.enroll.delegateISEtoken.update.applet.requiredVersion=1.4.4d40a449
 op.enroll.delegateISEtoken.update.symmetricKeys.enable=false
 op.enroll.delegateISEtoken.update.symmetricKeys.requiredVersion=1
+op.format.delegateISEtoken.auth.enable=true
+op.format.delegateISEtoken.cuidMustMatchKDD=false
+op.format.delegateISEtoken.enableBoundedGPKeyVersion=true
+op.format.delegateISEtoken.minimumGPKeyVersion=01
+op.format.delegateISEtoken.maximumGPKeyVersion=FF
+op.format.delegateISEtoken.rollbackKeyVersionOnPutKeyFailure=false
+op.format.delegateISEtoken.validateCardKeyInfoAgainstTokenDB=true
+op.format.delegateISEtoken.auth.id=ldap3
+op.format.delegateISEtoken.ca.conn=ca1
+op.format.delegateISEtoken.cardmgr_instance=A0000000030000
+op.format.delegateISEtoken.issuerinfo.enable=true
+op.format.delegateISEtoken.issuerinfo.value=http://[PKI_HOSTNAME]:[PKI_UNSECURE_PORT]/tps/phoneHome
+op.format.delegateISEtoken.loginRequest.enable=true
+op.format.delegateISEtoken.revokeCert=false
+op.format.delegateISEtoken.tks.conn=tks1
+op.format.delegateISEtoken.update.applet.directory=/usr/share/pki/tps/applets
+op.format.delegateISEtoken.update.applet.emptyToken.enable=true
+op.format.delegateISEtoken.update.applet.encryption=true
+op.format.delegateISEtoken.update.applet.requiredVersion=1.4.4d40a449
+op.format.delegateISEtoken.update.symmetricKeys.enable=false
+op.format.delegateISEtoken.update.symmetricKeys.requiredVersion=1
 op.enroll.externalRegAddToToken._000=#########################################
 op.enroll.externalRegAddToToken._001=# for externalReg recovering certs/keys only
 op.enroll.externalRegAddToToken._002=#########################################
@@ -668,6 +690,7 @@ op.enroll.externalRegAddToToken.auth.id=ldap1
 op.enroll.externalRegAddToToken.cardmgr_instance=A0000000030000
 op.enroll.externalRegAddToToken.issuerinfo.enable=true
 op.enroll.externalRegAddToToken.issuerinfo.value=http://[PKI_HOSTNAME]:[PKI_UNSECURE_PORT]/tps/phoneHome
+op.enroll.externalRegAddToToken.keyGen.encryption.ca.conn=ca1
 op.enroll.externalRegAddToToken.keyGen.encryption.private.keyCapabilities.decrypt=true
 op.enroll.externalRegAddToToken.keyGen.encryption.private.keyCapabilities.derive=false
 op.enroll.externalRegAddToToken.keyGen.encryption.private.keyCapabilities.encrypt=false
@@ -698,6 +721,9 @@ op.enroll.externalRegAddToToken.keyGen.encryption.recovery.onHold.revokeCert=fal
 op.enroll.externalRegAddToToken.keyGen.signing.recovery.destroyed.revokeCert=false
 op.enroll.externalRegAddToToken.keyGen.signing.recovery.keyCompromise.revokeCert=false
 op.enroll.externalRegAddToToken.keyGen.signing.recovery.onHold.revokeCert=false
+op.enroll.externalRegAddToToken.keyGen.encryption.serverKeygen.archive=true
+op.enroll.externalRegAddToToken.keyGen.encryption.serverKeygen.drm.conn=kra1
+op.enroll.externalRegAddToToken.keyGen.encryption.serverKeygen.enable=true
 op.enroll.externalRegAddToToken.keyGen.tokenName=$auth.cn$
 op.enroll.externalRegAddToToken.loginRequest.enable=true
 op.enroll.externalRegAddToToken.pkcs11obj.compress.enable=true
@@ -724,8 +750,8 @@ op.format.externalRegAddToToken.update.applet.directory=/usr/share/pki/tps/apple
 op.format.externalRegAddToToken.update.applet.emptyToken.enable=true
 op.format.externalRegAddToToken.update.applet.encryption=true
 op.format.externalRegAddToToken.update.applet.requiredVersion=1.4.4d40a449
-op.format.externalRegAddToToken.update.symmetricKeys.enable=true
-op.format.externalRegAddToToken.update.symmetricKeys.requiredVersion=2
+op.format.externalRegAddToToken.update.symmetricKeys.enable=false
+op.format.externalRegAddToToken.update.symmetricKeys.requiredVersion=1
 op.enroll._000=#########################################
 op.enroll._001=# Default Operations
 op.enroll._002=#
diff --git a/base/tps/src/org/dogtagpki/server/tps/cms/KRARemoteRequestHandler.java b/base/tps/src/org/dogtagpki/server/tps/cms/KRARemoteRequestHandler.java
index aea41a29c..89304cbc9 100644
--- a/base/tps/src/org/dogtagpki/server/tps/cms/KRARemoteRequestHandler.java
+++ b/base/tps/src/org/dogtagpki/server/tps/cms/KRARemoteRequestHandler.java
@@ -18,6 +18,7 @@
 
 package org.dogtagpki.server.tps.cms;
 
+import java.math.BigInteger;
 import java.util.Hashtable;
 
 import org.dogtagpki.server.connector.IRemoteRequest;
@@ -75,7 +76,7 @@ public class KRARemoteRequestHandler extends RemoteRequestHandler
                 (TPSSubsystem) CMS.getSubsystem(TPSSubsystem.ID);
         HttpConnector conn =
                 (HttpConnector) subsystem.getConnectionManager().getConnector(connid);
-        CMS.debug("KRARemoteRequestHandler: serverSideKeyGen(): sending request to CA");
+        CMS.debug("KRARemoteRequestHandler: serverSideKeyGen(): sending request to KRA");
         HttpResponse resp;
         String request;
         if (isECC) {
@@ -231,28 +232,65 @@ public class KRARemoteRequestHandler extends RemoteRequestHandler
             String sDesKey,
             String b64cert)
             throws EBaseException {
+        return recoverKey(cuid, userid, sDesKey, b64cert, BigInteger.valueOf(0));
+    }
+
+    public KRARecoverKeyResponse recoverKey(
+            String cuid,
+            String userid,
+            String sDesKey,
+            String b64cert,
+            BigInteger keyid)
+            throws EBaseException {
 
         CMS.debug("KRARemoteRequestHandler: recoverKey(): begins.");
-        if (cuid == null || userid == null || sDesKey == null || b64cert == null) {
+        if (b64cert == null && keyid == BigInteger.valueOf(0)) {
+            throw new EBaseException("KRARemoteRequestHandler: recoverKey(): one of b64cert or kid has to be a valid value");
+        }
+        if (cuid == null || userid == null || sDesKey == null) {
             throw new EBaseException("KRARemoteRequestHandler: recoverKey(): input parameter null.");
         }
 
         TPSSubsystem subsystem =
                 (TPSSubsystem) CMS.getSubsystem(TPSSubsystem.ID);
+        CMS.debug("KRARemoteRequestHandler: getting conn id: " + connid);
         HttpConnector conn =
                 (HttpConnector) subsystem.getConnectionManager().getConnector(connid);
-        CMS.debug("KRARemoteRequestHandler: recoverKey(): sending request to CA");
+        if (conn == null) {
+            CMS.debug("KRARemoteRequestHandler: recoverKey(): conn null");
+            throw new EBaseException("KRARemoteRequestHandler: recoverKey(): conn null");
+        }
+        CMS.debug("KRARemoteRequestHandler: recoverKey(): sending request to KRA");
 
+        String sendMsg = null;
+        if (b64cert != null) { // recover by cert
+            sendMsg = IRemoteRequest.TOKEN_CUID + "=" +
+                    cuid +
+                    "&" + IRemoteRequest.KRA_UserId + "=" +
+                    userid +
+                    "&" + IRemoteRequest.KRA_RECOVERY_CERT + "=" +
+                    b64cert +
+                    "&" + IRemoteRequest.KRA_Trans_DesKey + "=" +
+                    sDesKey;
+        } else if (keyid != BigInteger.valueOf(0)){ // recover by keyid ... keyid != BigInteger.valueOf(0)
+            CMS.debug("KRARemoteRequestHandler: recoverKey(): keyid = " + keyid);
+            sendMsg = IRemoteRequest.TOKEN_CUID + "=" +
+                    cuid +
+                    "&" + IRemoteRequest.KRA_UserId + "=" +
+                    userid +
+                    "&" + IRemoteRequest.KRA_RECOVERY_KEYID + "=" +
+                    keyid.toString() +
+                    "&" + IRemoteRequest.KRA_Trans_DesKey + "=" +
+                    sDesKey;
+        }
+        CMS.debug("KRARemoteRequestHandler: recoverKey(): sendMsg =" + sendMsg);
         HttpResponse resp =
                 conn.send("TokenKeyRecovery",
-                        "&" + IRemoteRequest.TOKEN_CUID + "=" +
-                                cuid +
-                                "&" + IRemoteRequest.KRA_UserId + "=" +
-                                userid +
-                                "&" + IRemoteRequest.KRA_RECOVERY_CERT + "=" +
-                                b64cert +
-                                "&" + IRemoteRequest.KRA_Trans_DesKey + "=" +
-                                sDesKey);
+                        sendMsg);
+        if (resp == null) {
+            throw new EBaseException(
+                    "KRARemoteRequestHandler: recoverKey(): No response object returned from connection.");
+        }
 
         String content = resp.getContent();
 
diff --git a/base/tps/src/org/dogtagpki/server/tps/engine/TPSEngine.java b/base/tps/src/org/dogtagpki/server/tps/engine/TPSEngine.java
index 4580b46ca..7672a9bb7 100644
--- a/base/tps/src/org/dogtagpki/server/tps/engine/TPSEngine.java
+++ b/base/tps/src/org/dogtagpki/server/tps/engine/TPSEngine.java
@@ -539,7 +539,7 @@ public class TPSEngine {
             boolean isECC) throws TPSException {
 
         CMS.debug("TPSEngine.serverSideKeyGen entering... keySize: " + keySize + " cuid: " + cuid + " userid: "
-                + userid + " drConnId: " + drmConnId + " wrappedDesKey: " + wrappedDesKey + " archive: " + archive
+                + userid + " drmConnId: " + drmConnId + " wrappedDesKey: " + wrappedDesKey + " archive: " + archive
                 + " isECC: " + isECC);
 
         if (cuid == null || userid == null || drmConnId == null || wrappedDesKey == null) {
diff --git a/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegAttrs.java b/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegAttrs.java
index 2c85fefec..af8f52117 100644
--- a/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegAttrs.java
+++ b/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegAttrs.java
@@ -1,10 +1,12 @@
 package org.dogtagpki.server.tps.main;
 
 import java.util.ArrayList;
+
+import org.dogtagpki.server.tps.engine.TPSEngine;
+
 import com.netscape.certsrv.apps.CMS;
 import com.netscape.certsrv.base.EBaseException;
 import com.netscape.certsrv.base.IConfigStore;
-import org.dogtagpki.server.tps.engine.TPSEngine;
 
 public class ExternalRegAttrs {
     public String ldapAttrNameTokenType;
@@ -26,28 +28,28 @@ public class ExternalRegAttrs {
         String configName = null;
 
         try {
-        configName = "auths.instance." + authId + ".externalReg.tokenTypeAttributeName";
-        CMS.debug(method + ": getting config: " + configName);
-        ldapAttrNameTokenType = configStore.getString(configName,
-            "tokenType");
+            configName = "auths.instance." + authId + ".externalReg.tokenTypeAttributeName";
+            CMS.debug(method + ": getting config: " + configName);
+            ldapAttrNameTokenType = configStore.getString(configName,
+                    "tokenType");
 
-        configName = "auths.instance." + authId + ".externalReg.cuidAttributeName";
-        CMS.debug(method + ": getting config: " + configName);
-        ldapAttrNameTokenCUID = configStore.getString(configName,
-            "tokenCUID");
+            configName = "auths.instance." + authId + ".externalReg.cuidAttributeName";
+            CMS.debug(method + ": getting config: " + configName);
+            ldapAttrNameTokenCUID = configStore.getString(configName,
+                    "tokenCUID");
 
-        configName = "auths.instance." + authId + ".externalReg.certs.recoverAttributeName";
-        CMS.debug(method + ": getting config: " + configName);
-        ldapAttrNameCertsToRecover = configStore.getString(configName,
-            "certsToRecover");
+            configName = "auths.instance." + authId + ".externalReg.certs.recoverAttributeName";
+            CMS.debug(method + ": getting config: " + configName);
+            ldapAttrNameCertsToRecover = configStore.getString(configName,
+                    "certsToRecover");
 
             String RH_Delegation_Cfg = TPSEngine.CFG_EXTERNAL_REG + "." +
-                TPSEngine.CFG_ER_DELEGATION + ".enable";
+                    TPSEngine.CFG_ER_DELEGATION + ".enable";
             isDelegation = configStore.getBoolean(RH_Delegation_Cfg, false);
         } catch (EBaseException e) {
             CMS.debug("ExternalRegAttrs: unable to obtain certain config values.  Default to be used");
         }
-        
+
         certsToRecover = new ArrayList<ExternalRegCertToRecover>();
     }
 
diff --git a/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegCertToRecover.java b/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegCertToRecover.java
index 69585849b..dfc54d221 100644
--- a/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegCertToRecover.java
+++ b/base/tps/src/org/dogtagpki/server/tps/main/ExternalRegCertToRecover.java
@@ -7,9 +7,10 @@ public class ExternalRegCertToRecover {
     BigInteger serial;
     String caConn;
     String kraConn;
-    boolean ignoreForUpdateCerts;
+    boolean isRetainable;
 
     public ExternalRegCertToRecover() {
+        isRetainable = false;
     }
 
     public void setKeyid(BigInteger keyid) {
@@ -44,11 +45,11 @@ public class ExternalRegCertToRecover {
         return kraConn;
     }
 
-    public void setIgnoreForUpdateCerts(boolean ignore) {
-        ignoreForUpdateCerts = ignore;
+    public void setIsRetainable(boolean retainable) {
+        isRetainable = retainable;
     }
 
-    public boolean getIgnoreForUpdateCerts() {
-        return ignoreForUpdateCerts;
+    public boolean getIsRetainable() {
+        return isRetainable;
     }
 }
diff --git a/base/tps/src/org/dogtagpki/server/tps/main/PKCS11Obj.java b/base/tps/src/org/dogtagpki/server/tps/main/PKCS11Obj.java
index 6224c3f81..40e795115 100644
--- a/base/tps/src/org/dogtagpki/server/tps/main/PKCS11Obj.java
+++ b/base/tps/src/org/dogtagpki/server/tps/main/PKCS11Obj.java
@@ -212,7 +212,7 @@ public class PKCS11Obj {
         return tokenName;
     }
 
-    int getObjectSpecCount()
+    public int getObjectSpecCount()
     {
         return objectSpecs.size();
     }
diff --git a/base/tps/src/org/dogtagpki/server/tps/processor/TPSEnrollProcessor.java b/base/tps/src/org/dogtagpki/server/tps/processor/TPSEnrollProcessor.java
index 265ce0491..75e2d0e6a 100644
--- a/base/tps/src/org/dogtagpki/server/tps/processor/TPSEnrollProcessor.java
+++ b/base/tps/src/org/dogtagpki/server/tps/processor/TPSEnrollProcessor.java
@@ -53,7 +53,6 @@ import org.mozilla.jss.pkcs11.PK11RSAPublicKey;
 import org.mozilla.jss.pkix.primitive.SubjectPublicKeyInfo;
 
 import com.netscape.certsrv.apps.CMS;
-import com.netscape.certsrv.authentication.IAuthCredentials;
 import com.netscape.certsrv.base.EBaseException;
 import com.netscape.certsrv.base.EPropertyNotFound;
 import com.netscape.certsrv.base.IConfigStore;
@@ -82,7 +81,8 @@ public class TPSEnrollProcessor extends TPSProcessor {
     }
 
     private void enroll() throws TPSException, IOException {
-        CMS.debug("TPSEnrollProcessor enroll: entering...");
+        String method = "TPSEnrollProcessor.enroll:";
+        CMS.debug(method + " entering...");
         String auditMsg = null;
         TPSSubsystem tps = (TPSSubsystem) CMS.getSubsystem(TPSSubsystem.ID);
         TPSTokenPolicy tokenPolicy = new TPSTokenPolicy(tps);
@@ -102,16 +102,16 @@ public class TPSEnrollProcessor extends TPSProcessor {
         }
         appletInfo.setAid(getCardManagerAID());
 
-        CMS.debug("TPSEnrollProcessor.enroll: token cuid: " + appletInfo.getCUIDhexStringPlain());
+        CMS.debug(method + " token cuid: " + appletInfo.getCUIDhexStringPlain());
         boolean isTokenPresent = false;
 
         tokenRecord = isTokenRecordPresent(appletInfo);
 
         if (tokenRecord != null) {
-            CMS.debug("TPSEnrollProcessor.enroll: found token...");
+            CMS.debug(method + " found token...");
             isTokenPresent = true;
         } else {
-            CMS.debug("TPSEnrollProcessor.enroll: token does not exist in tokendb... create one in memory");
+            CMS.debug(method + " token does not exist in tokendb... create one in memory");
             tokenRecord = new TokenRecord();
             tokenRecord.setId(appletInfo.getCUIDhexStringPlain());
         }
@@ -136,7 +136,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             try {
                 authId = configStore.getString(configName);
             } catch (EBaseException e) {
-                CMS.debug("TPSEnrollProcessor.enroll: Internal Error obtaining mandatory config values. Error: " + e);
+                CMS.debug(method + " Internal Error obtaining mandatory config values. Error: " + e);
                 auditMsg = "TPS error getting config values from config store." + e.toString();
                 tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                         "failure");
@@ -144,8 +144,6 @@ public class TPSEnrollProcessor extends TPSProcessor {
                 throw new TPSException(auditMsg, TPSStatus.STATUS_ERROR_MISCONFIGURATION);
             }
 
-            /* get user login and password - set in "login" */
-            IAuthCredentials userCred;
             try {
                 CMS.debug("In TPSEnrollProcessor.enroll: isExternalReg: calling requestUserId");
                 TPSAuthenticator userAuth =
@@ -153,7 +151,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
                 processAuthentication(TPSEngine.ENROLL_OP, userAuth, cuid, tokenRecord);
             } catch (Exception e) {
                 // all exceptions are considered login failure
-                CMS.debug("TPSEnrollProcessor.enroll:: authentication exception thrown: " + e);
+                CMS.debug(method + ": authentication exception thrown: " + e);
                 auditMsg = "ExternalReg authentication failed, status = STATUS_ERROR_LOGIN";
 
                 tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
@@ -166,7 +164,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             ExternalRegAttrs erAttrs;
             try {
                 erAttrs = processExternalRegAttrs(authId);
-            } catch (EBaseException ee) {
+            } catch (Exception ee) {
                 auditMsg = "after processExternalRegAttrs: " + ee.toString();
                 tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                         "failure");
@@ -182,15 +180,22 @@ public class TPSEnrollProcessor extends TPSProcessor {
              * then any token can be used.
              */
             if (erAttrs.getTokenCUID() != null) {
-                CMS.debug("TPSEnrollProcessor.enroll: erAttrs.getTokenCUID()=" + erAttrs.getTokenCUID());
-                CMS.debug("TPSEnrollProcessor.enroll: tokenRecord.getId()=" + tokenRecord.getId());
-                if (!tokenRecord.getId().equals(erAttrs.getTokenCUID())) {
-                    auditMsg = "isExternalReg: token CUID not matching record:" + erAttrs.getTokenCUID();
-                    CMS.debug("TPSEnrollProcessor.enroll:" + auditMsg);
+                CMS.debug(method + " checking if token cuid matches record cuid");
+                CMS.debug(method + " erAttrs.getTokenCUID()=" + erAttrs.getTokenCUID());
+                CMS.debug(method + " tokenRecord.getId()=" + tokenRecord.getId());
+                if (!tokenRecord.getId().equalsIgnoreCase(erAttrs.getTokenCUID())) {
+                    auditMsg = "isExternalReg: token CUID not matching record:" + tokenRecord.getId() + " : " +
+                            erAttrs.getTokenCUID();
+                    CMS.debug(method + auditMsg);
                     tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                             "failure");
                     throw new TPSException(auditMsg, TPSStatus.STATUS_ERROR_NOT_TOKEN_OWNER);
+                } else {
+                    auditMsg = "isExternalReg: token CUID matches record";
+                    CMS.debug(method + auditMsg);
                 }
+            } else {
+                CMS.debug(method + " no need to check if token cuid matches record");
             }
 
             session.setExternalRegAttrs(erAttrs);
@@ -202,7 +207,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             } else {
                 // get the default externalReg tokenType
                 configName = "externalReg.default.tokenType";
-                CMS.debug("TPSEnrollProcessor.enroll: externalReg user entry does not contain tokenType...setting to config: "
+                CMS.debug(method + " externalReg user entry does not contain tokenType...setting to default config: "
                         + configName);
                 try {
                     tokenType = configStore.getString(configName,
@@ -211,7 +216,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
                             tokenType);
                     setSelectedTokenType(tokenType);
                 } catch (EBaseException e) {
-                    CMS.debug("TPSEnrollProcessor.enroll: Internal Error obtaining mandatory config values. Error: "
+                    CMS.debug(method + " Internal Error obtaining mandatory config values. Error: "
                             + e);
                     auditMsg = "TPS error getting config values from config store." + e.toString();
                     tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
@@ -230,20 +235,20 @@ public class TPSEnrollProcessor extends TPSProcessor {
 
             tokenType = resolveTokenProfile(resolverInstName, appletInfo.getCUIDhexString(), appletInfo.getMSNString(),
                     appletInfo.getMajorVersion(), appletInfo.getMinorVersion());
-            CMS.debug("TPSEnrollProcessor.enroll: resolved tokenType: " + tokenType);
+            CMS.debug(method + " resolved tokenType: " + tokenType);
         }
 
         checkProfileStateOK();
 
         boolean do_force_format = false;
         if (isTokenPresent) {
-            CMS.debug("TPSEnrollProcessor.enroll: token exists in tokendb");
+            CMS.debug(method + " token exists in tokendb");
 
             TokenStatus newState = TokenStatus.ACTIVE;
             // Check for transition to ACTIVE status.
 
             if (!tps.engine.isOperationTransitionAllowed(tokenRecord.getTokenStatus(), newState)) {
-                CMS.debug("TPSEnrollProcessor.enroll: token transition disallowed " +
+                CMS.debug(method + " token transition disallowed " +
                         tokenRecord.getTokenStatus() +
                         " to " + newState);
                 auditMsg = "Operation for CUID " + cuid +
@@ -265,7 +270,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             if (!isExternalReg &&
                     !tokenPolicy.isAllowdTokenReenroll(cuid) &&
                     !tokenPolicy.isAllowdTokenRenew(cuid)) {
-                CMS.debug("TPSEnrollProcessor.enroll: token renewal or reEnroll disallowed ");
+                CMS.debug(method + " token renewal or reEnroll disallowed ");
                 auditMsg = "Operation renewal or reEnroll for CUID " + cuid +
                         " Disabled";
                 tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
@@ -275,10 +280,10 @@ public class TPSEnrollProcessor extends TPSProcessor {
                         TPSStatus.STATUS_ERROR_DISABLED_TOKEN);
             } else {
                 auditMsg = "isExternalReg: skip token policy (reenroll, renewal) check";
-                CMS.debug("TPSEnrollProcessor.enroll:" + auditMsg);
+                CMS.debug(method + auditMsg);
             }
         } else {
-            CMS.debug("TPSEnrollProcessor.enroll: token does not exist");
+            CMS.debug(method + " token does not exist");
             tokenRecord.setStatus("uninitialized");
 
             checkAllowUnknownToken(TPSEngine.OP_FORMAT_PREFIX);
@@ -289,7 +294,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             checkAndAuthenticateUser(appletInfo, tokenType);
 
         if (do_force_format) {
-            CMS.debug("TPSEnrollProcessor.enroll: About to force format first due to policy.");
+            CMS.debug(method + " About to force format first due to policy.");
             //We will skip the auth step inside of format
             format(true);
         } else {
@@ -298,7 +303,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             appletInfo = getAppletInfo();
         }
 
-        CMS.debug("TPSEnrollProcessor.enroll: Finished updating applet if needed.");
+        CMS.debug(method + " Finished updating applet if needed.");
 
         //Check and upgrade keys if called for
         SecureChannel channel = checkAndUpgradeSymKeys(appletInfo,tokenRecord);
@@ -317,7 +322,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
         try {
             pkcs11objx = getCurrentObjectsOnToken(channel);
         } catch (DataFormatException e) {
-            auditMsg = "TPSEnrollProcessor.enroll: Failed to parse original token data: " + e.toString();
+            auditMsg = method + " Failed to parse original token data: " + e.toString();
             tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                     "failure");
 
@@ -347,8 +352,13 @@ public class TPSEnrollProcessor extends TPSProcessor {
         certsInfo.setStartProgress(15);
         certsInfo.setEndProgress(90);
 
+        // TODO:
+        // remove the not-to-be-retained cert objects from the pkcs11obj
+        // cleanObjectListBeforeExternalRecovery(certsInfo);
+
         boolean renewed = false;
         boolean recovered = false;
+
         TPSStatus status = TPSStatus.STATUS_NO_ERROR;
 
         if (!isExternalReg) {
@@ -371,7 +381,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             //tps.tdb.tdbActivity(ActivityDatabase.OP_RENEWAL, tokenRecord, session.getIpAddress(), auditMsg, "success");
         } else {
             auditMsg = " generateCertsAfterRenewalRecoveryPolicy returned status=" + status;
-            CMS.debug("TPSEnrollProcessor.enroll:" + auditMsg);
+            CMS.debug(method + auditMsg);
             tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                     "failure");
             throw new TPSException(auditMsg);
@@ -379,20 +389,21 @@ public class TPSEnrollProcessor extends TPSProcessor {
         if (!isExternalReg) {
             auditMsg = "generateCertsAfterRenewalRecoveryPolicy returns status:"
                     + EndOpMsg.statusToInt(status) + " : " + statusString;
-            CMS.debug("TPSEnrollProcessor.enroll: " + auditMsg);
+            CMS.debug(method + auditMsg);
         }
         if (status == TPSStatus.STATUS_NO_ERROR) {
             if (!generateCertificates(certsInfo, channel, appletInfo)) {
+                CMS.debug(method + "generateCertificates returned false means cert enrollment unsuccessful");
                 // in case isExternalReg, leave the token alone, do not format
                 if (!isExternalReg) {
-                    CMS.debug("TPSEnrollProcessor.enroll:generateCertificates returned false means some certs failed enrollment;  clean up (format) the token");
+                    CMS.debug(method + "generateCertificates returned false means some certs failed enrollment;  clean up (format) the token");
                     format(true /*skipAuth*/);
                 }
                 tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                         "failure");
                 throw new TPSException("generateCertificates failed");
             } else {
-                CMS.debug("TPSEnrollProcessor.enroll:generateCertificates returned true means cert enrollment successful");
+                CMS.debug(method + "generateCertificates returned true means cert enrollment successful");
                 /*
                  * isExternalReg -
                  * ??  Renew if token has "RENEW=YES" set by admin
@@ -407,14 +418,22 @@ public class TPSEnrollProcessor extends TPSProcessor {
                     try {
                         TPSStatus recoverStatus = externalRegRecover(cuid, userid, channel, certsInfo, appletInfo,
                                 tokenRecord);
-                        CMS.debug("TPSEnrollProcessor.enroll: after externalRegRecover status is:" + recoverStatus);
+                        CMS.debug(method + " after externalRegRecover status is:" + recoverStatus);
                         if (recoverStatus == TPSStatus.STATUS_ERROR_RECOVERY_IS_PROCESSED) {
                             recovered = true;
                             //TODO:
                             //tps.tdb.tdbActivity(ActivityDatabase.OP_RECOVERY, tokenRecord, session.getIpAddress(), auditMsg, "success");
+                        } else {
+                            auditMsg = method + " externalRegRecover: recoverStatus=" + recoverStatus;
+                            CMS.debug(auditMsg);
+                            tps.tdb.tdbActivity(ActivityDatabase.OP_RECOVERY, tokenRecord, session.getIpAddress(),
+                                    auditMsg,
+                                    "failure");
+
+                            throw new TPSException(auditMsg, TPSStatus.STATUS_ERROR_BAD_STATUS);
                         }
                     } catch (EBaseException e) {
-                        auditMsg = "TPSEnrollProcessor.enroll: externalRegRecover: " + e;
+                        auditMsg = method + " externalRegRecover: " + e;
                         CMS.debug(auditMsg);
                         tps.tdb.tdbActivity(ActivityDatabase.OP_RECOVERY, tokenRecord, session.getIpAddress(),
                                 auditMsg,
@@ -434,7 +453,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
                 status == TPSStatus.STATUS_ERROR_RENEWAL_IS_PROCESSED &&
                 tokenPolicy.isAllowdTokenRenew(cuid)) {
             renewed = true;
-            CMS.debug("TPSEnrollProcessor.enroll: renewal happened.. ");
+            CMS.debug(method + " renewal happened.. ");
         }
 
         /*
@@ -450,7 +469,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
 
         int lastObjVer = pkcs11objx.getOldObjectVersion();
 
-        CMS.debug("TPSEnrollProcessor.enroll: getOldObjectVersion: returning: " + lastObjVer);
+        CMS.debug(method + " getOldObjectVersion: returning: " + lastObjVer);
 
         if (lastObjVer != 0) {
             while (lastObjVer == 0xff) {
@@ -459,7 +478,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             }
 
             lastObjVer = lastObjVer + 1;
-            CMS.debug("TPSEnrollProcessor.enroll: Setting objectVersion to: " + lastObjVer);
+            CMS.debug(method + " Setting objectVersion to: " + lastObjVer);
             pkcs11objx.setObjectVersion(lastObjVer);
 
         }
@@ -489,53 +508,29 @@ public class TPSEnrollProcessor extends TPSProcessor {
             throw new TPSException(auditMsg);
         }
         //update the tokendb with new certs
-        CMS.debug("TPSEnrollProcessor.enroll: updating tokendb with certs.");
+        CMS.debug(method + " updating tokendb with certs.");
         try {
             // clean up the cert records used to belong to this token in tokendb
             tps.tdb.tdbRemoveCertificatesByCUID(tokenRecord.getId());
         } catch (Exception e) {
             auditMsg = "Attempt to clean up record with tdbRemoveCertificatesByCUID failed; token probably clean; continue anyway:"
                     + e;
-            CMS.debug("TPSEnrollProcessor.enroll:" + auditMsg);
+            CMS.debug(method + auditMsg);
         }
-        CMS.debug("TPSEnrollProcessor.enroll: adding certs to token with tdbAddCertificatesForCUID...");
+        CMS.debug(method + " adding certs to token with tdbAddCertificatesForCUID...");
         ArrayList<TPSCertRecord> certRecords = certsInfo.toTPSCertRecords(tokenRecord.getId(), tokenRecord.getUserID());
         tps.tdb.tdbAddCertificatesForCUID(tokenRecord.getId(), certRecords);
-        CMS.debug("TPSEnrollProcessor.enroll: tokendb updated with certs to the cuid so that it reflects what's on the token");
+        CMS.debug(method + " tokendb updated with certs to the cuid so that it reflects what's on the token");
 
         auditMsg = "appletVersion=" + lastObjVer + "; tokenType =" + selectedTokenType + "; userid =" + userid;
         tps.tdb.tdbActivity(ActivityDatabase.OP_ENROLLMENT, tokenRecord, session.getIpAddress(), auditMsg,
                 "success");
 
-        CMS.debug("TPSEnrollProcessor.enroll: leaving ...");
+        CMS.debug(method + " leaving ...");
 
         statusUpdate(100, "PROGRESS_DONE_ENROLLMENT");
     }
 
-/*
-    protected void checkIsDelegation() throws TPSException {
-        String method = "TPSEnrollProcessor.checkIsDelegation:";
-        String auditMsg;
-
-        IConfigStore configStore = CMS.getConfigStore();
-        CMS.debug(method + "begins");
-        String RH_Delegation_Cfg = TPSEngine.CFG_EXTERNAL_REG + "." +
-                TPSEngine.CFG_ER_DELEGATION + ".enable";
-
-        try {
-            //These defaults are well known, it is safe to use them.
-
-            this.isDelegation = configStore.getBoolean(RH_Delegation_Cfg, false);
-            CMS.debug(method + " isDelegation: " + isDelegation);
-        } catch (EBaseException e1) {
-            auditMsg = "Internal Error obtaining config values. Error: " + e1;
-            CMS.debug(method + auditMsg);
-            throw new TPSException(method + auditMsg);
-        }
-
-    }
-*/
-
     private void writeFinalPKCS11ObjectToToken(PKCS11Obj pkcs11objx, AppletInfo ainfo, SecureChannel channel)
             throws TPSException, IOException {
         if (pkcs11objx == null || ainfo == null || channel == null) {
@@ -666,6 +661,12 @@ public class TPSEnrollProcessor extends TPSProcessor {
         return pkcs11objx;
     }
 
+
+    private boolean isInCertsToRetainList(X509CertImpl xCert, ArrayList<ExternalRegCertToRecover> toBeRetained) {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
     /*
      * generateCertsAfterRenewalRecoveryPolicy determines whether a renewal or recovery is needed;
      * if recovery is needed, it determines which certificates (from which old token)
@@ -691,6 +692,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
         try {
             tokenRecords = tps.tdb.tdbFindTokenRecordsByUID(userid);
         } catch (Exception e) {
+            //TODO: when do you get here?
             // no existing record, means no "renewal" or "recovery" actions needed
             auditMsg = "no token associated with user: " + userid;
             CMS.debug(method + auditMsg);
@@ -884,6 +886,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
         if (session == null || session.getExternalRegAttrs() == null ||
                 session.getExternalRegAttrs().getCertsToRecover() == null) {
             CMS.debug(method + "nothing to recover...");
+            return status;
         }
         CMS.debug(method + "number of certs to recover=" +
                 session.getExternalRegAttrs().getCertsToRecoverCount());
@@ -896,6 +899,12 @@ public class TPSEnrollProcessor extends TPSProcessor {
             String caConn = erCert.getCaConn();
             String kraConn = erCert.getKraConn();
 
+            if (serial == null || caConn == null) {
+                //bail out right away;  we don't do half-baked recovery
+                CMS.debug(method + "invalid exterenalReg cert");
+                status = TPSStatus.STATUS_ERROR_RECOVERY_FAILED;
+                return status;
+            }
             auditMsg = "ExternalReg cert record: serial=" +
                     serial.toString();
 
@@ -909,15 +918,14 @@ public class TPSEnrollProcessor extends TPSProcessor {
             }
 
             String retCertB64 = certResp.getCertB64();
-            CMS.debug(method + "recovering:  retCertB64: " + retCertB64);
-            byte[] cert_bytes = Utils.base64decode(retCertB64);
-
-            TPSBuffer cert_bytes_buf = new TPSBuffer(cert_bytes);
-            CMS.debug(method + "recovering: retCertB64: "
-                    + cert_bytes_buf.toHexString());
-            if (retCertB64 != null)
-                CMS.debug(method + "recovering: cert b64 =" + retCertB64);
-            else {
+            if (retCertB64 != null) {
+                CMS.debug(method + "recovered:  retCertB64: " + retCertB64);
+                byte[] cert_bytes = Utils.base64decode(retCertB64);
+
+                TPSBuffer cert_bytes_buf = new TPSBuffer(cert_bytes);
+                CMS.debug(method + "recovered: retCertB64: "
+                        + cert_bytes_buf.toHexString());
+            } else {
                 auditMsg = "recovering cert b64 not found";
                 CMS.debug(method + auditMsg);
                 return TPSStatus.STATUS_ERROR_RECOVERY_FAILED;
@@ -926,7 +934,7 @@ public class TPSEnrollProcessor extends TPSProcessor {
             // recover keys
             KRARecoverKeyResponse keyResp = null;
             if (kraConn != null) {
-                auditMsg = "kraConn not null";
+                auditMsg = "kraConn not null:" + kraConn;
                 CMS.debug(method + auditMsg);
                 KRARemoteRequestHandler kraRH = new KRARemoteRequestHandler(kraConn);
                 if (channel.getDRMWrappedDesKey() == null) {
@@ -937,8 +945,21 @@ public class TPSEnrollProcessor extends TPSProcessor {
                     auditMsg = "channel.getDRMWrappedDesKey() not null";
                     CMS.debug(method + auditMsg);
                 }
-                keyResp = kraRH.recoverKey(cuid, userid, Util.specialURLEncode(channel.getDRMWrappedDesKey()),
-                        Util.uriEncode(retCertB64));
+
+                // if keyid > 0, recovder by keyid
+                if (keyid != null && keyid.compareTo(BigInteger.valueOf(0))==1) {
+                    auditMsg = "recovering by keyid: "+ keyid.toString();
+                    CMS.debug(method + auditMsg);
+
+                    keyResp = kraRH.recoverKey(cuid, userid, Util.specialURLEncode(channel.getDRMWrappedDesKey()),
+                            null, keyid);
+                } else {// otherwise, recover by cert
+                    auditMsg = "recovering by cert";
+                    CMS.debug(method + auditMsg);
+
+                    keyResp = kraRH.recoverKey(cuid, userid, Util.specialURLEncode(channel.getDRMWrappedDesKey()),
+                            Util.uriEncode(retCertB64));
+                }
                 if (keyResp == null) {
                     auditMsg = "recovering key not found";
                     CMS.debug(method + auditMsg);
@@ -1727,7 +1748,13 @@ public class TPSEnrollProcessor extends TPSProcessor {
             int currentCertIndex = certsInfo.getCurrentCertIndex();
             int totalNumCerts = certsInfo.getNumCertsToEnroll();
 
-            int progressBlock = (certsEndProgress - certsStartProgress) / totalNumCerts;
+            int progressBlock = 0;
+            if (totalNumCerts != 0) {
+                progressBlock =
+                   (certsEndProgress - certsStartProgress) / totalNumCerts;
+            } else {//TODO need to make this more accurate
+                CMS.debug("TPSEnrollProcessor.generateCertificate: totalNumCerts =0, progressBlock left at 0");
+            }
 
             int startCertProgValue = certsStartProgress + currentCertIndex * progressBlock;
 
diff --git a/base/tps/src/org/dogtagpki/server/tps/processor/TPSProcessor.java b/base/tps/src/org/dogtagpki/server/tps/processor/TPSProcessor.java
index 500dad412..27d88c2f1 100644
--- a/base/tps/src/org/dogtagpki/server/tps/processor/TPSProcessor.java
+++ b/base/tps/src/org/dogtagpki/server/tps/processor/TPSProcessor.java
@@ -1546,7 +1546,7 @@ public class TPSProcessor {
      * - parse the multi-valued attributes
      * @returns ExternalRegAttrs
      */
-    ExternalRegAttrs processExternalRegAttrs(/*IAuthToken authToken,*/String authId) throws EBaseException {
+    ExternalRegAttrs processExternalRegAttrs(/*IAuthToken authToken,*/String authId) throws NumberFormatException, EBaseException {
         String method = "processExternalRegAttrs";
         String configName;
         String tVal;
@@ -1563,10 +1563,11 @@ public class TPSProcessor {
             tVal = configStore.getString(configName,
                     "externalRegAddToToken");
             CMS.debug(method + ": set default tokenType:" + tVal);
+            erAttrs.setTokenType(tVal);
         } else {
             CMS.debug(method + ": retrieved tokenType:" + vals[0]);
+            erAttrs.setTokenType(vals[0]);
         }
-        erAttrs.setTokenType(vals[0]);
 
         CMS.debug(method + ": getting from authToken:"
                 + erAttrs.ldapAttrNameTokenCUID);
@@ -1584,19 +1585,27 @@ public class TPSProcessor {
                 + erAttrs.ldapAttrNameCertsToRecover);
         vals = authToken.getInStringArray(erAttrs.ldapAttrNameCertsToRecover);
         if (vals != null) {
+            // if any cert is mis-configured, the whole thing will bail
             for (String val : vals) {
                 CMS.debug(method + ": retrieved certsToRecover:" + val);
                 /*
                  * Each cert is represented as
-                 *    (serial#, caID, keyID, drmID)
+                 *    (serial#, caID, keyID, kraID)
                  * e.g.
-                 *    (1234, ca1, 81, drm1)
+                 *    (1234, ca1, 81, kra1)
                  *    note: numbers above are in decimal
+                 *    note: if keyID is less than or equal to 0, then recovery will be done by cert
+                 *          otherwise recovery is done by keyID
+                 *    note: if it only contains the serial# and caID (missing keyID and kraID)
+                 *          then it is used for retaining certs already existing on token
                  */
                 String[] items = val.split(",");
+                if (items.length !=2 && items.length !=4)
+                    throw new EBaseException(method + ": certsToRecover format error");
                 ExternalRegCertToRecover erCert =
                         new ExternalRegCertToRecover();
-                for (int i = 0; i < items.length; i++) {
+                int i = 0;
+                for (i = 0; i < items.length; i++) {
                     if (i == 0)
                         erCert.setSerial(new BigInteger(items[i]));
                     else if (i == 1)
@@ -1606,8 +1615,15 @@ public class TPSProcessor {
                     else if (i == 3)
                         erCert.setKraConn(items[i]);
                 }
+                /* TODO: for phase 3, retenable certs/keys
+                if (i<3) {
+                    erCert.setIsRetainable(true);
+                }
+                */
                 erAttrs.addCertToRecover(erCert);
             }
+        } else {
+            CMS.debug(method + ": certsToRecover attribute not found");
         }
 
         /*
@@ -1790,7 +1806,7 @@ public class TPSProcessor {
                 ExternalRegAttrs erAttrs;
                 try {
                     erAttrs = processExternalRegAttrs(/*authToken,*/authId);
-                } catch (EBaseException ee) {
+                } catch (Exception ee) {
                     auditMsg = "processExternalRegAttrs: " + ee.toString();
                     tps.tdb.tdbActivity(ActivityDatabase.OP_FORMAT, tokenRecord, session.getIpAddress(), auditMsg,
                             "failure");
@@ -2932,7 +2948,6 @@ public class TPSProcessor {
      *   then the returned value will be:
      *       John.Doe,e=JohnDoe@EXAMPLE.org,o=Example Org
      *
-     * TODO: It could be made more efficient
      */
     protected String mapPattern(LinkedHashMap<String, String> map, String inPattern) throws TPSException {
 
@@ -2946,6 +2961,11 @@ public class TPSProcessor {
         final char delim = '$';
         String pattern = inPattern;
 
+        /*
+         * Outer (while) loop searches for next token (in the format of $xxx$) to be mapped
+         *   when a pattern is found
+         *     inner (for) loop goes through all mappable params that the token maps to
+         */
         while (true) {
             String patternToMap = null;
             int firstPos = 0;
@@ -2966,7 +2986,7 @@ public class TPSProcessor {
 
             patternToMap = pattern.substring(firstPos + 1, nextPos);
 
-            CMS.debug("TPSProcessor.mapPattern: patternTo map: " + patternToMap);
+            //CMS.debug("TPSProcessor.mapPattern: patternTo map: " + patternToMap);
 
             String piece1 = "";
             if (firstPos >= 1)
@@ -2980,12 +3000,12 @@ public class TPSProcessor {
                 String key = entry.getKey();
 
                 String value = entry.getValue();
-                CMS.debug("TPSProcessor.mapPattern: Exposed: key: " + key + " Param: " + value);
+                //CMS.debug("TPSProcessor.mapPattern: Exposed: key: " + key + " Param: " + value);
 
                 if (key.equalsIgnoreCase(patternToMap)) {
                     CMS.debug("TPSProcessor.mapPattern: found match: key: " + key + " mapped to: " + value);
                     patternMapped = value;
-                    CMS.debug("TPSProcessor.mapPattern: pattern mapped: " + patternMapped);
+                    //CMS.debug("TPSProcessor.mapPattern: pattern mapped: " + patternMapped);
                     break;
                 }