Bugzilla Bug #522343 Add asynchronous key recovery mode

git-svn-id: svn+ssh://svn.fedorahosted.org/svn/pki/trunk@1027 c9f7a03b-bd48-0410-a16d-cbbf54688b0b

11 files changed, 1009 insertions, 21 deletions

diff --git a/pki/base/common/src/LogMessages_en.properties b/pki/base/common/src/LogMessages_en.properties
index 17cfe998..60fe43cd 100644
--- a/pki/base/common/src/LogMessages_en.properties
+++ b/pki/base/common/src/LogMessages_en.properties
@@ -1241,6 +1241,7 @@ CMSGW_ERROR_MARKING_CERT_REVOKED=Error encountered while marking certificate rev
 CMSGW_ERROR_MARKING_CERT_REVOKED_1=Error encountered while marking certificate revoked . {0}
 CMSGW_NO_RECOVERY_TOKEN_FOUND_1=No Recovery Token Found for recovery reference number {0}.
 CMSGW_INVALID_AGENT_3=Agent: {0} cannot retrieve PKCS #12 for recovery reference number {2}. Agent: {1} initialized the request.
+CMSGW_INVALID_AGENT_ASYNC_3=Agent: {0} cannot retrieve PKCS #12 for recovery request id {2}. Agent: {1} initialized the request.
 CMSGW_ERROR_DISPLAY_FILE=I/O Error encountered while outputting file.
 CMSGW_FILE_NOT_FOUND=File was not found
 CMSGW_NO_CERTS_FROM_CA=Error encountered while issuing certificates.
@@ -1972,6 +1973,14 @@ LOGGING_SIGNED_AUDIT_SERVER_SIDE_KEYGEN_PROCESSED_3=<type=SERVER_SIDE_KEYGEN_PRO
 #
 LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_4=<type=KEY_RECOVERY_REQUEST>:[AuditEvent=KEY_RECOVERY_REQUEST][SubjectID={0}][Outcome={1}][RecoveryID={2}][PubKey={3}] key recovery request made
 #
+# LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_ASYNC
+# - used when asynchronous key recovery request is made
+# RequestID  must be the recovery request ID
+# PubKey must be the base-64 encoded public key associated with
+#    the private key to be recovered
+#
+LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_ASYNC_4=<type=KEY_RECOVERY_REQUEST>:[AuditEvent=KEY_RECOVERY_REQUEST][SubjectID={0}][Outcome={1}][RequestID={2}][PubKey={3}] asynchronous key recovery request made
+#
 # LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN
 # - used when DRM agents login as recovery agents to approve
 #       key recovery requests
@@ -1989,6 +1998,14 @@ LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN_4=<type=KEY_RECOVERY_AGENT_LOGIN>:
 #
 LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_4=<type=KEY_RECOVERY_PROCESSED>:[AuditEvent=KEY_RECOVERY_PROCESSED][SubjectID={0}][Outcome={1}][RecoveryID={2}][RecoveryAgents={3}] key recovery request processed
 #
+# LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC
+# - used when key recovery request is processed
+# RequestID must be the recovery request ID
+# RecoveryAgents must be a comma-separated list of
+#       UIDs of the recovery agents approving this request
+#
+LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC_4=<type=KEY_RECOVERY_PROCESSED>:[AuditEvent=KEY_RECOVERY_PROCESSED][SubjectID={0}][Outcome={1}][ReQUESTID={2}][RecoveryAgents={3}] asynchronous key recovery request processed
+#
 # LOGGING_SIGNED_AUDIT_KEY_GEN_ASYMMETRIC
 # - used when asymmetric keys are generated
 #   (like when CA certificate requests are generated -
diff --git a/pki/base/common/src/UserMessages_en.properties b/pki/base/common/src/UserMessages_en.properties
index df5cfce4..6f272471 100644
--- a/pki/base/common/src/UserMessages_en.properties
+++ b/pki/base/common/src/UserMessages_en.properties
@@ -235,6 +235,7 @@ CMS_GW_DECODE_CRL_FAILED=Failed to DER decode the Certificate Revocation List.
 CMS_GW_CERT_SERIAL_NOT_FOUND=Certificate Serial Number {0} not found
 CMS_GW_NO_RECOVERY_TOKEN_FOUND=No Recovery Token Found for recovery reference number {0}.
 CMS_GW_INVALID_AGENT=Agent: {0} cannot retrieve PKCS #12 for recovery reference number {2}. Agent: {1} initialized the request.
+CMS_GW_INVALID_AGENT_ASYNC=Agent: {0} cannot retrieve PKCS #12 for recovery request id {2}. Agent: {1} initialized the request.
 CMS_GW_REDIRECTING_ADMINENROLL_ERROR=Error encountered while accessing the adminEnroll page.{0}
 CMS_GW_NO_PUB_MODULE=Publishing module is disabled.
 CMS_GW_CONVERT_DN_TO_X500NAME_ERROR=Cannot convert the subject name from a String to an X500 Name form. \nCheck to make sure valid characters are in the subject name. \nFor example, the email address should only have IA5String characters\nand the country should only have PrintableString characters and have 2 characters exactly.
diff --git a/pki/base/common/src/com/netscape/certsrv/kra/IKeyService.java b/pki/base/common/src/com/netscape/certsrv/kra/IKeyService.java
index e90a992d..5c2da2d0 100644
--- a/pki/base/common/src/com/netscape/certsrv/kra/IKeyService.java
+++ b/pki/base/common/src/com/netscape/certsrv/kra/IKeyService.java
@@ -43,6 +43,43 @@ public interface IKeyService {
      */
     public int getNoOfRequiredAgents() throws EBaseException;
 
+   /**
+    * is async recovery request status APPROVED -
+    *   i.e. all required # of recovery agents approved
+    * @param reqID request id
+    * @return true if  # of recovery required agents approved; false otherwise
+    */
+    public boolean isApprovedAsyncKeyRecovery(String reqID)
+        throws EBaseException;
+
+   /**
+    * get async recovery request initiating agent
+    * @param reqID request id
+    * @return agentUID
+    */
+    public String getInitAgentAsyncKeyRecovery(String reqID)
+        throws EBaseException;
+
+    /**
+     * Initiate asynchronous key recovery
+     * @param kid key identifier
+     * @param cert certificate embedded in PKCS12
+     * @return requestId
+     * @exception EBaseException failed to initiate async recovery
+     */
+    public String initAsyncKeyRecovery(BigInteger kid, X509CertImpl cert, String agent)
+         throws EBaseException;
+
+    /**
+     * add approving agent in asynchronous key recovery
+     * @param reqID request id
+     * @param agentID agent id
+     * @return requestId
+     * @exception EBaseException failed to initiate async recovery
+     */
+    public void addAgentAsyncKeyRecovery(String reqID, String agentID)
+         throws EBaseException;
+
     /**
      * Performs administrator-initiated key recovery.
      *
@@ -56,7 +93,31 @@ public interface IKeyService {
      */
     public byte[] doKeyRecovery(BigInteger kid,
         Credential creds[], String pwd, X509CertImpl cert,
-        String delivery, String nickname) throws EBaseException;
+        String delivery, String nickname, String agent) throws EBaseException;
+
+     /**
+     * Async Recovers key for administrators. This method is
+     * invoked by the agent operation of the key recovery servlet.
+     * <P>
+     *
+     * <ul>
+     * <li>signed.audit LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST used whenever
+     * a user private key recovery request is made (this is when the DRM
+     * receives the request)
+     * <li>signed.audit LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED used whenever
+     * a user private key recovery request is processed (this is when the DRM
+     * processes the request)
+     * </ul>
+     * @param reqID  request id
+     * @param password password of the PKCS12 package
+     * subsystem
+     * @exception EBaseException failed to recover key
+     * @return a byte array containing the key
+     */
+    public byte[] doKeyRecovery(
+        String reqID,
+        String password)
+        throws EBaseException;
 
     /**
      * Retrieves recovery identifier.
diff --git a/pki/base/common/src/com/netscape/cms/servlet/key/GetAsyncPk12.java b/pki/base/common/src/com/netscape/cms/servlet/key/GetAsyncPk12.java
new file mode 100644
index 00000000..eb510bf5
--- /dev/null
+++ b/pki/base/common/src/com/netscape/cms/servlet/key/GetAsyncPk12.java
@@ -0,0 +1,241 @@
+// --- BEGIN COPYRIGHT BLOCK ---
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; version 2 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// (C) 2007 Red Hat, Inc.
+// All rights reserved.
+// --- END COPYRIGHT BLOCK ---
+package com.netscape.cms.servlet.key;
+
+
+import com.netscape.cms.servlet.common.*;
+import com.netscape.cms.servlet.base.*;
+
+import java.io.*;
+import java.util.*;
+import java.net.*;
+import java.util.*;
+import java.text.*;
+import java.math.*;
+import java.security.*;
+import java.security.cert.X509Certificate;
+import javax.servlet.*;
+import javax.servlet.http.*;
+import netscape.security.x509.*;
+import com.netscape.certsrv.common.*;
+import com.netscape.certsrv.authority.*;
+import com.netscape.certsrv.base.*;
+ 
+import com.netscape.certsrv.dbs.*;
+import com.netscape.certsrv.dbs.keydb.*;
+
+import com.netscape.cms.servlet.*;
+import com.netscape.certsrv.request.*;
+import com.netscape.certsrv.request.IRequest;
+import com.netscape.certsrv.logging.*;
+import com.netscape.certsrv.kra.*;
+import com.netscape.certsrv.apps.*;
+import com.netscape.certsrv.authentication.*;
+import com.netscape.certsrv.authorization.*;
+
+
+/**
+ * Get the recovered key in PKCS#12 format
+ *   - for asynchronous key recovery only
+ *
+ */
+public class GetAsyncPk12 extends CMSServlet {
+
+    private final static String INFO = "getAsyncPk12";
+
+    private final static String TPL_FILE = "finishAsyncRecovery.template";
+
+    private final static String IN_PASSWORD = "p12Password";
+    private final static String IN_PASSWORD_AGAIN = "p12PasswordAgain";
+    private final static String OUT_RECOVERY_SUCCESS = "recoverySuccess";
+    private final static String OUT_ERROR = "errorDetails";
+
+    private com.netscape.certsrv.kra.IKeyService mService = null;
+    private final static String OUT_STATUS = "status";
+
+    private String mFormPath = null;
+
+    /**
+     * Constructs getAsyncPk12 servlet.
+     */
+    public GetAsyncPk12() {
+        super();
+    }
+
+    /**
+     * initialize the servlet. This servlet uses the template file
+     * "finishAsyncRecovery.template" to process the response.
+     *
+     * @param sc servlet configuration, read from the web.xml file
+     */
+    public void init(ServletConfig sc) throws ServletException {
+        super.init(sc);
+        mFormPath = "/agent/" + mAuthority.getId() + "/" + TPL_FILE;
+        mService = (com.netscape.certsrv.kra.IKeyService) mAuthority;
+
+        mTemplates.remove(CMSRequest.SUCCESS);
+        if (mOutputTemplatePath != null)
+            mFormPath = mOutputTemplatePath;
+    }
+
+    /**
+     * Returns serlvet information.
+     */
+    public String getServletInfo() { 
+        return INFO; 
+    }
+
+    /**
+     * Process the HTTP request.
+     * <ul>
+     * <li>http.param reqID request id for recovery
+     * </ul>
+     *
+     * @param cmsReq the object holding the request and response information
+     */
+    public void process(CMSRequest cmsReq) throws EBaseException {
+
+        HttpServletRequest req = cmsReq.getHttpReq();
+        HttpServletResponse resp = cmsReq.getHttpResp();
+
+        IAuthToken authToken = authenticate(cmsReq);
+
+        AuthzToken authzToken = null;
+
+        try {
+            authzToken = authorize(mAclMethod, authToken,
+                        mAuthzResourceName, "download");
+        } catch (EAuthzAccessDenied e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("ADMIN_SRVLT_AUTH_FAILURE", e.toString()));
+        } catch (Exception e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("ADMIN_SRVLT_AUTH_FAILURE", e.toString()));
+        }
+
+        if (authzToken == null) {
+            cmsReq.setStatus(CMSRequest.UNAUTHORIZED);
+            return;
+        }
+
+        CMSTemplate form = null;
+        Locale[] locale = new Locale[1];
+
+        try {
+            form = getTemplate(mFormPath, req, locale);
+        } catch (IOException e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("CMSGW_ERR_GET_TEMPLATE", mFormPath, e.toString()));
+            throw new ECMSGWException(
+              CMS.getUserMessage("CMS_GW_DISPLAY_TEMPLATE_ERROR"));
+        }
+
+        cmsReq.setStatus(CMSRequest.SUCCESS);
+        IArgBlock header = CMS.createArgBlock();
+        IArgBlock fixed = CMS.createArgBlock();
+        CMSTemplateParams argSet = new CMSTemplateParams(header, fixed);
+        int seq = -1;
+
+        // get status and populate argSet
+        try {
+            String reqID = req.getParameter("reqID");
+            header.addStringValue("reqID", reqID);
+
+            // only the init DRM agent can get the pkcs12
+            SessionContext sContext = SessionContext.getContext();
+            String agent = null;
+
+            if (sContext != null) {
+                agent = (String) sContext.get(SessionContext.USER_ID);
+            }
+
+            if (agent == null ) {
+                CMS.debug( "GetAsyncPk12::process() - agent is null!" );
+                throw new EBaseException( "agent is null" );
+            }
+
+            String initAgent = "undefined";
+            initAgent = mService.getInitAgentAsyncKeyRecovery(reqID);
+
+            if ((initAgent.equals("undefined")) || !agent.equals(initAgent)) {
+                log(ILogger.LL_SECURITY,
+                    CMS.getLogMessage("CMSGW_INVALID_AGENT_ASYNC_3",
+                        reqID, initAgent));
+                throw new ECMSGWException(
+                  CMS.getUserMessage("CMS_GW_INVALID_AGENT_ASYNC",
+                        reqID, initAgent));
+            }
+
+            // The async recovery request must be in "approved" state
+            //  i.e. all required # of recovery agents approved
+            if (mService.isApprovedAsyncKeyRecovery(reqID) != true) {
+                CMS.debug("GetAsyncPk12::process() - # required recovery agents not met");
+                throw new EBaseException( "# required recovery agents not met" );
+            }
+
+            String password = req.getParameter(IN_PASSWORD);
+            String passwordAgain = req.getParameter(IN_PASSWORD_AGAIN);
+
+            if (password == null || password.equals("")) {
+                header.addStringValue(OUT_ERROR, "PKCS12 password not found");
+                throw new EBaseException( "PKCS12 password not found" );
+            }
+            if (passwordAgain == null || !passwordAgain.equals(password)) {
+                header.addStringValue(OUT_ERROR, "PKCS12 password not matched");
+                throw new EBaseException( "PKCS12 password not matched" );
+            }
+
+            // got all approval, return pk12
+            byte pkcs12[] = mService.doKeyRecovery(reqID, password);
+
+            if (pkcs12 != null) {
+                try {
+                    resp.setContentType("application/x-pkcs12");
+                    resp.getOutputStream().write(pkcs12);
+                    mRenderResult = false;
+                    return;
+                } catch (IOException e) {
+                    header.addStringValue(OUT_ERROR,
+                        CMS.getUserMessage(locale[0], "CMS_BASE_INTERNAL_ERROR", e.toString()));
+                }
+            } else if (((IKeyRecoveryAuthority) mService).getError(reqID) != null) {
+                // error in recovery process 
+                header.addStringValue(OUT_ERROR, 
+                    ((IKeyRecoveryAuthority) mService).getError(reqID));
+            } else {
+                // pk12 hasn't been created yet. Shouldn't get here
+            }
+        } catch (EBaseException e) {
+            header.addStringValue(OUT_ERROR, e.toString(locale[0]));
+        }
+
+        try {
+            ServletOutputStream out = resp.getOutputStream();
+
+            resp.setContentType("text/html");
+            form.renderOutput(out, argSet);
+        } catch (IOException e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("CMSGW_ERR_STREAM_TEMPLATE", e.toString()));
+            throw new ECMSGWException(
+              CMS.getUserMessage("CMS_GW_DISPLAY_TEMPLATE_ERROR"));
+        }
+
+        cmsReq.setStatus(CMSRequest.SUCCESS);
+    }
+}
diff --git a/pki/base/common/src/com/netscape/cms/servlet/key/GrantAsyncRecovery.java b/pki/base/common/src/com/netscape/cms/servlet/key/GrantAsyncRecovery.java
new file mode 100644
index 00000000..dbb5356c
--- /dev/null
+++ b/pki/base/common/src/com/netscape/cms/servlet/key/GrantAsyncRecovery.java
@@ -0,0 +1,288 @@
+// --- BEGIN COPYRIGHT BLOCK ---
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; version 2 of the License.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// (C) 2010 Red Hat, Inc.
+// All rights reserved.
+// --- END COPYRIGHT BLOCK ---
+package com.netscape.cms.servlet.key;
+
+import com.netscape.certsrv.kra.IKeyRecoveryAuthority;
+import com.netscape.cms.servlet.common.*;
+import com.netscape.cms.servlet.base.*;
+
+import java.io.*;
+import java.util.*;
+import java.net.*;
+import java.util.*;
+import java.text.*;
+import java.math.*;
+import java.security.*;
+import java.security.cert.X509Certificate;
+import javax.servlet.*;
+import javax.servlet.http.*;
+import netscape.security.x509.*;
+import com.netscape.certsrv.common.*;
+import com.netscape.certsrv.authority.*;
+import com.netscape.certsrv.base.*;
+ 
+import com.netscape.certsrv.dbs.*;
+import com.netscape.certsrv.dbs.keydb.*;
+
+import com.netscape.cms.servlet.*;
+import com.netscape.certsrv.request.*;
+import com.netscape.certsrv.request.IRequest;
+import com.netscape.certsrv.logging.*;
+import com.netscape.certsrv.kra.*;
+import com.netscape.certsrv.apps.*;
+import com.netscape.certsrv.authentication.*;
+import com.netscape.certsrv.authorization.*;
+
+
+/**
+ * Approve an asynchronous key recovery request
+ *
+ */
+public class GrantAsyncRecovery extends CMSServlet {
+
+    private final static String INFO = "grantAsyncRecovery";
+    private final static String TPL_FILE = "grantAsyncRecovery.template";
+
+    private final static String IN_SERIALNO = "serialNumber";
+    private final static String IN_REQ_ID = "reqID";
+    private final static String IN_UID = "uid";
+    private final static String IN_CERT = "cert";
+
+    private final static String OUT_OP = "op";
+    private final static String OUT_SERIALNO = IN_SERIALNO;
+    private final static String OUT_RECOVERY_SUCCESS = "recoverySuccess";
+    private final static String OUT_SERVICE_URL = "serviceURL";
+    private final static String OUT_ERROR = "errorDetails";
+
+    private IKeyService mService = null;
+    private String mFormPath = null;
+
+    private final static String LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN =
+        "LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN_4";
+
+    /**
+     * Constructs EA servlet.
+     */
+    public GrantAsyncRecovery() {
+        super();
+    }
+
+    /**
+     * initialize the servlet. This servlet uses the template file
+     * 'grantAsyncRecovery.template' to process the response.
+     *
+     * @param sc servlet configuration, read from the web.xml file
+     */
+    public void init(ServletConfig sc) throws ServletException {
+        super.init(sc);
+        mFormPath = "/" + mAuthority.getId() + "/" + TPL_FILE;
+        mService = (IKeyService) mAuthority;
+
+        mTemplates.remove(CMSRequest.SUCCESS);
+
+        if (mOutputTemplatePath != null)
+            mFormPath = mOutputTemplatePath;
+    }
+
+    /**
+     * Returns serlvet information.
+     */
+    public String getServletInfo() { 
+        return INFO; 
+    }
+
+    /**
+     * Process the HTTP request.
+     * <ul>
+     * <li>http.param reqID request ID of the request to approve
+     * <li>http.param agentID User ID of the agent approving the request
+
+     * </ul>
+     *
+     * @param cmsReq the object holding the request and response information
+     */
+    public void process(CMSRequest cmsReq) throws EBaseException {
+
+        HttpServletRequest req = cmsReq.getHttpReq();
+        HttpServletResponse resp = cmsReq.getHttpResp();
+
+        CMS.debug("GrantAsyncRecovery: process() begins");
+
+        IAuthToken authToken = authenticate(cmsReq);
+
+        AuthzToken authzToken = null;
+
+        try {
+            authzToken = authorize(mAclMethod, authToken,
+                        mAuthzResourceName, "recover");
+        } catch (EAuthzAccessDenied e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("ADMIN_SRVLT_AUTH_FAILURE", e.toString()));
+        } catch (Exception e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("ADMIN_SRVLT_AUTH_FAILURE", e.toString()));
+        }
+
+        if (authzToken == null) {
+            cmsReq.setStatus(CMSRequest.UNAUTHORIZED);
+            return;
+        }
+
+        CMSTemplate form = null;
+        Locale[] locale = new Locale[1];
+
+        try {
+            form = getTemplate(mFormPath, req, locale);
+        } catch (IOException e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("CMSGW_ERR_GET_TEMPLATE", mFormPath, e.toString()));
+            throw new ECMSGWException(
+              CMS.getUserMessage("CMS_GW_DISPLAY_TEMPLATE_ERROR"));
+        }
+
+        IArgBlock header = CMS.createArgBlock();
+        IArgBlock fixed = CMS.createArgBlock();
+        CMSTemplateParams argSet = new CMSTemplateParams(header, fixed);
+
+        int seq = -1;
+
+        String agentID = authToken.getInString("uid");
+        CMS.debug("GrantAsyncRecovery: process() agent uid=" + agentID);
+        CMS.debug("GrantAsyncRecovery: process() request id=" + req.getParameter("reqID"));
+        try {
+            process(argSet, header, 
+                req.getParameter("reqID"),
+                agentID,
+                req, resp, locale[0]);
+        } catch (NumberFormatException e) {
+            header.addStringValue(OUT_ERROR,
+                CMS.getUserMessage(locale[0], "CMS_BASE_INTERNAL_ERROR", e.toString()));
+        }
+        try {
+            ServletOutputStream out = resp.getOutputStream();
+
+            resp.setContentType("text/html");
+            form.renderOutput(out, argSet);
+        } catch (IOException e) {
+            log(ILogger.LL_FAILURE,
+                CMS.getLogMessage("CMSGW_ERR_STREAM_TEMPLATE", e.toString()));
+            throw new ECMSGWException(
+              CMS.getUserMessage("CMS_GW_DISPLAY_TEMPLATE_ERROR"));
+        }
+        cmsReq.setStatus(CMSRequest.SUCCESS);
+    }
+
+    /**
+     * Update agent approval list
+     * <P>
+     *
+     * <ul>
+     * <li>signed.audit LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN used
+     * whenever DRM agents login as recovery agents to approve key recovery
+     * requests
+     * </ul>
+     * @param argSet CMS template parameters
+     * @param header argument block
+     * @param reqID string containing the recovery request ID
+     * @param agentID string containing the agent ID
+     * @param req HTTP servlet request
+     * @param resp HTTP servlet response
+     * @param locale the system locale
+     */
+    private void process(CMSTemplateParams argSet,
+        IArgBlock header, String reqID,
+        String agentID,
+        HttpServletRequest req, HttpServletResponse resp,
+        Locale locale) {
+        String auditMessage = null;
+        String auditSubjectID = auditSubjectID();
+        String auditRequestID = reqID;
+        String auditAgentID = agentID;
+
+        // "normalize" the "reqID"
+        if (auditRequestID != null) {
+            auditRequestID = auditRequestID.trim();
+
+            if (auditRequestID.equals("")) {
+                auditRequestID = ILogger.UNIDENTIFIED;
+            }
+        } else {
+            auditRequestID = ILogger.UNIDENTIFIED;
+        }
+
+        // "normalize" the "auditAgentID"
+        if (auditAgentID != null) {
+            auditAgentID = auditAgentID.trim();
+
+            if (auditAgentID.equals("")) {
+                auditAgentID = ILogger.UNIDENTIFIED;
+            }
+        } else {
+            auditAgentID = ILogger.UNIDENTIFIED;
+        }
+
+        try {
+            header.addStringValue(OUT_OP,
+                req.getParameter(OUT_OP));
+            header.addStringValue(OUT_SERVICE_URL,
+                req.getRequestURI());
+
+            // update approving agent list
+            mService.addAgentAsyncKeyRecovery(reqID, agentID);
+
+            header.addStringValue("requestID", reqID);
+            header.addStringValue("agentID", agentID);
+
+            // store a message in the signed audit log file
+            auditMessage = CMS.getLogMessage(
+                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN,
+                        auditSubjectID,
+                        ILogger.SUCCESS,
+                        auditRequestID,
+                        auditAgentID);
+
+            audit(auditMessage);
+
+        } catch (EBaseException e) {
+            header.addStringValue(OUT_ERROR, e.toString(locale));
+
+            // store a message in the signed audit log file
+            auditMessage = CMS.getLogMessage(
+                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN,
+                        auditSubjectID,
+                        ILogger.FAILURE,
+                        auditRequestID,
+                        auditAgentID);
+
+            audit(auditMessage);
+        } catch (Exception e) {
+            header.addStringValue(OUT_ERROR, e.toString());
+
+            // store a message in the signed audit log file
+            auditMessage = CMS.getLogMessage(
+                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_AGENT_LOGIN,
+                        auditSubjectID,
+                        ILogger.FAILURE,
+                        auditRequestID,
+                        auditAgentID);
+
+            audit(auditMessage);
+        }
+    }
+}
+
diff --git a/pki/base/common/src/com/netscape/cms/servlet/key/RecoverBySerial.java b/pki/base/common/src/com/netscape/cms/servlet/key/RecoverBySerial.java
index e88abccb..7882b815 100644
--- a/pki/base/common/src/com/netscape/cms/servlet/key/RecoverBySerial.java
+++ b/pki/base/common/src/com/netscape/cms/servlet/key/RecoverBySerial.java
@@ -176,14 +176,33 @@ public class RecoverBySerial extends CMSServlet {
 
         try {
             String localAgents = req.getParameter("localAgents");
+            String initAsyncRecovery = req.getParameter("initAsyncRecovery");
 
             // this information is needed within the server for
             // various signed audit log messages to report
             ctx = SessionContext.getContext();
-            ctx.put(SessionContext.RECOVERY_ID,
-                req.getParameter("recoveryID"));
 
-            byte pkcs12[] = process(form, argSet, header, 
+            /*
+               When Recovery is first initiated, if it is in asynch mode,
+               no pkcs#12 password is needed.
+               The initiating agent uid will be recorded in the recovery
+               request.
+               Later, as approving agents submit their approvals, they will
+               also be listed in the request.
+             */
+            if ((initAsyncRecovery != null) &&
+                   initAsyncRecovery.equalsIgnoreCase("ON")) {
+              process(form, argSet, header,
+                  req.getParameter(IN_SERIALNO),
+                  req.getParameter(IN_CERT),
+                  req, resp, locale[0]);
+
+              int requiredNumber = mService.getNoOfRequiredAgents();
+              header.addIntegerValue("noOfRequiredAgents", requiredNumber);
+            } else {
+              ctx.put(SessionContext.RECOVERY_ID,
+                req.getParameter("recoveryID"));
+              byte pkcs12[] = process(form, argSet, header, 
                     req.getParameter(IN_SERIALNO), 
                     req.getParameter("localAgents"),
                     req.getParameter(IN_PASSWORD),
@@ -193,13 +212,14 @@ public class RecoverBySerial extends CMSServlet {
                     req.getParameter(IN_NICKNAME),
                     req, resp, locale[0]);
 
-            if (pkcs12 != null) {
+              if (pkcs12 != null) {
                 //resp.setStatus(HttpServletResponse.SC_OK);
                 resp.setContentType("application/x-pkcs12");
                 //resp.setContentLength(pkcs12.length);
                 resp.getOutputStream().write(pkcs12);
                 mRenderResult = false;
                 return;
+              }
             }
         } catch (NumberFormatException e) {
             header.addStringValue(OUT_ERROR,
@@ -228,6 +248,61 @@ public class RecoverBySerial extends CMSServlet {
     }
 
     /**
+     * Async Key Recovery - request initiation
+     */
+    private void  process(CMSTemplate form, CMSTemplateParams argSet,
+        IArgBlock header, String seq, String cert,
+        HttpServletRequest req, HttpServletResponse resp,
+        Locale locale) {
+
+        // seq is the key id
+        if (seq == null) {
+            header.addStringValue(OUT_ERROR, "sequence number not found");
+            return;
+        }
+        X509CertImpl x509cert = null;
+
+        if (cert == null) {
+            header.addStringValue(OUT_ERROR, "certificate not found");
+            return;
+        } else {
+            try {
+                x509cert = Cert.mapCert(cert);
+            } catch (IOException e) {
+                header.addStringValue(OUT_ERROR, e.toString());
+            }
+        }
+        if (x509cert == null) {
+            header.addStringValue(OUT_ERROR, "invalid X.509 certificate");
+            return;
+        }
+
+        SessionContext sContext = SessionContext.getContext();
+
+        try {
+            String reqID = mService.initAsyncKeyRecovery(
+                  new BigInteger(seq), x509cert,
+                      (String) sContext.get(SessionContext.USER_ID));
+            header.addStringValue(OUT_SERIALNO, req.getParameter(IN_SERIALNO));
+            header.addStringValue("requestID", reqID);
+        } catch (EBaseException e) {
+            String error =
+                "Failed to recover key for key id " +
+                seq + ".\nException: " + e.toString();
+
+            CMS.getLogger().log(ILogger.EV_SYSTEM,
+                ILogger.S_KRA, ILogger.LL_FAILURE, error);
+            try {
+                ((IKeyRecoveryAuthority) mService).createError(seq, error);
+            } catch (EBaseException eb) {
+                CMS.getLogger().log(ILogger.EV_SYSTEM,
+                    ILogger.S_KRA, ILogger.LL_FAILURE, eb.toString());
+            }
+        }
+        return;
+    }
+
+    /**
      * Recovers a key. The p12 will be protected by the password
      * provided by the administrator.
      */
@@ -269,6 +344,12 @@ public class RecoverBySerial extends CMSServlet {
         try {
             Credential creds[] = null;
 
+            SessionContext sContext = SessionContext.getContext();
+            String agent = null;
+
+            if (sContext != null) {
+                agent = (String) sContext.get(SessionContext.USER_ID);
+            }
 	   if (CMS.getConfigStore().getBoolean("kra.keySplitting")) {
             if (localAgents == null) {
                 String recoveryID = req.getParameter("recoveryID");
@@ -283,12 +364,6 @@ public class RecoverBySerial extends CMSServlet {
 
                 header.addStringValue("recoveryID", recoveryID);
 
-                SessionContext sContext = SessionContext.getContext();
-                String agent = null;
-
-                if (sContext != null) {
-                    agent = (String) sContext.get(SessionContext.USER_ID);
-                }
                 params.put("agent", agent);
 
                 // new thread to wait for pk12
@@ -330,7 +405,7 @@ public class RecoverBySerial extends CMSServlet {
             byte pkcs12[] = mService.doKeyRecovery(
                     new BigInteger(seq),
                     creds, password, x509cert,	
-                    delivery, nickname);
+                    delivery, nickname, agent);
 
             return pkcs12;
           } else {
@@ -346,12 +421,6 @@ public class RecoverBySerial extends CMSServlet {
 
                 header.addStringValue("recoveryID", recoveryID);
 
-                SessionContext sContext = SessionContext.getContext();
-                String agent = null;
-
-                if (sContext != null) {
-                    agent = (String) sContext.get(SessionContext.USER_ID);
-                }
                 params.put("agent", agent);
 
                 // new thread to wait for pk12
@@ -423,11 +492,14 @@ public class RecoverBySerial extends CMSServlet {
                 return;
             }
 
+            SessionContext sContext = SessionContext.getContext();
+
             try {
                 byte pkcs12[] = mService.doKeyRecovery(
                         new BigInteger(theSeq),
                         creds, thePassword, theCert,
-                        theDelivery, theNickname);
+                        theDelivery, theNickname,
+                        (String) sContext.get(SessionContext.USER_ID));
 
                 ((IKeyRecoveryAuthority) mService).createPk12(theRecoveryID, pkcs12);
             } catch (EBaseException e) {
diff --git a/pki/base/common/src/com/netscape/cms/servlet/request/KeyReqParser.java b/pki/base/common/src/com/netscape/cms/servlet/request/KeyReqParser.java
index 4c633cd0..08a99b9f 100644
--- a/pki/base/common/src/com/netscape/cms/servlet/request/KeyReqParser.java
+++ b/pki/base/common/src/com/netscape/cms/servlet/request/KeyReqParser.java
@@ -71,6 +71,10 @@ public class KeyReqParser extends ReqParser {
             BigInteger kid = req.getExtDataInBigInteger("serialNumber");
 
             arg.addStringValue(OUTPUT_SERIALNO, kid.toString());
+
+            // for async recovery
+            String agents = (String) req.getExtDataInString("approvingAgents");
+            arg.addStringValue("approvingAgents", agents);
         } else {
             System.out.println("Bad Request " + type);
             // invalid request
diff --git a/pki/base/common/src/com/netscape/cms/servlet/request/QueryReq.java b/pki/base/common/src/com/netscape/cms/servlet/request/QueryReq.java
index 98f62416..664c4e83 100644
--- a/pki/base/common/src/com/netscape/cms/servlet/request/QueryReq.java
+++ b/pki/base/common/src/com/netscape/cms/servlet/request/QueryReq.java
@@ -51,6 +51,7 @@ public class QueryReq extends CMSServlet {
     private final static String IN_SHOW_ALL = "showAll";
     private final static String IN_SHOW_WAITING = "showWaiting";
     private final static String IN_SHOW_IN_SERVICE = "showInService";
+    private final static String IN_SHOW_PENDING= "showPending";
     private final static String IN_SHOW_CANCELLED = "showCancelled";
     private final static String IN_SHOW_REJECTED = "showRejected";
     private final static String IN_SHOW_COMPLETED = "showCompleted";
@@ -184,6 +185,8 @@ public class QueryReq extends CMSServlet {
             filter = "(requeststate=pending)";
         } else if (p.equals(IN_SHOW_IN_SERVICE)) {
             filter = "(requeststate=svc_pending)";
+        } else if (p.equals(IN_SHOW_PENDING)) {
+            filter = "(requeststate=pending)";
         } else if (p.equals(IN_SHOW_CANCELLED)) {
             filter = "(requeststate=canceled)";
         } else if (p.equals(IN_SHOW_REJECTED)) {
diff --git a/pki/base/kra/shared/webapps/kra/WEB-INF/web.xml b/pki/base/kra/shared/webapps/kra/WEB-INF/web.xml
index 8af3e56e..8d7baae0 100644
--- a/pki/base/kra/shared/webapps/kra/WEB-INF/web.xml
+++ b/pki/base/kra/shared/webapps/kra/WEB-INF/web.xml
@@ -249,6 +249,25 @@
    </servlet>
 
    <servlet>
+      <servlet-name>  kraKRAGrantAsyncRecovery  </servlet-name>
+      <servlet-class> com.netscape.cms.servlet.key.GrantAsyncRecovery  </servlet-class>
+             <init-param><param-name>  GetClientCert  </param-name>
+                         <param-value> true        </param-value> </init-param>
+             <init-param><param-name>  AuthzMgr    </param-name>
+                         <param-value> BasicAclAuthz </param-value> </init-param>
+             <init-param><param-name>  authority   </param-name>
+                         <param-value> kra         </param-value> </init-param>
+             <init-param><param-name>  templatePath   </param-name>
+                         <param-value> /agent/kra/grantAsyncRecovery.template</param-value> </init-param>
+             <init-param><param-name>  ID          </param-name>
+                         <param-value> kraKRAGrantAsyncRecovery </param-value> </init-param>
+             <init-param><param-name>  AuthMgr     </param-name>
+                         <param-value> certUserDBAuthMgr </param-value> </init-param>
+             <init-param><param-name>  resourceID  </param-name>
+                         <param-value> certServer.kra.key </param-value> </init-param>
+   </servlet>
+
+   <servlet>
       <servlet-name>  kraports  </servlet-name>
       <servlet-class> com.netscape.cms.servlet.base.PortsServlet  </servlet-class>
              <init-param><param-name>  ID          </param-name>
@@ -605,6 +624,23 @@
    </servlet>
 
    <servlet>
+      <servlet-name>  kraKRAGetAsyncPk12  </servlet-name>
+      <servlet-class> com.netscape.cms.servlet.key.GetAsyncPk12  </servlet-class>
+             <init-param><param-name>  GetClientCert  </param-name>
+                         <param-value> true        </param-value> </init-param>
+             <init-param><param-name>  AuthzMgr    </param-name>
+                         <param-value> BasicAclAuthz </param-value> </init-param>
+             <init-param><param-name>  authority   </param-name>
+                         <param-value> kra         </param-value> </init-param>
+             <init-param><param-name>  ID          </param-name>
+                         <param-value> kraKRAGetAsyncPk12 </param-value> </init-param>
+             <init-param><param-name>  AuthMgr     </param-name>
+                         <param-value> certUserDBAuthMgr </param-value> </init-param>
+             <init-param><param-name>  resourceID  </param-name>
+                         <param-value> certServer.kra.key </param-value> </init-param>
+   </servlet>
+
+   <servlet>
       <servlet-name>  kraGrantRecovery  </servlet-name>
       <servlet-class> com.netscape.cms.servlet.base.DisplayHtmlServlet  </servlet-class>
              <init-param><param-name>  GetClientCert  </param-name>
@@ -945,6 +981,11 @@
    </servlet-mapping>
 
    <servlet-mapping>
+      <servlet-name>  kraKRAGrantAsyncRecovery  </servlet-name>
+      <url-pattern>   /agent/kra/grantAsyncRecovery  </url-pattern>
+   </servlet-mapping>
+
+   <servlet-mapping>
       <servlet-name>  kraports  </servlet-name>
       <url-pattern>   /ee/kra/ports  </url-pattern>
    </servlet-mapping>
@@ -1035,6 +1076,11 @@
    </servlet-mapping>
 
    <servlet-mapping>
+      <servlet-name>  kraKRAGetAsyncPk12  </servlet-name>
+      <url-pattern>   /agent/kra/getAsyncPk12  </url-pattern>
+   </servlet-mapping>
+
+   <servlet-mapping>
       <servlet-name>  kraGrantRecovery  </servlet-name>
       <url-pattern>   /agent/kra/grantRecovery.html  </url-pattern>
    </servlet-mapping>
diff --git a/pki/base/kra/src/com/netscape/kra/KeyRecoveryAuthority.java b/pki/base/kra/src/com/netscape/kra/KeyRecoveryAuthority.java
index c55a949c..ce2a03b2 100644
--- a/pki/base/kra/src/com/netscape/kra/KeyRecoveryAuthority.java
+++ b/pki/base/kra/src/com/netscape/kra/KeyRecoveryAuthority.java
@@ -124,8 +124,12 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
         "LOGGING_SIGNED_AUDIT_PRIVATE_KEY_ARCHIVE_PROCESSED_3";
     private final static String LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST =
         "LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_4";
+    private final static String LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_ASYNC =
+        "LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_ASYNC_4";
     private final static String LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED =
         "LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_4";
+    private final static String LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC =
+        "LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC_4";
 
     /**
      * Constructs an escrow authority.
@@ -622,6 +626,7 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
 
         synchronized (lock) {
             while (dc.size() < getNoOfRequiredAgents()) { 
+	    CMS.debug("KeyRecoveryAuthority: cfu in synchronized lock for getDistributedCredentials");
                 try {
                     lock.wait();
                 } catch (InterruptedException e) {
@@ -783,6 +788,154 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
     }
 
     /**
+     * async key recovery initiation
+     */
+    public String initAsyncKeyRecovery(BigInteger kid, X509CertImpl cert, String agent)
+         throws EBaseException {
+
+        String auditPublicKey = auditPublicKey(cert);
+        String auditRecoveryID = "undefined";
+        String auditMessage = null;
+        String auditSubjectID = auditSubjectID();
+
+        IRequestQueue queue = null;
+        IRequest r = null;
+
+        try {
+            queue = getRequestQueue();
+            r = queue.newRequest(KRAService.RECOVERY);
+
+            r.setExtData(RecoveryService.ATTR_SERIALNO, kid);
+            r.setExtData(RecoveryService.ATTR_USER_CERT, cert);
+            // first one in the "approvingAgents" list is the initiating agent
+            r.setExtData(RecoveryService.ATTR_APPROVE_AGENTS, agent);
+            r.setRequestStatus(RequestStatus.PENDING);
+            queue.updateRequest(r);
+            auditRecoveryID = r.getRequestId().toString();
+
+            // store a message in the signed audit log file
+            auditMessage = CMS.getLogMessage(
+                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_ASYNC,
+                        auditSubjectID,
+                        ILogger.SUCCESS,
+                        auditRecoveryID,
+                        auditPublicKey);
+
+            audit(auditMessage);
+        } catch (EBaseException eAudit1) {
+            // store a message in the signed audit log file
+            auditMessage = CMS.getLogMessage(
+                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST_ASYNC,
+                        auditSubjectID,
+                        ILogger.FAILURE,
+                        auditRecoveryID,
+                        auditPublicKey);
+
+            audit(auditMessage);
+
+            throw eAudit1;
+        }
+
+        //NO call to queue.processRequest(r) because it is only initiating
+        return r.getRequestId().toString();
+    }
+
+   /**
+    * is async recovery request status APPROVED -
+    *   i.e. all required # of recovery agents approved
+    */
+    public boolean isApprovedAsyncKeyRecovery(String reqID)
+        throws EBaseException {
+        IRequestQueue queue = null;
+        IRequest r = null;
+
+        queue = getRequestQueue();
+        r = queue.findRequest(new RequestId(reqID));
+        if ((r.getRequestStatus() == RequestStatus.APPROVED)) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+   /**
+    * get async recovery request initiating agent
+    */
+    public String getInitAgentAsyncKeyRecovery(String reqID)
+        throws EBaseException {
+        IRequestQueue queue = null;
+        IRequest r = null;
+
+        queue = getRequestQueue();
+        r = queue.findRequest(new RequestId(reqID));
+
+        String agents = r.getExtDataInString(RecoveryService.ATTR_APPROVE_AGENTS);
+        if (agents != null) {
+            int i = agents.indexOf(",");
+            if (i == -1) {
+                return agents;
+            }
+            return agents.substring(0, i);
+        } else { // no approvingAgents existing, can't be async recovery
+            CMS.debug("getInitAgentAsyncKeyRecovery: no approvingAgents in request");
+        }
+
+        return null;
+    }
+
+   /**
+    * add async recovery agent to approving agent list of the recovery request
+    * record
+    * This method will check to see if the agent belongs to the recovery group
+    * first before adding.
+    */
+    public void addAgentAsyncKeyRecovery(String reqID, String agentID)
+        throws EBaseException {
+        IRequestQueue queue = null;
+        IRequest r = null;
+
+        // check if the uid is in the specified group
+        IUGSubsystem ug = (IUGSubsystem) CMS.getSubsystem(CMS.SUBSYSTEM_UG);
+        if (!ug.isMemberOf(agentID, mConfig.getString("recoveryAgentGroup"))) {
+            // invalid group
+            throw new EBaseException(CMS.getUserMessage("CMS_KRA_CREDENTIALS_NOT_EXIST"));
+        }
+
+        queue = getRequestQueue();
+        r = queue.findRequest(new RequestId(reqID));
+
+        String agents = r.getExtDataInString(RecoveryService.ATTR_APPROVE_AGENTS);
+        if (agents != null) {
+            int count = 0;
+            StringTokenizer st = new StringTokenizer(agents, ",");
+            for (; st.hasMoreTokens();) {
+                String a = st.nextToken();
+                // first one is the initiating agent
+                if ((count != 0) && a.equals(agentID)) {
+                  // duplicated uid
+                  throw new EBaseException(CMS.getUserMessage("CMS_KRA_CREDENTIALS_EXIST"));
+                }
+                count++;
+            }
+
+            // note: if count==1 and required agents is 1, it's good to add
+            // and it'd look like "agent1,agent1" - that's the only dup allowed
+            if (count <= getNoOfRequiredAgents()) { //all good, add it
+                r.setExtData(RecoveryService.ATTR_APPROVE_AGENTS,
+                      agents+","+agentID);
+                if (count == getNoOfRequiredAgents()) {
+                    r.setRequestStatus(RequestStatus.APPROVED);
+                } else {
+                    r.setRequestStatus(RequestStatus.PENDING);
+                }
+                queue.updateRequest(r);
+            }
+        } else { // no approvingAgents existing, can't be async recovery
+            CMS.debug("addAgentAsyncKeyRecovery: no approvingAgents in request. Async recovery request not initiated?");
+        }
+    }
+
+    /**
      * Recovers key for administrators. This method is
      * invoked by the agent operation of the key recovery servlet.
      * <P>
@@ -808,7 +961,8 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
     public byte[] doKeyRecovery(BigInteger kid,
         Credential creds[], String password, 
         X509CertImpl cert,
-        String delivery, String nickname) 
+        String delivery, String nickname,
+        String agent) 
         throws EBaseException {
         String auditMessage = null;
         String auditSubjectID = auditSubjectID();
@@ -820,7 +974,7 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
         IRequest r = null;
         Hashtable params = null;
 
-
+        CMS.debug("KeyRecoveryAuthority: in synchronous doKeyRecovery()");
         // ensure that any low-level exceptions are reported
         // to the signed audit log and stored as failures
         try {
@@ -843,6 +997,8 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
                     r.setExtData(RecoveryService.ATTR_NICKNAME, nickname);
                 }
             }
+            // for both sync and async recovery
+            r.setExtData(RecoveryService.ATTR_APPROVE_AGENTS, agent);
 
             // store a message in the signed audit log file
             auditMessage = CMS.getLogMessage(
@@ -919,6 +1075,102 @@ public class KeyRecoveryAuthority implements IAuthority, IKeyService, IKeyRecove
         }
     }
 
+     /**
+     * Async Recovers key for administrators. This method is
+     * invoked by the agent operation of the key recovery servlet.
+     * <P>
+     *
+     * <ul>
+     * <li>signed.audit LOGGING_SIGNED_AUDIT_KEY_RECOVERY_REQUEST used whenever
+     * a user private key recovery request is made (this is when the DRM
+     * receives the request)
+     * <li>signed.audit LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED used whenever
+     * a user private key recovery request is processed (this is when the DRM
+     * processes the request)
+     * </ul>
+     * @param requestID  request id
+     * @param password password of the PKCS12 package
+     * subsystem
+     * @exception EBaseException failed to recover key
+     * @return a byte array containing the key
+     */
+    public byte[] doKeyRecovery(
+        String reqID, 
+        String password)
+        throws EBaseException {
+        String auditMessage = null;
+        String auditSubjectID = auditSubjectID();
+        String auditRecoveryID = reqID;
+        String auditAgents = ILogger.SIGNED_AUDIT_EMPTY_VALUE;
+        String auditPublicKey = ILogger.SIGNED_AUDIT_EMPTY_VALUE;
+
+        IRequestQueue queue = null;
+        IRequest r = null;
+        Hashtable params = null;
+
+        CMS.debug("KeyRecoveryAuthority: in asynchronous doKeyRecovery()");
+        queue = getRequestQueue();
+        r = queue.findRequest(new RequestId(reqID));
+
+        auditAgents = 
+            r.getExtDataInString(RecoveryService.ATTR_APPROVE_AGENTS);
+
+        // set transient parameters
+        params = createVolatileRequest(r.getRequestId());
+        params.put(RecoveryService.ATTR_TRANSPORT_PWD, password);
+
+        // ensure that any low-level exceptions are reported
+        // to the signed audit log and stored as failures
+        try {
+            CMS.debug("KeyRecoveryAuthority: in asynchronous doKeyRecovery(), request state ="+ r.getRequestStatus().toString());
+            // can only process requests in begin state
+            r.setRequestStatus(RequestStatus.BEGIN);
+            queue.processRequest(r);
+
+            if (r.getExtDataInString(IRequest.ERROR) == null) {
+                byte pkcs12[] = (byte[]) params.get(
+                        RecoveryService.ATTR_PKCS12);
+
+                // store a message in the signed audit log file
+                auditMessage = CMS.getLogMessage(
+                            LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC,
+                            auditSubjectID,
+                            ILogger.SUCCESS,
+                            auditRecoveryID,
+                            auditAgents);
+
+                audit(auditMessage);
+
+                destroyVolatileRequest(r.getRequestId());
+
+                return pkcs12;
+            } else {
+                // store a message in the signed audit log file
+                auditMessage = CMS.getLogMessage(
+                            LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC,
+                            auditSubjectID,
+                            ILogger.FAILURE,
+                            auditRecoveryID,
+                            auditAgents);
+
+                audit(auditMessage);
+
+                throw new EBaseException(r.getExtDataInString(IRequest.ERROR));
+            }
+        } catch (EBaseException eAudit1) {
+            // store a message in the signed audit log file
+            auditMessage = CMS.getLogMessage(
+                        LOGGING_SIGNED_AUDIT_KEY_RECOVERY_PROCESSED_ASYNC,
+                        auditSubjectID,
+                        ILogger.FAILURE,
+                        auditRecoveryID,
+                        auditAgents);
+
+            audit(auditMessage);
+            throw eAudit1;
+        }
+    }
+
     /**
      * Constructs a recovery request and submits it
      * to the request subsystem for processing.
diff --git a/pki/base/kra/src/com/netscape/kra/RecoveryService.java b/pki/base/kra/src/com/netscape/kra/RecoveryService.java
index 39253ab6..94915640 100644
--- a/pki/base/kra/src/com/netscape/kra/RecoveryService.java
+++ b/pki/base/kra/src/com/netscape/kra/RecoveryService.java
@@ -85,6 +85,9 @@ public class RecoveryService implements IService {
     public static final String ATTR_USER_CERT = "cert"; 
     public static final String ATTR_DELIVERY = "delivery";
 
+    // for Async Key Recovery
+    public static final String ATTR_APPROVE_AGENTS = "approvingAgents";
+
     private IKeyRecoveryAuthority mKRA = null;
     private IKeyRepository mStorage = null;
     private IStorageKeyUnit mStorageUnit = null;