Make sure the CA is running when starting services

- Provide a function for determinig the CA status using Dogtag 10's new
  getStatus endpoint.
  This must be done over HTTPS, but since our client certificate may not be set
  up yet, we need HTTPS without client authentication.
  Rather than copying from the existing http_request and https_request
  function, shared code is factored out to a common helper.
- Call the new function when restarting the CA service. Since our Service
  can only be extended in platform-specific code, do this for Fedora only.
  Also, the status is only checked with Dogtag 10+.
- When a restart call in cainstance failed, users were refered to the
  installation log, but no info was actually logged. Log the exception.

https://fedorahosted.org/freeipa/ticket/3084

3 files changed, 162 insertions, 61 deletions

diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index 907cebc61..5cf5a9df8 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -130,6 +130,15 @@ def configured_constants(api=None):
         return Dogtag9Constants
 
 
+def error_from_xml(doc, message_template):
+    try:
+        item_node = doc.getElementsByTagName("Error")
+        reason = item_node[0].childNodes[0].data
+        return errors.RemoteRetrieveError(reason=reason)
+    except Exception, e:
+        return errors.RemoteRetrieveError(reason=message_template % e)
+
+
 def get_ca_certchain(ca_host=None):
     """
     Retrieve the CA Certificate chain from the configured Dogtag server.
@@ -151,13 +160,8 @@ def get_ca_certchain(ca_host=None):
                 item_node = doc.getElementsByTagName("ChainBase64")
                 chain = item_node[0].childNodes[0].data
             except IndexError:
-                try:
-                    item_node = doc.getElementsByTagName("Error")
-                    reason = item_node[0].childNodes[0].data
-                    raise errors.RemoteRetrieveError(reason=reason)
-                except Exception, e:
-                    raise errors.RemoteRetrieveError(
-                        reason=_("Retrieving CA cert chain failed: %s") % e)
+                raise error_from_xml(
+                    doc, _("Retrieving CA cert chain failed: %s"))
         finally:
             if doc:
                 doc.unlink()
@@ -167,32 +171,107 @@ def get_ca_certchain(ca_host=None):
 
     return chain
 
+
+def ca_status(ca_host=None):
+    """Return the status of the CA, and the httpd proxy in front of it
+
+    The returned status can be:
+    - running
+    - starting
+    - Service Temporarily Unavailable
+    """
+    if ca_host is None:
+        ca_host = api.env.ca_host
+    # Use port 443 to test the proxy as well
+    status, reason, headers, body = unauthenticated_https_request(
+        ca_host, 443, '/ca/admin/ca/getStatus')
+    if status == 503:
+        # Service temporarily unavailable
+        return reason
+    elif status != 200:
+        raise errors.RemoteRetrieveError(
+            reason=_("Retrieving CA status failed: %s") % reason)
+    doc = xml.dom.minidom.parseString(body)
+    try:
+        item_node = doc.getElementsByTagName("XMLResponse")[0]
+        item_node = item_node.getElementsByTagName("Status")[0]
+        return item_node.childNodes[0].data
+    except IndexError:
+        raise error_from_xml(doc, _("Retrieving CA status failed: %s"))
+
+
 def https_request(host, port, url, secdir, password, nickname, **kw):
     """
-    :param url: The URL to post to.
+    :param url: The path (not complete URL!) to post to.
     :param kw:  Keyword arguments to encode into POST body.
     :return:   (http_status, http_reason_phrase, http_headers, http_body)
                as (integer, unicode, dict, str)
 
     Perform a client authenticated HTTPS request
     """
-    if isinstance(host, unicode):
-        host = host.encode('utf-8')
-    uri = 'https://%s%s' % (ipautil.format_netloc(host, port), url)
-    post = urlencode(kw)
-    root_logger.debug('https_request %r', uri)
-    root_logger.debug('https_request post %r', post)
-    request_headers = {"Content-type": "application/x-www-form-urlencoded",
-                       "Accept": "text/plain"}
-    try:
+
+    def connection_factory(host, port):
         conn = nsslib.NSSConnection(host, port, dbdir=secdir)
         conn.set_debuglevel(0)
         conn.connect()
-        conn.sock.set_client_auth_data_callback(nsslib.client_auth_data_callback,
-                                                nickname,
-                                                password, nss.get_default_certdb())
-        conn.request("POST", url, post, request_headers)
+        conn.sock.set_client_auth_data_callback(
+            nsslib.client_auth_data_callback,
+            nickname, password, nss.get_default_certdb())
+        return conn
+
+    body = urlencode(kw)
+    return _httplib_request(
+            'https', host, port, url, connection_factory, body)
+
+
+def http_request(host, port, url, **kw):
+    """
+    :param url: The path (not complete URL!) to post to.
+    :param kw: Keyword arguments to encode into POST body.
+    :return:   (http_status, http_reason_phrase, http_headers, http_body)
+                as (integer, unicode, dict, str)
+
+    Perform an HTTP request.
+    """
+    body = urlencode(kw)
+    return _httplib_request(
+        'http', host, port, url, httplib.HTTPConnection, body)
+
+
+def unauthenticated_https_request(host, port, url, **kw):
+    """
+    :param url: The path (not complete URL!) to post to.
+    :param kw: Keyword arguments to encode into POST body.
+    :return:   (http_status, http_reason_phrase, http_headers, http_body)
+                as (integer, unicode, dict, str)
+
+    Perform an unauthenticated HTTPS request.
+    """
+    body = urlencode(kw)
+    return _httplib_request(
+        'https', host, port, url, httplib.HTTPSConnection, body)
+
 
+def _httplib_request(
+        protocol, host, port, path, connection_factory, request_body):
+    """
+    :param request_body: Request body
+    :param connection_factory: Connection class to use. Will be called
+        with the host and port arguments.
+
+    Perform a HTTP(s) request.
+    """
+    if isinstance(host, unicode):
+        host = host.encode('utf-8')
+    uri = '%s://%s%s' % (protocol, ipautil.format_netloc(host, port), path)
+    root_logger.info('request %r', uri)
+    root_logger.debug('request body %r', request_body)
+    try:
+        conn = connection_factory(host, port)
+        conn.request('POST', uri,
+            body=request_body,
+            headers={'Content-type': 'application/x-www-form-urlencoded'},
+        )
         res = conn.getresponse()
 
         http_status = res.status
@@ -203,42 +282,9 @@ def https_request(host, port, url, secdir, password, nickname, **kw):
     except Exception, e:
         raise NetworkError(uri=uri, error=str(e))
 
-    return http_status, http_reason_phrase, http_headers, http_body
+    root_logger.debug('request status %d',        http_status)
+    root_logger.debug('request reason_phrase %r', http_reason_phrase)
+    root_logger.debug('request headers %s',       http_headers)
+    root_logger.debug('request body %r',          http_body)
 
-def http_request(host, port, url, **kw):
-        """
-        :param url: The URL to post to.
-        :param kw: Keyword arguments to encode into POST body.
-        :return:   (http_status, http_reason_phrase, http_headers, http_body)
-                   as (integer, unicode, dict, str)
-
-        Perform an HTTP request.
-        """
-        if isinstance(host, unicode):
-            host = host.encode('utf-8')
-        uri = 'http://%s%s' % (ipautil.format_netloc(host, port), url)
-        post = urlencode(kw)
-        root_logger.info('request %r', uri)
-        root_logger.debug('request post %r', post)
-        conn = httplib.HTTPConnection(host, port)
-        try:
-            conn.request('POST', url,
-                body=post,
-                headers={'Content-type': 'application/x-www-form-urlencoded'},
-            )
-            res = conn.getresponse()
-
-            http_status = res.status
-            http_reason_phrase = unicode(res.reason, 'utf-8')
-            http_headers = res.msg.dict
-            http_body = res.read()
-            conn.close()
-        except NSPRError, e:
-            raise NetworkError(uri=uri, error=str(e))
-
-        root_logger.debug('request status %d',        http_status)
-        root_logger.debug('request reason_phrase %r', http_reason_phrase)
-        root_logger.debug('request headers %s',       http_headers)
-        root_logger.debug('request body %r',          http_body)
-
-        return http_status, http_reason_phrase, http_headers, http_body
+    return http_status, http_reason_phrase, http_headers, http_body
diff --git a/ipapython/platform/fedora16.py b/ipapython/platform/fedora16.py
index 794c39e20..005d44d08 100644
--- a/ipapython/platform/fedora16.py
+++ b/ipapython/platform/fedora16.py
@@ -17,9 +17,13 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
-from ipapython import ipautil
-from ipapython.platform import base, redhat, systemd
 import os
+import time
+
+from ipapython import ipautil, dogtag
+from ipapython.platform import base, redhat, systemd
+from ipapython.ipa_log_manager import root_logger
+from ipalib import api
 
 # All what we allow exporting directly from this module
 # Everything else is made available through these symbols when they are
@@ -128,6 +132,47 @@ class Fedora16SSHService(Fedora16Service):
     def get_config_dir(self, instance_name=""):
         return '/etc/ssh'
 
+
+class Fedora16CAService(Fedora16Service):
+    def __wait_until_running(self):
+        # We must not wait for the httpd proxy if httpd is not set up yet.
+        # Unfortunately, knownservices.httpd.is_installed() can return
+        # false positives, so check for existence of our configuration file.
+        # TODO: Use a cleaner solution
+        if not os.path.exists('/etc/httpd/conf.d/ipa.conf'):
+            root_logger.debug(
+                'The httpd proxy is not installed, skipping wait for CA')
+            return
+        if dogtag.install_constants.DOGTAG_VERSION < 10:
+            # The server status information isn't available on DT 9
+            root_logger.debug('Using Dogtag 9, skipping wait for CA')
+            return
+        root_logger.debug('Waiting until the CA is running')
+        timeout = api.env.startup_timeout
+        op_timeout = time.time() + timeout
+        while time.time() < op_timeout:
+            status = dogtag.ca_status()
+            root_logger.debug('The CA status is: %s' % status)
+            if status == 'running':
+                break
+            root_logger.debug('Waiting for CA to start...')
+            time.sleep(1)
+        else:
+            raise RuntimeError('CA did not start in %ss' % timeout)
+
+    def start(self, instance_name="", capture_output=True, wait=True):
+        super(Fedora16CAService, self).start(
+            instance_name, capture_output=capture_output, wait=wait)
+        if wait:
+            self.__wait_until_running()
+
+    def restart(self, instance_name="", capture_output=True, wait=True):
+        super(Fedora16CAService, self).restart(
+            instance_name, capture_output=capture_output, wait=wait)
+        if wait:
+            self.__wait_until_running()
+
+
 # Redirect directory server service through special sub-class due to its
 # special handling of instances
 def f16_service(name):
@@ -137,6 +182,8 @@ def f16_service(name):
         return Fedora16IPAService(name)
     if name == 'sshd':
         return Fedora16SSHService(name)
+    if name in ('pki-cad', 'pki_cad', 'pki-tomcatd', 'pki_tomcatd'):
+        return Fedora16CAService(name)
     return Fedora16Service(name)
 
 class Fedora16Services(base.KnownServices):
diff --git a/ipaserver/install/cainstance.py b/ipaserver/install/cainstance.py
index 5a23e35d1..1f950b990 100644
--- a/ipaserver/install/cainstance.py
+++ b/ipaserver/install/cainstance.py
@@ -41,6 +41,7 @@ from ipapython import certmonger
 from ipalib import pkcs10, x509
 from ipapython.dn import DN
 import subprocess
+import traceback
 
 from nss.error import NSPRError
 import nss.nss as nss
@@ -395,6 +396,7 @@ class CADSInstance(service.Service):
                 sys.exit(1)
         except Exception:
             # TODO: roll back here?
+            root_logger.debug(traceback.format_exc())
             root_logger.critical("Failed to restart the directory server. See the installation log for details.")
 
     def uninstall(self):
@@ -867,6 +869,7 @@ class CAInstance(service.Service):
             self.restart(self.dogtag_constants.PKI_INSTANCE_NAME)
         except Exception:
             # TODO: roll back here?
+            root_logger.debug(traceback.format_exc())
             root_logger.critical("Failed to restart the certificate server. See the installation log for details.")
 
     def __disable_nonce(self):
@@ -1551,6 +1554,11 @@ def install_replica_ca(config, postinstall=False):
                           master_host=config.master_host_name,
                           subject_base=config.subject_base)
 
+    if postinstall:
+        # Restart httpd since we changed its config
+        ipaservices.knownservices.httpd.restart()
+
+
     # The dogtag DS instance needs to be restarted after installation.
     # The procedure for this is: stop dogtag, stop DS, start DS, start
     # dogtag