Log unhandled exceptions in certificate renewal scripts.

https://fedorahosted.org/freeipa/ticket/4093

Reviewed-By: Petr Viktorin <pviktori@redhat.com>

6 files changed, 218 insertions, 176 deletions

diff --git a/install/restart_scripts/renew_ca_cert b/install/restart_scripts/renew_ca_cert
index 09acfc236..2ae869db9 100644
--- a/install/restart_scripts/renew_ca_cert
+++ b/install/restart_scripts/renew_ca_cert
@@ -26,6 +26,7 @@ import tempfile
 import syslog
 import random
 import time
+import traceback
 from ipalib import api
 from ipapython.dn import DN
 from ipalib import errors
@@ -42,80 +43,86 @@ from ipapython import certmonger
 # has renewed a CA subsystem certificate a copy is put into the replicated
 # tree so it can be shared with the other IPA servers.
 
-nickname = sys.argv[1]
+def main():
+    nickname = sys.argv[1]
 
-api.bootstrap(context='restart')
-api.finalize()
+    api.bootstrap(context='restart')
+    api.finalize()
 
-configured_constants = dogtag.configured_constants(api)
-alias_dir = configured_constants.ALIAS_DIR
-dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
-dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+    configured_constants = dogtag.configured_constants(api)
+    alias_dir = configured_constants.ALIAS_DIR
+    dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
+    dogtag_instance = configured_constants.PKI_INSTANCE_NAME
 
-# Fetch the new certificate
-db = certs.CertDB(api.env.realm, nssdir=alias_dir)
-cert = db.get_cert_from_db(nickname, pem=False)
+    # Fetch the new certificate
+    db = certs.CertDB(api.env.realm, nssdir=alias_dir)
+    cert = db.get_cert_from_db(nickname, pem=False)
 
-if not cert:
-    syslog.syslog(syslog.LOG_ERR, 'No certificate %s found.' % nickname)
-    sys.exit(1)
+    if not cert:
+        syslog.syslog(syslog.LOG_ERR, 'No certificate %s found.' % nickname)
+        sys.exit(1)
 
-# Update or add it
-tmpdir = tempfile.mkdtemp(prefix = "tmp-")
-try:
-    dn = DN(('cn',nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-    principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-    ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-    conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-    conn.connect(ccache=ccache)
+    # Update or add it
+    tmpdir = tempfile.mkdtemp(prefix = "tmp-")
     try:
-        entry_attrs = conn.get_entry(dn, ['usercertificate'])
-        entry_attrs['usercertificate'] = cert
-        conn.update_entry(entry_attrs)
-    except errors.NotFound:
-        entry_attrs = conn.make_entry(
-            dn,
-            objectclass=['top', 'pkiuser', 'nscontainer'],
-            usercertificate=[cert])
-        conn.add_entry(entry_attrs)
-    except errors.EmptyModlist:
-        pass
-    conn.disconnect()
-except Exception, e:
-    syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s' % e)
-finally:
-    shutil.rmtree(tmpdir)
+        dn = DN(('cn',nickname), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
+        principal = str('host/%s@%s' % (api.env.host, api.env.realm))
+        ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
+        conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
+        conn.connect(ccache=ccache)
+        try:
+            entry_attrs = conn.get_entry(dn, ['usercertificate'])
+            entry_attrs['usercertificate'] = cert
+            conn.update_entry(entry_attrs)
+        except errors.NotFound:
+            entry_attrs = conn.make_entry(
+                dn,
+                objectclass=['top', 'pkiuser', 'nscontainer'],
+                usercertificate=[cert])
+            conn.add_entry(entry_attrs)
+        except errors.EmptyModlist:
+            pass
+        conn.disconnect()
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s' % e)
+    finally:
+        shutil.rmtree(tmpdir)
 
-# Done withing stopped_service context, CA restarted here
-update_cert_config(nickname, cert)
+    # Done withing stopped_service context, CA restarted here
+    update_cert_config(nickname, cert)
 
-if nickname == 'subsystemCert cert-pki-ca':
-    update_people_entry('pkidbuser', cert)
+    if nickname == 'subsystemCert cert-pki-ca':
+        update_people_entry('pkidbuser', cert)
 
-if nickname == 'auditSigningCert cert-pki-ca':
-    # Fix trust on the audit cert
-    db = certs.CertDB(api.env.realm, nssdir=alias_dir)
-    args = ['-M',
-            '-n', nickname,
-            '-t', 'u,u,Pu',
-           ]
+    if nickname == 'auditSigningCert cert-pki-ca':
+        # Fix trust on the audit cert
+        db = certs.CertDB(api.env.realm, nssdir=alias_dir)
+        args = ['-M',
+                '-n', nickname,
+                '-t', 'u,u,Pu',
+               ]
+        try:
+            db.run_certutil(args)
+            syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir))
+        except ipautil.CalledProcessError:
+             syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir))
+
+    # Now we can start the CA. Using the ipaservices start should fire
+    # off the servlet to verify that the CA is actually up and responding so
+    # when this returns it should be good-to-go. The CA was stopped in the
+    # pre-save state.
+    syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
     try:
-        db.run_certutil(args)
-        syslog.syslog(syslog.LOG_NOTICE, 'Updated trust on certificate %s in %s' % (nickname, db.secdir))
-    except ipautil.CalledProcessError:
-         syslog.syslog(syslog.LOG_ERR, 'Updating trust on certificate %s failed in %s' % (nickname, db.secdir))
+        dogtag_service.start(dogtag_instance)
+    except Exception, e:
+        syslog.syslog(
+            syslog.LOG_ERR,
+            "Cannot start %s: %s" % (dogtag_service.service_name, e))
+    else:
+        syslog.syslog(
+            syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
 
-# Now we can start the CA. Using the ipaservices start should fire
-# off the servlet to verify that the CA is actually up and responding so
-# when this returns it should be good-to-go. The CA was stopped in the
-# pre-save state.
-syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
 try:
-    dogtag_service.start(dogtag_instance)
-except Exception, e:
-    syslog.syslog(
-        syslog.LOG_ERR,
-        "Cannot start %s: %s" % (dogtag_service.service_name, e))
-else:
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
diff --git a/install/restart_scripts/renew_ra_cert b/install/restart_scripts/renew_ra_cert
index cb3e3683b..0d731144b 100644
--- a/install/restart_scripts/renew_ra_cert
+++ b/install/restart_scripts/renew_ra_cert
@@ -24,6 +24,7 @@ import shutil
 import tempfile
 import syslog
 import time
+import traceback
 from ipapython import services as ipaservices
 from ipapython import ipautil
 from ipaserver.install import certs
@@ -33,60 +34,66 @@ from ipapython.dn import DN
 from ipalib import errors
 from ipaserver.plugins.ldap2 import ldap2
 
-api.bootstrap(context='restart')
-api.finalize()
+def main():
+    api.bootstrap(context='restart')
+    api.finalize()
 
-# Fetch the new certificate
-db = certs.CertDB(api.env.realm)
-dercert = db.get_cert_from_db('ipaCert', pem=False)
+    # Fetch the new certificate
+    db = certs.CertDB(api.env.realm)
+    dercert = db.get_cert_from_db('ipaCert', pem=False)
 
-# Load it into dogtag
-update_people_entry('ipara', dercert)
+    # Load it into dogtag
+    update_people_entry('ipara', dercert)
 
-attempts = 0
-updated = False
+    attempts = 0
+    updated = False
 
-# Store it in the IPA LDAP server
-while attempts < 10:
-    conn = None
-    tmpdir = None
-    try:
-        tmpdir = tempfile.mkdtemp(prefix="tmp-")
-        dn = DN(('cn','ipaCert'), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
-        principal = str('host/%s@%s' % (api.env.host, api.env.realm))
-        ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
-        conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
-        conn.connect(ccache=ccache)
+    # Store it in the IPA LDAP server
+    while attempts < 10:
+        conn = None
+        tmpdir = None
         try:
-            entry_attrs = conn.get_entry(dn, ['usercertificate'])
-            entry_attrs['usercertificate'] = dercert
-            conn.update_entry(entry_attrs)
-        except errors.NotFound:
-            entry_attrs = conn.make_entry(
-                dn,
-                objectclass=['top', 'pkiuser', 'nscontainer'],
-                usercertificate=[dercert])
-            conn.add_entry(entry_attrs)
-        except errors.EmptyModlist:
-            pass
-        updated = True
-        break
-    except Exception, e:
-        syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s. Sleeping 30s' % e)
-        time.sleep(30)
-        attempts += 1
-    finally:
-        if conn is not None and conn.isconnected():
-            conn.disconnect()
-        if tmpdir is not None:
-            shutil.rmtree(tmpdir)
+            tmpdir = tempfile.mkdtemp(prefix="tmp-")
+            dn = DN(('cn','ipaCert'), ('cn', 'ca_renewal'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
+            principal = str('host/%s@%s' % (api.env.host, api.env.realm))
+            ccache = ipautil.kinit_hostprincipal('/etc/krb5.keytab', tmpdir, principal)
+            conn = ldap2(shared_instance=False, ldap_uri=api.env.ldap_uri)
+            conn.connect(ccache=ccache)
+            try:
+                entry_attrs = conn.get_entry(dn, ['usercertificate'])
+                entry_attrs['usercertificate'] = dercert
+                conn.update_entry(entry_attrs)
+            except errors.NotFound:
+                entry_attrs = conn.make_entry(
+                    dn,
+                    objectclass=['top', 'pkiuser', 'nscontainer'],
+                    usercertificate=[dercert])
+                conn.add_entry(entry_attrs)
+            except errors.EmptyModlist:
+                pass
+            updated = True
+            break
+        except Exception, e:
+            syslog.syslog(syslog.LOG_ERR, 'Updating renewal certificate failed: %s. Sleeping 30s' % e)
+            time.sleep(30)
+            attempts += 1
+        finally:
+            if conn is not None and conn.isconnected():
+                conn.disconnect()
+            if tmpdir is not None:
+                shutil.rmtree(tmpdir)
+
+    if not updated:
+        syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0])
+        sys.exit(1)
 
-if not updated:
-    syslog.syslog(syslog.LOG_ERR, '%s: Giving up. This script may be safely re-executed.' % sys.argv[0])
-    sys.exit(1)
+    # Now restart Apache so the new certificate is available
+    try:
+        ipaservices.knownservices.httpd.restart()
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
 
-# Now restart Apache so the new certificate is available
 try:
-    ipaservices.knownservices.httpd.restart()
-except Exception, e:
-    syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
diff --git a/install/restart_scripts/restart_dirsrv b/install/restart_scripts/restart_dirsrv
index 9b22d08c1..03e6a5c42 100644
--- a/install/restart_scripts/restart_dirsrv
+++ b/install/restart_scripts/restart_dirsrv
@@ -21,20 +21,27 @@
 
 import sys
 import syslog
+import traceback
 from ipapython import services as ipaservices
 from ipalib import api
 
-try:
-    instance = sys.argv[1]
-except IndexError:
-    instance = ""
+def main():
+    try:
+        instance = sys.argv[1]
+    except IndexError:
+        instance = ""
+
+    api.bootstrap(context='restart')
+    api.finalize()
 
-api.bootstrap(context='restart')
-api.finalize()
+    syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted dirsrv instance '%s'" % instance)
 
-syslog.syslog(syslog.LOG_NOTICE, "certmonger restarted dirsrv instance '%s'" % instance)
+    try:
+        ipaservices.knownservices.dirsrv.restart(instance)
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e)))
 
 try:
-    ipaservices.knownservices.dirsrv.restart(instance)
-except Exception, e:
-    syslog.syslog(syslog.LOG_ERR, "Cannot restart dirsrv (instance: '%s'): %s" % (instance, str(e)))
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
diff --git a/install/restart_scripts/restart_httpd b/install/restart_scripts/restart_httpd
index 16a41ee5b..98337bc85 100644
--- a/install/restart_scripts/restart_httpd
+++ b/install/restart_scripts/restart_httpd
@@ -20,11 +20,18 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import syslog
+import traceback
 from ipapython import services as ipaservices
 
-syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd')
+def main():
+    syslog.syslog(syslog.LOG_NOTICE, 'certmonger restarted httpd')
+
+    try:
+        ipaservices.knownservices.httpd.restart()
+    except Exception, e:
+        syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
 
 try:
-    ipaservices.knownservices.httpd.restart()
-except Exception, e:
-    syslog.syslog(syslog.LOG_ERR, "Cannot restart httpd: %s" % str(e))
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
diff --git a/install/restart_scripts/restart_pkicad b/install/restart_scripts/restart_pkicad
index 9a3d48057..4e14577ae 100644
--- a/install/restart_scripts/restart_pkicad
+++ b/install/restart_scripts/restart_pkicad
@@ -21,54 +21,61 @@
 
 import sys
 import syslog
+import traceback
 from ipapython import services as ipaservices
 from ipapython import dogtag
 from ipaserver.install import certs
 from ipalib import api
 
-nickname = sys.argv[1]
+def main():
+    nickname = sys.argv[1]
 
-api.bootstrap(context='restart')
-api.finalize()
+    api.bootstrap(context='restart')
+    api.finalize()
 
-configured_constants = dogtag.configured_constants(api)
-alias_dir = configured_constants.ALIAS_DIR
-dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
-dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+    configured_constants = dogtag.configured_constants(api)
+    alias_dir = configured_constants.ALIAS_DIR
+    dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
+    dogtag_instance = configured_constants.PKI_INSTANCE_NAME
 
-# dogtag opens its NSS database in read/write mode so we need it
-# shut down so certmonger can open it read/write mode. This avoids
-# database corruption. It should already be stopped by the pre-command
-# but lets be sure.
-if dogtag_service.is_running(dogtag_instance):
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
+    # dogtag opens its NSS database in read/write mode so we need it
+    # shut down so certmonger can open it read/write mode. This avoids
+    # database corruption. It should already be stopped by the pre-command
+    # but lets be sure.
+    if dogtag_service.is_running(dogtag_instance):
+        syslog.syslog(
+            syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
+        try:
+            dogtag_service.stop(dogtag_instance)
+        except Exception, e:
+            syslog.syslog(
+                syslog.LOG_ERR,
+                "Cannot stop %s: %s" % (dogtag_service.service_name, e))
+        else:
+            syslog.syslog(
+                syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
+
+    # Fix permissions on the audit cert if we're updating it
+    if nickname == 'auditSigningCert cert-pki-ca':
+        db = certs.CertDB(api.env.realm, nssdir=alias_dir)
+        args = ['-M',
+                '-n', nickname,
+                '-t', 'u,u,Pu',
+               ]
+        db.run_certutil(args)
+
+    syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
     try:
-        dogtag_service.stop(dogtag_instance)
+        dogtag_service.start(dogtag_instance)
     except Exception, e:
         syslog.syslog(
             syslog.LOG_ERR,
-            "Cannot stop %s: %s" % (dogtag_service.service_name, e))
+            "Cannot start %s: %s" % (dogtag_service.service_name, e))
     else:
         syslog.syslog(
-            syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
-
-# Fix permissions on the audit cert if we're updating it
-if nickname == 'auditSigningCert cert-pki-ca':
-    db = certs.CertDB(api.env.realm, nssdir=alias_dir)
-    args = ['-M',
-            '-n', nickname,
-            '-t', 'u,u,Pu',
-           ]
-    db.run_certutil(args)
+            syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
 
-syslog.syslog(syslog.LOG_NOTICE, 'Starting %s' % dogtag_service.service_name)
 try:
-    dogtag_service.start(dogtag_instance)
-except Exception, e:
-    syslog.syslog(
-        syslog.LOG_ERR,
-        "Cannot start %s: %s" % (dogtag_service.service_name, e))
-else:
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Started %s" % dogtag_service.service_name)
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())
diff --git a/install/restart_scripts/stop_pkicad b/install/restart_scripts/stop_pkicad
index c275eaee4..e58e04401 100644
--- a/install/restart_scripts/stop_pkicad
+++ b/install/restart_scripts/stop_pkicad
@@ -21,23 +21,30 @@
 
 import sys
 import syslog
+import traceback
 from ipapython import services as ipaservices
 from ipapython import dogtag
 from ipalib import api
 
-api.bootstrap(context='restart')
-api.finalize()
+def main():
+    api.bootstrap(context='restart')
+    api.finalize()
 
-configured_constants = dogtag.configured_constants(api)
-dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
-dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+    configured_constants = dogtag.configured_constants(api)
+    dogtag_service = ipaservices.knownservices[configured_constants.SERVICE_NAME]
+    dogtag_instance = configured_constants.PKI_INSTANCE_NAME
+
+    syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
+    try:
+        dogtag_service.stop(dogtag_instance)
+    except Exception, e:
+        syslog.syslog(
+            syslog.LOG_ERR, "Cannot stop %s: %s" % (dogtag_service.service_name, e))
+    else:
+        syslog.syslog(
+            syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
 
-syslog.syslog(syslog.LOG_NOTICE, "Stopping %s" % dogtag_service.service_name)
 try:
-    dogtag_service.stop(dogtag_instance)
-except Exception, e:
-    syslog.syslog(
-        syslog.LOG_ERR, "Cannot stop %s: %s" % (dogtag_service.service_name, e))
-else:
-    syslog.syslog(
-        syslog.LOG_NOTICE, "Stopped %s" % dogtag_service.service_name)
+    main()
+except Exception:
+    syslog.syslog(syslog.LOG_ERR, traceback.format_exc())