Handle import and export of external certs

Ticket 1742 has a case where a third party CA certificate has
been added by IPA to the dogtag certdb for the proxy cert.
There is no way to ensure that this certificate is imported
when the system is cloned.

This patch will allow the user to import third party certificates
into a dogtag instance through CLI commands (pki-server).
The certs are tracked by a new instance level configuration file
external_certs.conf.

Then, when cloning:

1.  When the pk12 file is created by the pki-server ca-clone-prepare
    command, the external certs are automatically included.
2.  When creating the clone, the new pki_server_pk12_path and
    password must be provided.  Also, a copy of the
    external_certs.conf file must be provided.
3.  This copy will be read and merged with the existing
    external_certs.conf if one exists.

10 files changed, 363 insertions, 25 deletions

diff --git a/base/common/python/pki/nssdb.py b/base/common/python/pki/nssdb.py
index a428e397a..a6b2fa30f 100644
--- a/base/common/python/pki/nssdb.py
+++ b/base/common/python/pki/nssdb.py
@@ -395,7 +395,8 @@ class NSSDatabase(object):
 
         subprocess.check_call(cmd)
 
-    def import_cert_chain(self, nickname, cert_chain_file, trust_attributes=None):
+    def import_cert_chain(self, nickname, cert_chain_file,
+                          trust_attributes=None):
 
         tmpdir = tempfile.mkdtemp()
 
@@ -407,16 +408,18 @@ class NSSDatabase(object):
                     nickname=nickname,
                     cert_file=cert_chain_file,
                     trust_attributes=trust_attributes)
-                return self.get_cert(
-                    nickname=nickname,
-                    output_format='base64')
+                return (
+                    self.get_cert(nickname=nickname, output_format='base64'),
+                    [nickname]
+                )
 
-            elif file_type == 'pkcs7': # import PKCS #7 cert chain
-                return self.import_pkcs7(
+            elif file_type == 'pkcs7':  # import PKCS #7 cert chain
+                chain, nicks = self.import_pkcs7(
                     pkcs7_file=cert_chain_file,
                     nickname=nickname,
                     trust_attributes=trust_attributes,
                     output_format='base64')
+                return chain, nicks
 
             else: # import PKCS #7 data without header/footer
                 with open(cert_chain_file, 'r') as f:
@@ -427,17 +430,18 @@ class NSSDatabase(object):
                 with open(tmp_cert_chain_file, 'w') as f:
                     f.write(pkcs7_data)
 
-                self.import_pkcs7(
+                chain, nicks = self.import_pkcs7(
                     pkcs7_file=tmp_cert_chain_file,
                     nickname=nickname,
                     trust_attributes=trust_attributes)
 
-                return base64_data
+                return base64_data, nicks
 
         finally:
             shutil.rmtree(tmpdir)
 
-    def import_pkcs7(self, pkcs7_file, nickname, trust_attributes=None, output_format='pem'):
+    def import_pkcs7(self, pkcs7_file, nickname, trust_attributes=None,
+                     output_format='pem'):
 
         tmpdir = tempfile.mkdtemp()
 
@@ -453,6 +457,7 @@ class NSSDatabase(object):
             # parse PEM output into separate PEM certificates
             certs = []
             lines = []
+            nicks = []
             state = 'header'
 
             for line in output.splitlines():
@@ -494,6 +499,7 @@ class NSSDatabase(object):
                     n = '%s #%d' % (nickname, counter)
 
                 self.add_cert(n, cert_file, trust_attributes)
+                nicks.append(n)
 
                 counter += 1
 
@@ -501,12 +507,13 @@ class NSSDatabase(object):
             with open(pkcs7_file, 'r') as f:
                 data = f.read()
 
-            return convert_pkcs7(data, 'pem', output_format)
+            return convert_pkcs7(data, 'pem', output_format), nicks
 
         finally:
             shutil.rmtree(tmpdir)
 
-    def import_pkcs12(self, pkcs12_file, pkcs12_password=None, pkcs12_password_file=None):
+    def import_pkcs12(self, pkcs12_file, pkcs12_password=None,
+                      pkcs12_password_file=None):
 
         tmpdir = tempfile.mkdtemp()
 
diff --git a/base/server/python/pki/server/__init__.py b/base/server/python/pki/server/__init__.py
index 2c735f0ef..43082bb63 100644
--- a/base/server/python/pki/server/__init__.py
+++ b/base/server/python/pki/server/__init__.py
@@ -342,6 +342,13 @@ class PKISubsystem(object):
         return str(self.instance) + '/' + self.name
 
 
+class ExternalCert(object):
+
+    def __init__(self, nickname=None, token=None):
+        self.nickname = nickname
+        self.token = token
+
+
 @functools.total_ordering
 class PKIInstance(object):
 
@@ -357,6 +364,9 @@ class PKIInstance(object):
 
         self.conf_dir = os.path.join(self.base_dir, 'conf')
         self.password_conf = os.path.join(self.conf_dir, 'password.conf')
+        self.external_certs_conf = os.path.join(
+            self.conf_dir, 'external_certs.conf')
+        self.external_certs = []
 
         self.nssdb_dir = os.path.join(self.base_dir, 'alias')
         self.lib_dir = os.path.join(self.base_dir, 'lib')
@@ -440,6 +450,8 @@ class PKIInstance(object):
             value = parts[1]
             self.passwords[name] = value
 
+        self.load_external_certs(self.external_certs_conf)
+
         # load subsystems
         for subsystem_name in os.listdir(self.registry_dir):
             if subsystem_name in SUBSYSTEM_TYPES:
@@ -450,6 +462,30 @@ class PKIInstance(object):
                 subsystem.load()
                 self.subsystems.append(subsystem)
 
+    def load_external_certs(self, conf_file):
+        self.external_certs = PKIInstance.read_external_certs(conf_file)
+
+    @staticmethod
+    def read_external_certs(conf_file):
+        external_certs = []
+        # load external certs data
+        if os.path.exists(conf_file) and os.stat(conf_file).st_size > 0:
+            tmp_certs = {}
+            lines = open(conf_file).read().splitlines()
+            for line in lines:
+                m = re.search('(\\d+)\\.(\\w+)=(.*)', line)
+                if not m:
+                    raise pki.PKIException('Error parsing %s' % conf_file)
+                indx = m.group(1)
+                attr = m.group(2)
+                value = m.group(3)
+                if indx not in tmp_certs:
+                    tmp_certs[indx] = ExternalCert()
+
+                setattr(tmp_certs[indx], attr, value)
+            external_certs = tmp_certs.values()
+        return external_certs
+
     def get_password(self, name):
         if name in self.passwords:
             return self.passwords[name]
@@ -465,6 +501,76 @@ class PKIInstance(object):
             token=token,
             password=self.get_password(token))
 
+    def external_cert_exists(self, nickname, token):
+        for cert in self.external_certs:
+            if cert.nickname == nickname and cert.token == token:
+                return True
+        return False
+
+    def add_external_cert(self, nickname, token):
+        if self.external_cert_exists(nickname, token):
+            return
+        self.external_certs.append(ExternalCert(nickname, token))
+        self.save_external_cert_data()
+
+    def delete_external_cert(self, nickname, token):
+        for cert in self.external_certs:
+            if cert.nickname == nickname and cert.token == token:
+                self.external_certs.remove(cert)
+        self.save_external_cert_data()
+
+    def save_external_cert_data(self):
+        with io.open(self.external_certs_conf, 'wb') as f:
+            indx = 0
+            for cert in self.external_certs:
+                f.write('%s.nickname=%s\n' % (str(indx), cert.nickname))
+                f.write('%s.token=%s\n' % (str(indx), cert.token))
+                indx += 1
+
+    def export_external_certs(self, pkcs12_file, pkcs12_password_file,
+                              new_file=False):
+        for cert in self.external_certs:
+            nickname = cert.nickname
+            token = cert.token
+            if token == 'Internal Key Storage Token':
+                token = 'internal'
+            nssdb_password = self.get_password(token)
+
+            tmpdir = tempfile.mkdtemp()
+
+            try:
+                nssdb_password_file = os.path.join(tmpdir, 'password.txt')
+                with open(nssdb_password_file, 'w') as f:
+                    f.write(nssdb_password)
+
+                # add the certificate, key, and chain
+                cmd = [
+                    'pki',
+                    '-d', self.nssdb_dir,
+                    '-C', nssdb_password_file
+                ]
+
+                if token and token != 'internal':
+                    cmd.extend(['--token', token])
+
+                cmd.extend([
+                    'pkcs12-cert-add',
+                    '--pkcs12', pkcs12_file,
+                    '--pkcs12-password-file', pkcs12_password_file,
+                ])
+
+                if new_file:
+                    cmd.extend(['--new-file'])
+
+                cmd.extend([
+                    nickname
+                ])
+
+                subprocess.check_call(cmd)
+
+            finally:
+                shutil.rmtree(tmpdir)
+
     def get_subsystem(self, name):
         for subsystem in self.subsystems:
             if name == subsystem.name:
diff --git a/base/server/python/pki/server/cli/ca.py b/base/server/python/pki/server/cli/ca.py
index af0d941f5..c6da12143 100644
--- a/base/server/python/pki/server/cli/ca.py
+++ b/base/server/python/pki/server/cli/ca.py
@@ -398,9 +398,13 @@ class CAClonePrepareCLI(pki.cli.CLI):
 
             subsystem.export_system_cert(
                 'subsystem', pkcs12_file, pkcs12_password_file, new_file=True)
-            subsystem.export_system_cert('signing', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('ocsp_signing', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('audit_signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'ocsp_signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'audit_signing', pkcs12_file, pkcs12_password_file)
+            instance.export_external_certs(pkcs12_file, pkcs12_password_file)
 
         finally:
             shutil.rmtree(tmpdir)
diff --git a/base/server/python/pki/server/cli/instance.py b/base/server/python/pki/server/cli/instance.py
index 16a3355c3..3119629e1 100644
--- a/base/server/python/pki/server/cli/instance.py
+++ b/base/server/python/pki/server/cli/instance.py
@@ -26,6 +26,7 @@ import os
 import sys
 
 import pki.cli
+import pki.nssdb
 import pki.server
 import pki.server.cli.nuxwdog
 
@@ -44,6 +45,8 @@ class InstanceCLI(pki.cli.CLI):
         self.add_module(InstanceMigrateCLI())
         self.add_module(InstanceNuxwdogEnableCLI())
         self.add_module(InstanceNuxwdogDisableCLI())
+        self.add_module(InstanceExternalCertAddCLI())
+        self.add_module(InstanceExternalCertDeleteCLI())
 
     @staticmethod
     def print_instance(instance):
@@ -528,3 +531,185 @@ class InstanceNuxwdogDisableCLI(pki.cli.CLI):
         module.disable_nuxwdog(instance)  # pylint: disable=no-member,maybe-no-member
 
         self.print_message('Nuxwdog disabled for instance %s.' % instance_name)
+
+
+class InstanceExternalCertAddCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(InstanceExternalCertAddCLI, self).__init__(
+            'externalcert-add',
+            'Add external certificate or chain to the instance')
+
+    def print_help(self):
+        print('Usage: pki-server instance-externalcert-add [OPTIONS]')
+        print()
+        print('  -i, --instance <instance ID>       Instance ID (default: pki-tomcat).')
+        print('      --cert-file <path>             Input file containing the external certificate or certificate chain.')
+        print('      --trust-args <trust-args>      Trust args (default \",,\").')
+        print('      --nickname <nickname>          Nickname to be used.')
+        print('      --token <token_name>           Token (default: internal).')
+        print('  -v, --verbose                      Run in verbose mode.')
+        print('      --help                         Show help message.')
+        print()
+
+    def execute(self, argv):
+        try:
+            opts, _ = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'cert-file=', 'trust-args=', 'nickname=','token=',
+                'verbose', 'help'])
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        cert_file = None
+        trust_args = '\",,\"'
+        nickname = None
+        token = 'internal'
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o == '--cert-file':
+                cert_file = a
+
+            elif o == '--trust-args':
+                trust_args = a
+
+            elif o == '--nickname':
+                nickname = a
+
+            elif o == '--token':
+                token = a
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        if not cert_file:
+            print('ERROR: missing input file containing certificate')
+            self.print_help()
+            sys.exit(1)
+
+        if not nickname:
+            print('ERROR: missing nickname')
+            self.print_help()
+            sys.exit(1)
+
+        instance = pki.server.PKIInstance(instance_name)
+        instance.load()
+
+        if instance.external_cert_exists(nickname, token):
+            print('ERROR: Certificate already imported for instance %s.' %
+                  instance_name)
+            sys.exit(1)
+
+        nicks = self.import_certs(
+            instance, cert_file, nickname, token, trust_args)
+        self.update_instance_config(instance, nicks, token)
+
+        self.print_message('Certificate imported for instance %s.' %
+                           instance_name)
+
+    def import_certs(self, instance, cert_file, nickname, token, trust_args):
+        password = instance.get_password(token)
+        certdb = pki.nssdb.NSSDatabase(
+            directory=instance.nssdb_dir,
+            password=password,
+            token=token)
+        _chain, nicks = certdb.import_cert_chain(
+            nickname, cert_file, trust_attributes=trust_args)
+        return nicks
+
+    def update_instance_config(self, instance, nicks, token):
+        for nickname in nicks:
+            instance.add_external_cert(nickname, token)
+
+
+class InstanceExternalCertDeleteCLI(pki.cli.CLI):
+
+    def __init__(self):
+        super(InstanceExternalCertDeleteCLI, self).__init__(
+            'externalcert-del',
+            'Delete external certificate from the instance')
+
+    def print_help(self):
+        print('Usage: pki-server instance-externalcert-del [OPTIONS]')
+        print()
+        print('  -i, --instance <instance ID>       Instance ID (default: pki-tomcat).')
+        print('      --nickname <nickname>          Nickname to be used.')
+        print('      --token <token_name>           Token (default: internal).')
+        print('  -v, --verbose                      Run in verbose mode.')
+        print('      --help                         Show help message.')
+        print()
+
+    def execute(self, argv):
+        try:
+            opts, _ = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=', 'nickname=','token=',
+                'verbose', 'help'])
+
+        except getopt.GetoptError as e:
+            print('ERROR: ' + str(e))
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        nickname = None
+        token = 'internal'
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o == '--nickname':
+                nickname = a
+
+            elif o == '--token':
+                token = a
+
+            elif o in ('-v', '--verbose'):
+                self.set_verbose(True)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                print('ERROR: unknown option ' + o)
+                self.print_help()
+                sys.exit(1)
+
+        if not nickname:
+            print('ERROR: missing nickname')
+            self.print_help()
+            sys.exit(1)
+
+        instance = pki.server.PKIInstance(instance_name)
+        instance.load()
+
+        self.remove_cert(instance, nickname, token)
+        instance.delete_external_cert(nickname, token)
+
+        self.print_message('Certificate removed from instance %s.' %
+                           instance_name)
+
+    def remove_cert(self, instance, nickname, token):
+        password = instance.get_password(token)
+        certdb = pki.nssdb.NSSDatabase(
+            directory=instance.nssdb_dir,
+            password=password,
+            token=token)
+        certdb.remove_cert(nickname)
diff --git a/base/server/python/pki/server/cli/kra.py b/base/server/python/pki/server/cli/kra.py
index d1b27dbc1..ba1bf5a97 100644
--- a/base/server/python/pki/server/cli/kra.py
+++ b/base/server/python/pki/server/cli/kra.py
@@ -131,9 +131,13 @@ class KRAClonePrepareCLI(pki.cli.CLI):
 
             subsystem.export_system_cert(
                 'subsystem', pkcs12_file, pkcs12_password_file, new_file=True)
-            subsystem.export_system_cert('transport', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('storage', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('audit_signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'transport', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'storage', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'audit_signing', pkcs12_file, pkcs12_password_file)
+            instance.export_external_certs(pkcs12_file, pkcs12_password_file)
 
         finally:
             shutil.rmtree(tmpdir)
diff --git a/base/server/python/pki/server/cli/ocsp.py b/base/server/python/pki/server/cli/ocsp.py
index 7b1b43487..45d7fca83 100644
--- a/base/server/python/pki/server/cli/ocsp.py
+++ b/base/server/python/pki/server/cli/ocsp.py
@@ -131,8 +131,11 @@ class OCSPClonePrepareCLI(pki.cli.CLI):
 
             subsystem.export_system_cert(
                 'subsystem', pkcs12_file, pkcs12_password_file, new_file=True)
-            subsystem.export_system_cert('signing', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('audit_signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'audit_signing', pkcs12_file, pkcs12_password_file)
+            instance.export_external_certs(pkcs12_file, pkcs12_password_file)
 
         finally:
             shutil.rmtree(tmpdir)
diff --git a/base/server/python/pki/server/cli/tks.py b/base/server/python/pki/server/cli/tks.py
index 39343db98..2bdfce84a 100644
--- a/base/server/python/pki/server/cli/tks.py
+++ b/base/server/python/pki/server/cli/tks.py
@@ -131,8 +131,11 @@ class TKSClonePrepareCLI(pki.cli.CLI):
 
             subsystem.export_system_cert(
                 'subsystem', pkcs12_file, pkcs12_password_file, new_file=True)
-            subsystem.export_system_cert('signing', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('audit_signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'audit_signing', pkcs12_file, pkcs12_password_file)
+            instance.export_external_certs(pkcs12_file, pkcs12_password_file)
 
         finally:
             shutil.rmtree(tmpdir)
diff --git a/base/server/python/pki/server/cli/tps.py b/base/server/python/pki/server/cli/tps.py
index 05045cb0d..731b9720c 100644
--- a/base/server/python/pki/server/cli/tps.py
+++ b/base/server/python/pki/server/cli/tps.py
@@ -131,8 +131,11 @@ class TPSClonePrepareCLI(pki.cli.CLI):
 
             subsystem.export_system_cert(
                 'subsystem', pkcs12_file, pkcs12_password_file, new_file=True)
-            subsystem.export_system_cert('signing', pkcs12_file, pkcs12_password_file)
-            subsystem.export_system_cert('audit_signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'signing', pkcs12_file, pkcs12_password_file)
+            subsystem.export_system_cert(
+                'audit_signing', pkcs12_file, pkcs12_password_file)
+            instance.export_external_certs(pkcs12_file, pkcs12_password_file)
 
         finally:
             shutil.rmtree(tmpdir)
diff --git a/base/server/python/pki/server/deployment/scriptlets/configuration.py b/base/server/python/pki/server/deployment/scriptlets/configuration.py
index 278ac644b..fc6877d36 100644
--- a/base/server/python/pki/server/deployment/scriptlets/configuration.py
+++ b/base/server/python/pki/server/deployment/scriptlets/configuration.py
@@ -162,7 +162,7 @@ class PkiScriptlet(pkiscriptlet.AbstractBasePkiScriptlet):
                 external_ca_cert_chain_nickname = deployer.mdict['pki_external_ca_cert_chain_nickname']
                 external_ca_cert_chain_file = deployer.mdict['pki_external_ca_cert_chain_path']
                 if external_ca_cert_chain_file:
-                    cert_chain = nssdb.import_cert_chain(
+                    cert_chain, _nicks = nssdb.import_cert_chain(
                         nickname=external_ca_cert_chain_nickname,
                         cert_chain_file=external_ca_cert_chain_file,
                         trust_attributes='CT,C,C')
diff --git a/base/server/python/pki/server/deployment/scriptlets/security_databases.py b/base/server/python/pki/server/deployment/scriptlets/security_databases.py
index 947c9a740..a2ba8f436 100644
--- a/base/server/python/pki/server/deployment/scriptlets/security_databases.py
+++ b/base/server/python/pki/server/deployment/scriptlets/security_databases.py
@@ -21,7 +21,9 @@
 
 from __future__ import absolute_import
 
+import os
 import pki.nssdb
+import pki.server
 
 # PKI Deployment Imports
 from .. import pkiconfig as config
@@ -90,7 +92,8 @@ class PkiScriptlet(pkiscriptlet.AbstractBasePkiScriptlet):
 
             # importing system certificates
 
-            pki_server_pkcs12_password = deployer.mdict['pki_server_pkcs12_password']
+            pki_server_pkcs12_password = deployer.mdict[
+                'pki_server_pkcs12_password']
             if not pki_server_pkcs12_password:
                 raise Exception('Missing pki_server_pkcs12_password property.')
 
@@ -102,6 +105,11 @@ class PkiScriptlet(pkiscriptlet.AbstractBasePkiScriptlet):
                 pkcs12_file=pki_server_pkcs12_path,
                 pkcs12_password=pki_server_pkcs12_password)
 
+            # update external CA file (if needed)
+            external_cert_path = deployer.mdict['pki_server_external_cert_path']
+            if external_cert_path is not None:
+                self.update_external_cert_conf(external_cert_path, deployer)
+
         if len(deployer.instance.tomcat_instance_subsystems()) < 2:
             # only create a self signed cert for a new instance
             #
@@ -175,6 +183,21 @@ class PkiScriptlet(pkiscriptlet.AbstractBasePkiScriptlet):
         deployer.file.delete(deployer.mdict['pki_shared_pfile'])
         return self.rv
 
+    def update_external_cert_conf(self, external_path, deployer):
+        external_certs = pki.server.PKIInstance.read_external_certs(
+            external_path)
+
+        if len(external_certs) > 0:
+            instance = pki.server.PKIInstance(
+                deployer.mdict['pki_instance_name'])
+            instance.load_external_certs(
+                os.path.join(deployer.mdict['pki_instance_configuration_path'],
+                             'external_certs.conf')
+            )
+
+            for cert in external_certs:
+                instance.add_external_cert(cert.nickname, cert.token)
+
     def destroy(self, deployer):
 
         config.pki_log.info(log.SECURITY_DATABASES_DESTROY_1, __name__,