Changes to use a single database for dogtag and IPA

New servers that are installed with dogtag 10 instances will use
a single database instance for dogtag and IPA, albeit with different
suffixes.  Dogtag will communicate with the instance through a
database user with permissions to modify the dogtag  suffix only.
This user will authenticate using client auth using the subsystem cert
for the instance.

This patch includes changes to allow the creation of masters and clones
with single ds instances.

7 files changed, 114 insertions, 66 deletions

diff --git a/install/tools/ipa-ca-install b/install/tools/ipa-ca-install
index 1c1b96a91..05dce8ae5 100755
--- a/install/tools/ipa-ca-install
+++ b/install/tools/ipa-ca-install
@@ -142,17 +142,32 @@ def main():
     config.dir = dir
     config.setup_ca = True
 
+    portfile = config.dir + "/dogtag_directory_port.txt"
+    if not ipautil.file_exists(portfile):
+        dogtag_master_ds_port = str(dogtag.Dogtag9Constants.DS_PORT)
+    else:
+        with open(portfile) as fd:
+            dogtag_master_ds_port = fd.read()
+
     if not options.skip_conncheck:
-        replica_conn_check(config.master_host_name, config.host_name, config.realm_name, True, options.admin_password)
+        replica_conn_check(
+            config.master_host_name, config.host_name, config.realm_name, True,
+            dogtag_master_ds_port, options.admin_password)
 
     # Configure the CA if necessary
-    (CA, cs) = cainstance.install_replica_ca(config, postinstall=True)
+    (CA, cs) = cainstance.install_replica_ca(
+        config, dogtag_master_ds_port, postinstall=True)
 
     # We need to ldap_enable the CA now that DS is up and running
     CA.ldap_enable('CA', config.host_name, config.dirman_password,
                    ipautil.realm_to_suffix(config.realm_name))
-    cs.add_simple_service('dogtagldap/%s@%s' % (config.host_name, config.realm_name))
-    cs.add_cert_to_service()
+    if not dogtag.install_constants.SHARED_DB:
+        cs.add_simple_service('dogtagldap/%s@%s' %
+            (config.host_name, config.realm_name))
+        cs.add_cert_to_service()
+    else:
+        CA.enable_client_auth_to_db()
+        CA.restart()
 
     # We need to restart apache as we drop a new config file in there
     ipaservices.knownservices.httpd.restart(capture_output=True)
diff --git a/install/tools/ipa-csreplica-manage b/install/tools/ipa-csreplica-manage
index 5fce48a37..f2924993f 100755
--- a/install/tools/ipa-csreplica-manage
+++ b/install/tools/ipa-csreplica-manage
@@ -34,7 +34,7 @@ from ipalib import api, errors, util
 from ipapython.dn import DN
 
 CACERT = "/etc/ipa/ca.crt"
-PORT = 7389
+PORT = dogtag.install_constants.DS_PORT
 
 # dict of command name and tuples of min/max num of args needed
 commands = {
diff --git a/install/tools/ipa-replica-conncheck b/install/tools/ipa-replica-conncheck
index 169e9dc9f..29c43f60b 100755
--- a/install/tools/ipa-replica-conncheck
+++ b/install/tools/ipa-replica-conncheck
@@ -21,6 +21,7 @@
 from ipapython.config import IPAOptionParser
 from ipapython import version
 from ipapython import ipautil
+from ipapython import dogtag
 from ipapython.ipautil import CalledProcessError
 from ipaserver.install import installutils
 import ipaclient.ipachangeconf
@@ -59,9 +60,6 @@ BASE_PORTS = [
                 CheckedPort(443, SOCK_STREAM, "HTTP Server: Secure port"),
              ]
 
-CA_PORTS  = [
-                CheckedPort(7389, SOCK_STREAM, "PKI-CA: Directory Service port"),
-            ]
 
 def print_info(msg):
     if not QUIET:
@@ -97,7 +95,8 @@ def parse_options():
     common_group.add_option("-c", "--check-ca", dest="check_ca",
                       action="store_true",
                       default=False,
-                      help="Check also ports for Certificate Authority")
+                      help="Check also ports for Certificate Authority "
+                        "(for servers installed before IPA 3.1)")
 
     common_group.add_option("", "--hostname", dest="hostname",
                       help="The hostname of this server (FQDN). "
@@ -281,7 +280,10 @@ def main():
 
     required_ports = BASE_PORTS
     if options.check_ca:
-        required_ports.extend(CA_PORTS)
+        # Check old Dogtag CA replication port
+        # New installs with unified databases use main DS port (checked above)
+        required_ports.append(CheckedPort(dogtag.Dogtag9Constants.DS_PORT,
+            SOCK_STREAM, "PKI-CA: Directory Service port"))
 
     if options.replica:
         print_info("Check connection from master to remote replica '%s':" % options.replica)
@@ -315,6 +317,8 @@ def main():
             responder.start()
             RESPONDERS.append(responder)
 
+        remote_check_opts = ['--replica %s' % options.hostname]
+
         if options.auto_master_check:
             (krb_fd, krb_name) = tempfile.mkstemp()
             os.close(krb_fd)
@@ -355,10 +359,6 @@ def main():
             if returncode != 0:
                 raise RuntimeError("Could not get ticket for master server: %s" % stderr)
 
-            remote_check_opts = ['--replica %s' % options.hostname]
-            if options.check_ca:
-                remote_check_opts.append('--check-ca')
-
             print_info("Execute check on remote master")
 
             stderr = ''
@@ -380,9 +380,6 @@ def main():
             print_info("")
             print_info("Please run the following command on remote master:")
 
-            remote_check_opts = ['--replica %s' % options.hostname]
-            if options.check_ca:
-                remote_check_opts.append('--check-ca')
             print_info("/usr/sbin/ipa-replica-conncheck " + " ".join(remote_check_opts))
             time.sleep(3600)
             print_info("Connection check timeout: terminating listening program")
diff --git a/install/tools/ipa-replica-install b/install/tools/ipa-replica-install
index e39698914..b7870fbe8 100755
--- a/install/tools/ipa-replica-install
+++ b/install/tools/ipa-replica-install
@@ -479,9 +479,19 @@ def main():
 
     installutils.verify_fqdn(config.master_host_name, options.no_host_dns)
 
+    portfile = config.dir + "/dogtag_directory_port.txt"
+    if not ipautil.file_exists(portfile):
+        dogtag_master_ds_port = str(dogtag.Dogtag9Constants.DS_PORT)
+    else:
+        with open(portfile) as fd:
+            dogtag_master_ds_port = fd.read()
+
     # check connection
     if not options.skip_conncheck:
-        replica_conn_check(config.master_host_name, config.host_name, config.realm_name, options.setup_ca, options.admin_password)
+        replica_conn_check(
+            config.master_host_name, config.host_name, config.realm_name,
+            options.setup_ca, dogtag_master_ds_port, options.admin_password)
+
 
     # check replica host IP resolution
     config.ip = installutils.get_server_ip_address(config.host_name, fstore, True, options)
@@ -597,21 +607,26 @@ def main():
         ntp = ntpinstance.NTPInstance()
         ntp.create_instance()
 
+    # Configure dirsrv
+    ds = install_replica_ds(config)
+
     # Configure the CA if necessary
-    (CA, cs) = cainstance.install_replica_ca(config)
+    (CA, cs) = cainstance.install_replica_ca(config, dogtag_master_ds_port)
 
     # Always try to install DNS records
     install_dns_records(config, options)
 
-    # Configure dirsrv
-    ds = install_replica_ds(config)
-
     # We need to ldap_enable the CA now that DS is up and running
     if CA and config.setup_ca:
         CA.ldap_enable('CA', config.host_name, config.dirman_password,
                        ipautil.realm_to_suffix(config.realm_name))
-        cs.add_simple_service('dogtagldap/%s@%s' % (config.host_name, config.realm_name))
-        cs.add_cert_to_service()
+        if not dogtag.install_constants.SHARED_DB:
+            cs.add_simple_service('dogtagldap/%s@%s' %
+                (config.host_name, config.realm_name))
+            cs.add_cert_to_service()
+        else:
+            CA.enable_client_auth_to_db()
+            CA.restart()
 
     krb = install_krb(config, setup_pkinit=options.setup_pkinit)
     http = install_http(config, auto_redirect=options.ui_redirect)
diff --git a/install/tools/ipa-replica-prepare b/install/tools/ipa-replica-prepare
index d67ed818e..f9d0eb646 100755
--- a/install/tools/ipa-replica-prepare
+++ b/install/tools/ipa-replica-prepare
@@ -209,8 +209,8 @@ def save_config(dir, realm_name, host_name,
     config.set("realm", "destination_host", dest_host)
     config.set("realm", "subject_base", str(subject_base))
     config.set("realm", "version", str(version.NUM_VERSION))
-    fd = open(dir + "/realm_info", "w")
-    config.write(fd)
+    with open(dir + "/realm_info", "w") as fd:
+        config.write(fd)
 
 def remove_file(fname, ignore_errors=True):
     try:
@@ -371,6 +371,10 @@ def main():
         except errors.CertificateOperationError, e:
             print "%s" % e
             sys.exit(1)
+        print "Saving dogtag Directory Server port"
+        port_fname = dir + "/dogtag_directory_port.txt"
+        with open(port_fname, "w") as fd:
+            fd.write("%s\n" % str(dogtag.configured_constants().DS_PORT))
 
     if options.http_pin:
         passwd = options.http_pin
diff --git a/install/tools/ipa-server-install b/install/tools/ipa-server-install
index 70e5153d7..3e5d9a9d4 100755
--- a/install/tools/ipa-server-install
+++ b/install/tools/ipa-server-install
@@ -482,9 +482,11 @@ def uninstall():
         print "ipa-client-install returned: " + str(e)
 
     ntpinstance.NTPInstance(fstore).uninstall()
-    cads_instance = cainstance.CADSInstance(dogtag_constants=dogtag_constants)
-    if cads_instance.is_configured():
-        cads_instance.uninstall()
+    if not dogtag.install_constants.SHARED_DB:
+        cads_instance = cainstance.CADSInstance(
+            dogtag_constants=dogtag_constants)
+        if cads_instance.is_configured():
+            cads_instance.uninstall()
     ca_instance = cainstance.CAInstance(
         api.env.realm, certs.NSS_DIR, dogtag_constants=dogtag_constants)
     if ca_instance.is_configured():
@@ -626,11 +628,11 @@ def main():
         return uninstall()
 
     if options.external_ca:
-        if cainstance.CADSInstance().is_configured():
+        if cainstance.is_step_one_done():
             print "CA is already installed.\nRun the installer with --external_cert_file and --external_ca_file."
             sys.exit(1)
     elif options.external_cert_file:
-        if not cainstance.CADSInstance().is_configured():
+        if not cainstance.is_step_one_done():
             # This can happen if someone passes external_ca_file without
             # already having done the first stage of the CA install.
             print "CA is not installed yet. To install with an external CA is a two-stage process.\nFirst run the installer with --external-ca."
@@ -913,6 +915,30 @@ def main():
         if not ntp.is_configured():
             ntp.create_instance()
 
+    # Create a directory server instance
+    ds = dsinstance.DsInstance(fstore=fstore)
+
+    if options.dirsrv_pin:
+        [pw_fd, pw_name] = tempfile.mkstemp()
+        os.write(pw_fd, options.dirsrv_pin)
+        os.close(pw_fd)
+
+    if options.dirsrv_pkcs12:
+        pkcs12_info = (options.dirsrv_pkcs12, pw_name)
+        try:
+            ds.create_instance(realm_name, host_name, domain_name,
+                               dm_password, pkcs12_info,
+                               subject_base=options.subject,
+                               hbac_allow=not options.hbac_allow)
+        finally:
+            os.remove(pw_name)
+    else:
+        ds.create_instance(realm_name, host_name, domain_name,
+                           dm_password, self_signed_ca=options.selfsign,
+                           idstart=options.idstart, idmax=options.idmax,
+                           subject_base=options.subject,
+                           hbac_allow=not options.hbac_allow)
+
     if options.selfsign:
         ca = certs.CertDB(realm_name, host_name=host_name,
                           subject_base=options.subject)
@@ -933,9 +959,13 @@ def main():
         else:
             external = 0
 
-        cs = cainstance.CADSInstance(host_name, realm_name, domain_name, dm_password)
-        if not cs.is_configured():
-            cs.create_instance(realm_name, host_name, domain_name, dm_password, subject_base=options.subject)
+        if not dogtag.install_constants.SHARED_DB:
+            cs = cainstance.CADSInstance(
+                host_name, realm_name, domain_name, dm_password)
+            if not cs.is_configured():
+                cs.create_instance(realm_name, host_name, domain_name,
+                    dm_password, subject_base=options.subject)
+
         ca = cainstance.CAInstance(realm_name, certs.NSS_DIR,
             dogtag_constants=dogtag.install_constants)
         if external == 0:
@@ -966,42 +996,25 @@ def main():
     # Now put the CA cert where other instances exepct it
     ca.publish_ca_cert("/etc/ipa/ca.crt")
 
-    # Create a directory server instance
-    ds = dsinstance.DsInstance(fstore=fstore)
-
-    if options.dirsrv_pin:
-        [pw_fd, pw_name] = tempfile.mkstemp()
-        os.write(pw_fd, options.dirsrv_pin)
-        os.close(pw_fd)
-
-    if options.dirsrv_pkcs12:
-        pkcs12_info = (options.dirsrv_pkcs12, pw_name)
-        try:
-            ds.create_instance(realm_name, host_name, domain_name,
-                               dm_password, pkcs12_info,
-                               subject_base=options.subject,
-                               hbac_allow=not options.hbac_allow)
-        finally:
-            os.remove(pw_name)
-    else:
-        ds.create_instance(realm_name, host_name, domain_name,
-                           dm_password, self_signed_ca=options.selfsign,
-                           idstart=options.idstart, idmax=options.idmax,
-                           subject_base=options.subject,
-                           hbac_allow=not options.hbac_allow)
+    # we now need to enable ssl on the ds
+    ds.enable_ssl()
+    ds.restart()
 
     # We need to ldap_enable the CA now that DS is up and running
     if not options.selfsign:
         ca.ldap_enable('CA', host_name, dm_password,
                        ipautil.realm_to_suffix(realm_name))
-
-        # Turn on SSL in the dogtag LDAP instance. This will get restarted
-        # later, we don't need SSL now.
-        cs.create_certdb()
-        cs.enable_ssl()
-        # Add the IPA service for storing the PKI-IPA server certificate.
-        cs.add_simple_service(cs.principal)
-        cs.add_cert_to_service()
+        if not dogtag.install_constants.SHARED_DB:
+            # Turn on SSL in the dogtag LDAP instance. This will get restarted
+            # later, we don't need SSL now.
+            cs.create_certdb()
+            cs.enable_ssl()
+            # Add the IPA service for storing the PKI-IPA server certificate.
+            cs.add_simple_service(cs.principal)
+            cs.add_cert_to_service()
+        else:
+            ca.enable_client_auth_to_db()
+            ca.restart()
 
     # Create a kerberos instance
     if options.pkinit_pin:
diff --git a/install/tools/ipactl b/install/tools/ipactl
index f931a2725..e8f3f6384 100755
--- a/install/tools/ipactl
+++ b/install/tools/ipactl
@@ -29,6 +29,7 @@ try:
     from ipalib import api, errors
     from ipapython import sysrestore
     from ipapython import config
+    from ipapython import dogtag
     from ipapython.dn import DN
     import ldap
     import ldap.sasl
@@ -64,7 +65,10 @@ def is_dirsrv_debugging_enabled():
     """
     debugging = False
     serverid = realm_to_serverid(api.env.realm)
-    for dse in ['/etc/dirsrv/slapd-PKI-IPA/', config_dirname(serverid)]:
+    dselist = [config_dirname(serverid)]
+    if not dogtag.install_constants.SHARED_DB:
+        dselist.append('/etc/dirsrv/slapd-PKI-IPA/')
+    for dse in dselist:
         try:
             fd = open(dse + 'dse.ldif', 'r')
         except IOError: