From 403b2c9b88a804b2dd4bf80bd2e175e9d7bfcd2e Mon Sep 17 00:00:00 2001
From: William Brown <firstyear@redhat.com>
Date: Mon, 15 Aug 2016 15:23:21 +1000
Subject: [PATCH 2/5] Ticket 48951 - dsadm and dsconf: refactor installer code.

Fix Description:  Move the SetupDs object out of tools to a seperate module in
the class hierarchy.

https://fedorahosted.org/389/ticket/48951

Author: wibrown

Review by: ???
---
 lib389/cli_adm/instance.py  |  51 ++++-
 lib389/instance/__init__.py |   7 +
 lib389/instance/setup.py    | 460 ++++++++++++++++++++++++++++++++++++++++++++
 lib389/tools.py             | 418 ----------------------------------------
 4 files changed, 517 insertions(+), 419 deletions(-)
 create mode 100644 lib389/instance/__init__.py
 create mode 100644 lib389/instance/setup.py

diff --git a/lib389/cli_adm/instance.py b/lib389/cli_adm/instance.py
index a5f6d87..905ac98 100644
--- a/lib389/cli_adm/instance.py
+++ b/lib389/cli_adm/instance.py
@@ -10,6 +10,11 @@ from lib389._constants import *
 from lib389.cli_base import _get_arg
 
 from lib389.tools import DirSrvTools
+from lib389.instance.setup import SetupDs
+from getpass import getpass
+import os
+import time
+import sys
 
 def instance_list(inst, log, args):
     instances = inst.list(all=True)
@@ -42,7 +47,41 @@ def instance_stop(inst, log, args):
     instance_action(inst, log, args, "stop")
 
 def instance_create(inst, log, args):
-    pass
+    if not args.ack:
+        sys.exit(0)
+    else:
+        print("""
+ _________________________________________
+/ This is not what you want! Press ctrl-c \\
+\ now ...                                 /
+ -----------------------------------------
+      \\                   / \\  //\\
+       \\    |\\___/|      /   \\//  \\\\
+            /0  0  \\__  /    //  | \\ \\
+           /     /  \\/_/    //   |  \\  \\
+           @_^_@'/   \\/_   //    |   \\   \\
+           //_^_/     \\/_ //     |    \\    \\
+        ( //) |        \\///      |     \\     \\
+      ( / /) _|_ /   )  //       |      \\     _\\
+    ( // /) '/,_ _ _/  ( ; -.    |    _ _\\.-~        .-~~~^-.
+  (( / / )) ,-{        _      `-.|.-~-.           .~         `.
+ (( // / ))  '/\\      /                 ~-. _ .-~      .-~^-.  \\
+ (( /// ))      `.   {            }                   /      \\  \\
+  (( / ))     .----~-.\\        \\-'                 .~         \\  `. \\^-.
+             ///.----..>        \\             _ -~             `.  ^-`  ^-_
+               ///-._ _ _ _ _ _ _}^ - - - - ~                     ~-- ,.-~
+                                                                  /.-~
+        """)
+    for i in range(1,6):
+        print('%s ...' % (5 - int(i)))
+        time.sleep(1)
+    print('Launching ...')
+    sd = SetupDs(args.verbose, args.dryrun, log)
+    if sd.create_from_inf(args.file):
+        print("Sucessfully created instance")
+    else:
+        print("Failed to create instance")
+        sys.exit(1)
 
 def create_parser(subparsers):
     instance_parser = subparsers.add_parser('instance', help="Manager instances of Directory Server")
@@ -59,3 +98,13 @@ def create_parser(subparsers):
     stop_parser = subcommands.add_parser('stop', help="Stop an instance of Directory Server, if it is currently running")
     stop_parser.add_argument('instance', nargs=1, help="The name of the instance to stop.")
     stop_parser.set_defaults(func=instance_stop)
+
+    create_parser = subcommands.add_parser('create', help="Create an instance of Directory Server. Can be interactive or silent with an inf answer file")
+    create_parser.add_argument('-n', '--dryrun', help="Validate system and configurations only. Do not alter the system.", action='store_true', default=False)
+    create_parser.add_argument('-f', '--file', help="Inf file to use with prepared answers")
+    create_parser.add_argument('--IsolemnlyswearthatIamuptonogood', dest="ack",
+                        help="""You are here likely here by mistake! You want setup-ds.pl!
+By setting this value you acknowledge and take responsibility for the fact this command is UNTESTED and NOT READY. You are ON YOUR OWN!
+""",
+                        action='store_true', default=False)
+    create_parser.set_defaults(func=instance_create)
diff --git a/lib389/instance/__init__.py b/lib389/instance/__init__.py
new file mode 100644
index 0000000..d664b4e
--- /dev/null
+++ b/lib389/instance/__init__.py
@@ -0,0 +1,7 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2016 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
diff --git a/lib389/instance/setup.py b/lib389/instance/setup.py
new file mode 100644
index 0000000..5bb3446
--- /dev/null
+++ b/lib389/instance/setup.py
@@ -0,0 +1,460 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2016 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+
+import os
+import ldap
+import shutil
+import sys
+import pwd
+import grp
+import re
+
+from lib389._constants import *
+from lib389.properties import *
+from lib389.passwd import password_hash, password_generate
+
+# The poc backend api
+from lib389.backend import Backends
+from lib389.utils import (
+    is_a_dn,
+    ensure_bytes,
+    ensure_str,
+    socket_check_open,)
+
+try:
+    # There are too many issues with this on EL7
+    # Out of the box, it's just outright broken ...
+    import six.moves.urllib.request
+    import six.moves.urllib.parse
+    import six.moves.urllib.error
+    import six
+except ImportError:
+    pass
+
+MAJOR, MINOR, _, _, _ = sys.version_info
+
+if MAJOR >= 3:
+    import configparser
+else:
+    import ConfigParser as configparser
+
+
+class SetupDs(object):
+    """
+    Implements the Directory Server installer.
+
+    This maybe subclassed, and a number of well known steps will be called.
+    This allows the inf to be shared, and for other installers to work in lock
+    step with this.
+
+    If you are subclassing you want to derive:
+
+    _validate_config_2(self, config):
+    _prepare(self, extra):
+    _install(self, extra):
+
+    If you are calling this from an INF, you can pass the config in
+    _validate_config, then stash the result into self.extra
+
+    If you have anything you need passed to your install helpers, this can
+    be given in create_from_args(extra) if you are calling as an api.
+
+    If you use create_from_inf, self.extra is passed to create_from_args for
+    you. You only need to over-load the three methods above.
+
+    A logging interface is provided to self.log that you should call.
+    """
+
+    def __init__(self, verbose=False, dryrun=False, log=None):
+        self.verbose = verbose
+        self.extra = None
+        self.dryrun = dryrun
+        # Expose the logger to our children.
+        self.log = log.getChild('SetupDs')
+        if self.verbose:
+            self.log.info('Running setup with verbose')
+
+    # Could be nicer if we did self._get_config_fallback_<type>?
+    def _get_config_fallback(self, config, group, attr, value, boolean=False, num=False):
+        try:
+            if boolean:
+                return config.getboolean(group, attr)
+            elif num:
+                return config.getint(group, attr)
+            else:
+                return config.get(group, attr)
+        except ValueError:
+            return value
+        except configparser.NoOptionError:
+            self.log.info("%s not specified:setting to default - %s" % (attr, value))
+            return value
+
+    def _validate_config_2(self, config):
+        pass
+
+    def _prepare(self, extra):
+        pass
+
+    def _install(self, extra):
+        pass
+
+    def _validate_ds_2_config(self, config):
+        assert config.has_section('slapd')
+        # Extract them in a way that create can understand.
+        general = {}
+        general['config_version'] = config.getint('general', 'config_version')
+        general['full_machine_name'] = config.get('general', 'full_machine_name')
+        general['strict_host_checking'] = self._get_config_fallback(config, 'general', 'strict_host_checking', True, boolean=True)
+        # Change this to detect if SELinux is running
+        general['selinux'] = self._get_config_fallback(config, 'general', 'selinux', False, boolean=True)
+
+        if self.verbose:
+            self.log.info("Configuration general %s" % general)
+
+        # Validate that we are a config_version=2
+        assert general['config_version'] >= 2
+
+        slapd = {}
+        # Can probably set these defaults out of somewhere else ...
+        slapd['instance_name'] = config.get('slapd', 'instance_name')
+        slapd['user'] = self._get_config_fallback(config, 'slapd', 'user', 'dirsrv')
+        slapd['group'] = self._get_config_fallback(config, 'slapd', 'group', 'dirsrv')
+        slapd['root_dn'] = self._get_config_fallback(config, 'slapd', 'root_dn', 'cn=Directory Manager')
+        slapd['root_password'] = config.get('slapd', 'root_password')
+        slapd['prefix'] = self._get_config_fallback(config, 'slapd', 'prefix', '/')
+
+        # How do we default, defaults to the DS version.
+        slapd['defaults'] = self._get_config_fallback(config, 'slapd', 'defaults', None)
+        slapd['port'] = self._get_config_fallback(config, 'slapd', 'port', 389, num=True)
+        slapd['secure_port'] = self._get_config_fallback(config, 'slapd', 'secure_port', 636, num=True)
+
+        # These are all the paths for DS, that are RELATIVE to the prefix
+        # This will need to change to cope with configure scripts from DS!
+        # perhaps these should be read as a set of DEFAULTs from a config file?
+        slapd['bin_dir'] = self._get_config_fallback(config, 'slapd', 'bin_dir', os.path.join(slapd['prefix'], 'bin'))
+        slapd['sysconf_dir'] = self._get_config_fallback(config, 'slapd', 'sysconf_dir', os.path.join(slapd['prefix'], 'etc'))
+        slapd['data_dir'] = self._get_config_fallback(config, 'slapd', 'data_dir', os.path.join(slapd['prefix'], 'share'))
+        slapd['local_state_dir'] = self._get_config_fallback(config, 'slapd', 'local_state_dir', os.path.join(slapd['prefix'], 'var'))
+
+        slapd['lib_dir'] = self._get_config_fallback(config, 'slapd', 'lib_dir', os.path.join(slapd['prefix'], 'usr/lib64/dirsrv'))
+        slapd['cert_dir'] = self._get_config_fallback(config, 'slapd', 'cert_dir', os.path.join(slapd['prefix'], 'etc/dirsrv/slapd-%s' % slapd['instance_name']))
+        slapd['config_dir'] = self._get_config_fallback(config, 'slapd', 'config_dir', os.path.join(slapd['prefix'], 'etc/dirsrv/slapd-%s' % slapd['instance_name']))
+
+        slapd['inst_dir'] = self._get_config_fallback(config, 'slapd', 'inst_dir', os.path.join(slapd['prefix'], 'var/lib/dirsrv/slapd-%s' % slapd['instance_name']))
+        slapd['backup_dir'] = self._get_config_fallback(config, 'slapd', 'backup_dir', os.path.join(slapd['inst_dir'], 'bak'))
+        slapd['db_dir'] = self._get_config_fallback(config, 'slapd', 'db_dir', os.path.join(slapd['inst_dir'], 'db'))
+        slapd['ldif_dir'] = self._get_config_fallback(config, 'slapd', 'ldif_dir', os.path.join(slapd['inst_dir'], 'ldif'))
+
+        slapd['lock_dir'] = self._get_config_fallback(config, 'slapd', 'lock_dir', os.path.join(slapd['prefix'], 'var/lock/dirsrv/slapd-%s' % slapd['instance_name']))
+        slapd['log_dir'] = self._get_config_fallback(config, 'slapd', 'log_dir', os.path.join(slapd['prefix'], 'var/log/dirsrv/slapd-%s' % slapd['instance_name']))
+        slapd['run_dir'] = self._get_config_fallback(config, 'slapd', 'run_dir', os.path.join(slapd['prefix'], 'var/run/dirsrv'))
+        slapd['sbin_dir'] = self._get_config_fallback(config, 'slapd', 'sbin_dir', os.path.join(slapd['prefix'], 'sbin'))
+        slapd['schema_dir'] = self._get_config_fallback(config, 'slapd', 'schema_dir', os.path.join(slapd['prefix'], 'etc/dirsrv/slapd-%s' % slapd['instance_name'], 'schema'))
+        slapd['tmp_dir'] = self._get_config_fallback(config, 'slapd', 'tmp_dir', '/tmp')
+
+        # Need to add all the default filesystem paths.
+
+        if self.verbose:
+            self.log.info("Configuration slapd %s" % slapd)
+
+        backends = []
+        for section in config.sections():
+            if section.startswith('backend-'):
+                be = {}
+                # TODO: Add the other BACKEND_ types
+                be[BACKEND_NAME] = section.replace('backend-', '')
+                be[BACKEND_SUFFIX] = config.get(section, 'suffix')
+                be[BACKEND_SAMPLE_ENTRIES] = config.getboolean(section, 'sample_entries')
+                backends.append(be)
+
+        if self.verbose:
+            self.log.info("Configuration backends %s" % backends)
+
+        return (general, slapd, backends)
+
+    def _validate_ds_config(self, config):
+        # This will move to lib389 later.
+        # Check we have all the sections.
+        # Make sure we have needed keys.
+        assert(config.has_section('general'))
+        assert(config.has_option('general', 'config_version'))
+        assert(config.get('general', 'config_version') >= '2')
+        if config.get('general', 'config_version') == '2':
+            # Call our child api to validate itself from the inf.
+            self._validate_config_2(config)
+            return self._validate_ds_2_config(config)
+        else:
+            self.log.info("Failed to validate configuration version.")
+            assert(False)
+
+    def create_from_inf(self, inf_path):
+        """
+        Will trigger a create from the settings stored in inf_path
+        """
+        # Get the inf file
+        if self.verbose:
+            self.log.info("Using inf from %s" % inf_path)
+        if not os.path.isfile(inf_path):
+            self.log.error("%s is not a valid file path" % inf_path)
+            return False
+        config = None
+        try:
+            config = configparser.SafeConfigParser()
+            config.read([inf_path])
+        except Exception as e:
+            self.log.error("Exception %s occured" % e)
+            return False
+
+        if self.verbose:
+            self.log.info("Configuration %s" % config.sections())
+
+        (general, slapd, backends) = self._validate_ds_config(config)
+
+        # Actually do the setup now.
+        self.create_from_args(general, slapd, backends, self.extra)
+
+        return True
+
+    def _prepare_ds(self, general, slapd, backends):
+        # Validate our arguments.
+        assert(slapd['user'] is not None)
+        # check the user exists
+        assert(pwd.getpwnam(slapd['user']))
+        slapd['user_uid'] = pwd.getpwnam(slapd['user']).pw_uid
+        assert(slapd['group'] is not None)
+        assert(grp.getgrnam(slapd['group']))
+        slapd['group_gid'] = grp.getgrnam(slapd['group']).gr_gid
+        # check this group exists
+        # Check that we are running as this user / group, or that we are root.
+        assert(os.geteuid() == 0 or getpass.getuser() == slapd['user'])
+
+        if self.verbose:
+            self.log.info("PASSED: user / group checking")
+
+        assert(general['full_machine_name'] is not None)
+        assert(general['strict_host_checking'] is not None)
+        if general['strict_host_checking'] is True:
+            # Check it resolves with dns
+            assert(socket.gethostbyname(general['full_machine_name']))
+            if self.verbose:
+                self.log.info("PASSED: Hostname strict checking")
+
+        assert(slapd['prefix'] is not None)
+        assert(os.path.exists(slapd['prefix']))
+        if self.verbose:
+            self.log.info("PASSED: prefix checking")
+
+        # We need to know the prefix before we can do the instance checks
+        assert(slapd['instance_name'] is not None)
+        # Check if the instance exists or not.
+        # Should I move this import? I think this prevents some recursion
+        from lib389 import DirSrv
+        ds = DirSrv(verbose=self.verbose)
+        ds.prefix = slapd['prefix']
+        insts = ds.list(serverid=slapd['instance_name'])
+        assert(len(insts) == 0)
+
+        if self.verbose:
+            self.log.info("PASSED: instance checking")
+
+        assert(slapd['root_dn'] is not None)
+        # Assert this is a valid DN
+        assert(is_a_dn(slapd['root_dn']))
+        assert(slapd['root_password'] is not None)
+        # Check if pre-hashed or not.
+        # !!!!!!!!!!!!!!
+
+        # Right now, the way that rootpw works on ns-slapd works, it force hashes the pw
+        # see https://fedorahosted.org/389/ticket/48859
+        if not re.match('^\{[A-Z0-9]+\}.*$', slapd['root_password']):
+            # We need to hash it. Call pwdhash-bin.
+            # slapd['root_password'] = password_hash(slapd['root_password'], prefix=slapd['prefix'])
+            pass
+        else:
+            pass
+
+        # Create a random string
+        # Hash it.
+        # This will be our temporary rootdn password so that we can do
+        # live mods and setup rather than static ldif manipulations.
+        self._raw_secure_password = password_generate()
+        self._secure_password = password_hash(self._raw_secure_password, prefix=slapd['prefix'])
+
+        if self.verbose:
+            self.log.info("PASSED: root user checking")
+
+        assert(slapd['defaults'] is not None)
+        if self.verbose:
+            self.log.info("PASSED: using config settings %s" % slapd['defaults'])
+
+        assert(slapd['port'] is not None)
+        assert(socket_check_open('::1', slapd['port']) is False)
+        assert(slapd['secure_port'] is not None)
+        assert(socket_check_open('::1', slapd['secure_port']) is False)
+        if self.verbose:
+            self.log.info("PASSED: network avaliability checking")
+
+        # Make assertions of the paths?
+
+        # Make assertions of the backends?
+
+    def create_from_args(self, general, slapd, backends=[], extra=None):
+        """
+        Actually does the setup. this is what you want to call as an api.
+        """
+        # Check we have privs to run
+
+        if self.verbose:
+            self.log.info("READY: preparing installation")
+        self._prepare_ds(general, slapd, backends)
+        # Call our child api to prepare itself.
+        self._prepare(extra)
+
+        if self.verbose:
+            self.log.info("READY: beginning installation")
+
+        if self.dryrun:
+            self.log.info("NOOP: dry run requested")
+        else:
+            # Actually trigger the installation.
+            self._install_ds(general, slapd, backends)
+            # Call the child api to do anything it needs.
+            self._install(extra)
+        self.log.info("FINISH: completed installation")
+
+    def _install_ds(self, general, slapd, backends):
+        """
+        Actually install the Ds from the dicts provided.
+
+        You should never call this directly, as it bypasses assertions.
+        """
+        # register the instance to /etc/sysconfig
+        # We do this first so that we can trick remove-ds.pl if needed.
+        # There may be a way to create this from template like the dse.ldif ...
+        initconfig = ""
+        with open("%s/dirsrv/config/template-initconfig" % slapd['sysconf_dir']) as template_init:
+            for line in template_init.readlines():
+                initconfig += line.replace('{{', '{', 1).replace('}}', '}', 1).replace('-', '_')
+        with open("%s/sysconfig/dirsrv-%s" % (slapd['sysconf_dir'], slapd['instance_name']), 'w') as f:
+            f.write(initconfig.format(
+                SERVER_DIR=slapd['lib_dir'],
+                SERVERBIN_DIR=slapd['sbin_dir'],
+                CONFIG_DIR=slapd['config_dir'],
+                INST_DIR=slapd['inst_dir'],
+                RUN_DIR=slapd['run_dir'],
+                DS_ROOT='',
+                PRODUCT_NAME='slapd',
+            ))
+
+        # Create all the needed paths
+        # we should only need to make bak_dir, cert_dir, config_dir, db_dir, ldif_dir, lock_dir, log_dir, run_dir? schema_dir,
+        for path in ('backup_dir', 'cert_dir', 'config_dir', 'db_dir', 'ldif_dir', 'lock_dir', 'log_dir', 'run_dir'):
+            if self.verbose:
+                self.log.info("ACTION: creating %s" % slapd[path])
+            try:
+                os.makedirs(slapd[path], mode=0o770)
+            except OSError:
+                pass
+            os.chown(slapd[path], slapd['user_uid'], slapd['group_gid'])
+
+        # Copy correct data to the paths.
+        # Copy in the schema
+        #  This is a little fragile, make it better.
+        shutil.copytree(os.path.join(slapd['sysconf_dir'], 'dirsrv/schema'), slapd['schema_dir'])
+        os.chown(slapd['schema_dir'], slapd['user_uid'], slapd['group_gid'])
+
+        # Copy in the collation
+        srcfile = os.path.join(slapd['sysconf_dir'], 'dirsrv/config/slapd-collations.conf')
+        dstfile = os.path.join(slapd['config_dir'], 'slapd-collations.conf')
+        shutil.copy2(srcfile, dstfile)
+        os.chown(slapd['schema_dir'], slapd['user_uid'], slapd['group_gid'])
+
+        # Selinux fixups?
+        # Restorecon of paths?
+        # Bind sockets to our type?
+
+        # Create certdb in sysconfidir
+        if self.verbose:
+            self.log.info("ACTION: Creating certificate database is %s" % slapd['cert_dir'])
+        # nss_create_new_database(slapd['cert_dir'])
+
+        # Create dse.ldif with a temporary root password.
+        # The template is in slapd['data_dir']/dirsrv/data/template-dse.ldif
+        # Variables are done with %KEY%.
+        # You could cheat and read it in, do a replace of % to { and } then use format?
+        if self.verbose:
+            self.log.info("ACTION: Creating dse.ldif")
+        dse = ""
+        with open(os.path.join(slapd['data_dir'], 'dirsrv', 'data', 'template-dse.ldif')) as template_dse:
+            for line in template_dse.readlines():
+                dse += line.replace('%', '{', 1).replace('%', '}', 1)
+
+        with open(os.path.join(slapd['config_dir'], 'dse.ldif'), 'w') as file_dse:
+            file_dse.write(dse.format(
+                schema_dir=slapd['schema_dir'],
+                lock_dir=slapd['lock_dir'],
+                tmp_dir=slapd['tmp_dir'],
+                cert_dir=slapd['cert_dir'],
+                ldif_dir=slapd['ldif_dir'],
+                bak_dir=slapd['backup_dir'],
+                run_dir=slapd['run_dir'],
+                inst_dir="",
+                log_dir=slapd['log_dir'],
+                fqdn=general['full_machine_name'],
+                ds_port=slapd['port'],
+                ds_user=slapd['user'],
+                rootdn=slapd['root_dn'],
+                # ds_passwd=slapd['root_password'],
+                ds_passwd=self._secure_password,  # We set our own password here, so we can connect and mod.
+                ds_suffix='',
+                config_dir=slapd['config_dir'],
+                db_dir=slapd['db_dir'],
+            ))
+
+        # open the connection to the instance.
+
+        # Should I move this import? I think this prevents some recursion
+        from lib389 import DirSrv
+        ds_instance = DirSrv(self.verbose)
+        args = {
+            SER_PORT: slapd['port'],
+            SER_SERVERID_PROP: slapd['instance_name'],
+            SER_ROOT_DN: slapd['root_dn'],
+            SER_ROOT_PW: self._raw_secure_password,
+            SER_DEPLOYED_DIR: slapd['prefix']
+        }
+
+        ds_instance.allocate(args)
+        # Does this work?
+        assert(ds_instance.exists())
+        # Start the server
+        ds_instance.start(timeout=60)
+        ds_instance.open()
+
+        # Create the backends as listed
+        # Load example data if needed.
+        for backend in backends:
+            ds_instance.backends.create(properties=backend)
+
+        # Make changes using the temp root
+        # Change the root password finally
+
+        # Complete.
+        ds_instance.config.set('nsslapd-rootpw',
+                               ensure_str(slapd['root_password']))
+
+    def _remove_ds(self):
+        """
+        The opposite of install: Removes an instance from the system.
+        This takes a backup of all relevant data, and removes the paths.
+        """
+        # This probably actually would need to be able to read the ldif, to
+        # know what to remove ...
+        for path in ('backup_dir', 'cert_dir', 'config_dir', 'db_dir',
+                     'ldif_dir', 'lock_dir', 'log_dir', 'run_dir'):
+            print(path)
diff --git a/lib389/tools.py b/lib389/tools.py
index 2d92bc7..58ab4ca 100644
--- a/lib389/tools.py
+++ b/lib389/tools.py
@@ -120,21 +120,6 @@ def runCmd(cmd, timeout_sec):
     timer.cancel()
     return proc.returncode
 
-# Could be nicer if we did _get_config_fallback_<type>?
-def _get_config_fallback(config, group, attr, value, boolean=False, num=False):
-    try:
-        if boolean:
-            return config.getboolean(group, attr)
-        elif num:
-            return config.getint(group, attr)
-        else:
-            return config.get(group, attr)
-    except ValueError:
-        return value
-    except configparser.NoOptionError:
-        log.info("%s not specified:setting to default - %s" % (attr, value))
-        return value
-
 class DirSrvTools(object):
     """DirSrv mix-in."""
 
@@ -1192,406 +1177,3 @@ class MockDirSrv(object):
             return 'ldaps://%s:%s' % (self.host, self.sslport)
         else:
             return 'ldap://%s:%s' % (self.host, self.port)
-
-
-class SetupDs(object):
-    """
-    Implements the Directory Server installer.
-
-    This maybe subclassed, and a number of well known steps will be called.
-    This allows the inf to be shared, and for other installers to work in lock
-    step with this.
-
-    If you are subclassing you want to derive:
-
-    _validate_config_2(self, config):
-    _prepare(self, extra):
-    _install(self, extra):
-
-    If you are calling this from an INF, you can pass the config in
-    _validate_config, then stash the result into self.extra
-
-    If you have anything you need passed to your install helpers, this can
-    be given in create_from_args(extra) if you are calling as an api.
-
-    If you use create_from_inf, self.extra is passed to create_from_args for
-    you. You only need to over-load the three methods above.
-
-    A logging interface is provided to self.log that you should call.
-    """
-
-    def __init__(self, verbose=False, dryrun=False):
-        self.verbose = verbose
-        self.extra = None
-        self.dryrun = dryrun
-        # Expose the logger to our children.
-        self.log = log
-        if self.verbose:
-            log.info('Running setup with verbose')
-
-    def _validate_config_2(self, config):
-        pass
-
-    def _prepare(self, extra):
-        pass
-
-    def _install(self, extra):
-        pass
-
-    def _validate_ds_2_config(self, config):
-        assert config.has_section('slapd')
-        # Extract them in a way that create can understand.
-        general = {}
-        general['config_version'] = config.getint('general', 'config_version')
-        general['full_machine_name'] = config.get('general', 'full_machine_name')
-        general['strict_host_checking'] = _get_config_fallback(config, 'general', 'strict_host_checking', True, boolean=True)
-        # Change this to detect if SELinux is running
-        general['selinux'] = _get_config_fallback(config, 'general', 'selinux', False, boolean=True)
-
-        if self.verbose:
-            log.info("Configuration general %s" % general)
-
-        # Validate that we are a config_version=2
-        assert general['config_version'] >= 2
-
-        slapd = {}
-        # Can probably set these defaults out of somewhere else ...
-        slapd['instance_name'] = config.get('slapd', 'instance_name')
-        slapd['user'] = _get_config_fallback(config, 'slapd', 'user', 'dirsrv')
-        slapd['group'] = _get_config_fallback(config, 'slapd', 'group', 'dirsrv')
-        slapd['root_dn'] = _get_config_fallback(config, 'slapd', 'root_dn', 'cn=Directory Manager')
-        slapd['root_password'] = config.get('slapd', 'root_password')
-        slapd['prefix'] = _get_config_fallback(config, 'slapd', 'prefix', '/')
-
-        # How do we default, defaults to the DS version.
-        slapd['defaults'] = _get_config_fallback(config, 'slapd', 'defaults', None)
-        slapd['port'] = _get_config_fallback(config, 'slapd', 'port', 389, num=True)
-        slapd['secure_port'] = _get_config_fallback(config, 'slapd', 'secure_port', 636, num=True)
-
-        # These are all the paths for DS, that are RELATIVE to the prefix
-        # This will need to change to cope with configure scripts from DS!
-        # perhaps these should be read as a set of DEFAULTs from a config file?
-        slapd['bin_dir'] = _get_config_fallback(config, 'slapd', 'bin_dir', os.path.join(slapd['prefix'], 'bin'))
-        slapd['sysconf_dir'] = _get_config_fallback(config, 'slapd', 'sysconf_dir', os.path.join(slapd['prefix'], 'etc'))
-        slapd['data_dir'] = _get_config_fallback(config, 'slapd', 'data_dir', os.path.join(slapd['prefix'], 'share'))
-        slapd['local_state_dir'] = _get_config_fallback(config, 'slapd', 'local_state_dir', os.path.join(slapd['prefix'], 'var'))
-
-        slapd['lib_dir'] = _get_config_fallback(config, 'slapd', 'lib_dir', os.path.join(slapd['prefix'], 'usr/lib64/dirsrv'))
-        slapd['cert_dir'] = _get_config_fallback(config, 'slapd', 'cert_dir', os.path.join(slapd['prefix'], 'etc/dirsrv/slapd-%s' % slapd['instance_name']))
-        slapd['config_dir'] = _get_config_fallback(config, 'slapd', 'config_dir', os.path.join(slapd['prefix'], 'etc/dirsrv/slapd-%s' % slapd['instance_name']))
-
-        slapd['inst_dir'] = _get_config_fallback(config, 'slapd', 'inst_dir', os.path.join(slapd['prefix'], 'var/lib/dirsrv/slapd-%s' % slapd['instance_name']))
-        slapd['backup_dir'] = _get_config_fallback(config, 'slapd', 'backup_dir', os.path.join(slapd['inst_dir'], 'bak'))
-        slapd['db_dir'] = _get_config_fallback(config, 'slapd', 'db_dir', os.path.join(slapd['inst_dir'], 'db'))
-        slapd['ldif_dir'] = _get_config_fallback(config, 'slapd', 'ldif_dir', os.path.join(slapd['inst_dir'], 'ldif'))
-
-        slapd['lock_dir'] = _get_config_fallback(config, 'slapd', 'lock_dir', os.path.join(slapd['prefix'], 'var/lock/dirsrv/slapd-%s' % slapd['instance_name']))
-        slapd['log_dir'] = _get_config_fallback(config, 'slapd', 'log_dir', os.path.join(slapd['prefix'], 'var/log/dirsrv/slapd-%s' % slapd['instance_name']))
-        slapd['run_dir'] = _get_config_fallback(config, 'slapd', 'run_dir', os.path.join(slapd['prefix'], 'var/run/dirsrv'))
-        slapd['sbin_dir'] = _get_config_fallback(config, 'slapd', 'sbin_dir', os.path.join(slapd['prefix'], 'sbin'))
-        slapd['schema_dir'] = _get_config_fallback(config, 'slapd', 'schema_dir', os.path.join(slapd['prefix'], 'etc/dirsrv/slapd-%s' % slapd['instance_name'], 'schema'))
-        slapd['tmp_dir'] = _get_config_fallback(config, 'slapd', 'tmp_dir', '/tmp')
-
-        # Need to add all the default filesystem paths.
-
-        if self.verbose:
-            log.info("Configuration slapd %s" % slapd)
-
-        backends = []
-        for section in config.sections():
-            if section.startswith('backend-'):
-                be = {}
-                # TODO: Add the other BACKEND_ types
-                be[BACKEND_NAME] = section.replace('backend-', '')
-                be[BACKEND_SUFFIX] = config.get(section, 'suffix')
-                be[BACKEND_SAMPLE_ENTRIES] = config.getboolean(section, 'sample_entries')
-                backends.append(be)
-
-        if self.verbose:
-            log.info("Configuration backends %s" % backends)
-
-        return (general, slapd, backends)
-
-    def _validate_ds_config(self, config):
-        # This will move to lib389 later.
-        # Check we have all the sections.
-        # Make sure we have needed keys.
-        assert(config.has_section('general'))
-        assert(config.has_option('general', 'config_version'))
-        assert(config.get('general', 'config_version') >= '2')
-        if config.get('general', 'config_version') == '2':
-            # Call our child api to validate itself from the inf.
-            self._validate_config_2(config)
-            return self._validate_ds_2_config(config)
-        else:
-            log.info("Failed to validate configuration version.")
-            assert(False)
-
-    def create_from_inf(self, inf_path):
-        """
-        Will trigger a create from the settings stored in inf_path
-        """
-        # Get the inf file
-        if self.verbose:
-            log.info("Using inf from %s" % inf_path)
-        if not os.path.isfile(inf_path):
-            log.error("%s is not a valid file path" % inf_path)
-            return False
-        config = None
-        try:
-            config = configparser.SafeConfigParser()
-            config.read([inf_path])
-        except Exception as e:
-            log.error("Exception %s occured" % e)
-            return False
-
-        if self.verbose:
-            log.info("Configuration %s" % config.sections())
-
-        (general, slapd, backends) = self._validate_ds_config(config)
-
-        # Actually do the setup now.
-        self.create_from_args(general, slapd, backends, self.extra)
-
-        return True
-
-    def _prepare_ds(self, general, slapd, backends):
-        # Validate our arguments.
-        assert(slapd['user'] is not None)
-        # check the user exists
-        assert(pwd.getpwnam(slapd['user']))
-        slapd['user_uid'] = pwd.getpwnam(slapd['user']).pw_uid
-        assert(slapd['group'] is not None)
-        assert(grp.getgrnam(slapd['group']))
-        slapd['group_gid'] = grp.getgrnam(slapd['group']).gr_gid
-        # check this group exists
-        # Check that we are running as this user / group, or that we are root.
-        assert(os.geteuid() == 0 or getpass.getuser() == slapd['user'])
-
-        if self.verbose:
-            log.info("PASSED: user / group checking")
-
-        assert(general['full_machine_name'] is not None)
-        assert(general['strict_host_checking'] is not None)
-        if general['strict_host_checking'] is True:
-            # Check it resolves with dns
-            assert(socket.gethostbyname(general['full_machine_name']))
-            if self.verbose:
-                log.info("PASSED: Hostname strict checking")
-
-        assert(slapd['prefix'] is not None)
-        assert(os.path.exists(slapd['prefix']))
-        if self.verbose:
-            log.info("PASSED: prefix checking")
-
-        # We need to know the prefix before we can do the instance checks
-        assert(slapd['instance_name'] is not None)
-        # Check if the instance exists or not.
-        # Should I move this import? I think this prevents some recursion
-        from lib389 import DirSrv
-        ds = DirSrv(verbose=self.verbose)
-        ds.prefix = slapd['prefix']
-        insts = ds.list(serverid=slapd['instance_name'])
-        assert(len(insts) == 0)
-
-        if self.verbose:
-            log.info("PASSED: instance checking")
-
-        assert(slapd['root_dn'] is not None)
-        # Assert this is a valid DN
-        assert(is_a_dn(slapd['root_dn']))
-        assert(slapd['root_password'] is not None)
-        # Check if pre-hashed or not.
-        # !!!!!!!!!!!!!!
-
-        # Right now, the way that rootpw works on ns-slapd works, it force hashes the pw
-        # see https://fedorahosted.org/389/ticket/48859
-        if not re.match('^\{[A-Z0-9]+\}.*$', slapd['root_password']):
-            # We need to hash it. Call pwdhash-bin.
-            # slapd['root_password'] = password_hash(slapd['root_password'], prefix=slapd['prefix'])
-            pass
-        else:
-            pass
-
-        # Create a random string
-        # Hash it.
-        # This will be our temporary rootdn password so that we can do
-        # live mods and setup rather than static ldif manipulations.
-        self._raw_secure_password = password_generate()
-        self._secure_password = password_hash(self._raw_secure_password, prefix=slapd['prefix'])
-
-        if self.verbose:
-            log.info("PASSED: root user checking")
-
-        assert(slapd['defaults'] is not None)
-        if self.verbose:
-            log.info("PASSED: using config settings %s" % slapd['defaults'])
-
-        assert(slapd['port'] is not None)
-        assert(socket_check_open('::1', slapd['port']) is False)
-        assert(slapd['secure_port'] is not None)
-        assert(socket_check_open('::1', slapd['secure_port']) is False)
-        if self.verbose:
-            log.info("PASSED: network avaliability checking")
-
-        # Make assertions of the paths?
-
-        # Make assertions of the backends?
-
-    def create_from_args(self, general, slapd, backends=[], extra=None):
-        """
-        Actually does the setup. this is what you want to call as an api.
-        """
-        # Check we have privs to run
-
-        if self.verbose:
-            log.info("READY: preparing installation")
-        self._prepare_ds(general, slapd, backends)
-        # Call our child api to prepare itself.
-        self._prepare(extra)
-
-        if self.verbose:
-            log.info("READY: beginning installation")
-
-        if self.dryrun:
-            log.info("NOOP: dry run requested")
-        else:
-            # Actually trigger the installation.
-            self._install_ds(general, slapd, backends)
-            # Call the child api to do anything it needs.
-            self._install(extra)
-        if self.verbose:
-            log.info("Directory Server is brought to you by the letter R and the number 27.")
-        log.info("FINISH: completed installation")
-
-    def _install_ds(self, general, slapd, backends):
-        """
-        Actually install the Ds from the dicts provided.
-
-        You should never call this directly, as it bypasses assertions.
-        """
-        # register the instance to /etc/sysconfig
-        # We do this first so that we can trick remove-ds.pl if needed.
-        # There may be a way to create this from template like the dse.ldif ...
-        initconfig = ""
-        with open("%s/dirsrv/config/template-initconfig" % slapd['sysconf_dir']) as template_init:
-            for line in template_init.readlines():
-                initconfig += line.replace('{{', '{', 1).replace('}}', '}', 1).replace('-', '_')
-        with open("%s/sysconfig/dirsrv-%s" % (slapd['sysconf_dir'], slapd['instance_name']), 'w') as f:
-            f.write(initconfig.format(
-                SERVER_DIR=slapd['lib_dir'],
-                SERVERBIN_DIR=slapd['sbin_dir'],
-                CONFIG_DIR=slapd['config_dir'],
-                INST_DIR=slapd['inst_dir'],
-                RUN_DIR=slapd['run_dir'],
-                DS_ROOT='',
-                PRODUCT_NAME='slapd',
-            ))
-
-        # Create all the needed paths
-        # we should only need to make bak_dir, cert_dir, config_dir, db_dir, ldif_dir, lock_dir, log_dir, run_dir? schema_dir,
-        for path in ('backup_dir', 'cert_dir', 'config_dir', 'db_dir', 'ldif_dir', 'lock_dir', 'log_dir', 'run_dir'):
-            if self.verbose:
-                log.info("ACTION: creating %s" % slapd[path])
-            try:
-                os.makedirs(slapd[path], mode=0o770)
-            except OSError:
-                pass
-            os.chown(slapd[path], slapd['user_uid'], slapd['group_gid'])
-
-        # Copy correct data to the paths.
-        # Copy in the schema
-        #  This is a little fragile, make it better.
-        shutil.copytree(os.path.join(slapd['sysconf_dir'], 'dirsrv/schema'), slapd['schema_dir'])
-        os.chown(slapd['schema_dir'], slapd['user_uid'], slapd['group_gid'])
-
-        # Copy in the collation
-        srcfile = os.path.join(slapd['sysconf_dir'], 'dirsrv/config/slapd-collations.conf')
-        dstfile = os.path.join(slapd['config_dir'], 'slapd-collations.conf')
-        shutil.copy2(srcfile, dstfile)
-        os.chown(slapd['schema_dir'], slapd['user_uid'], slapd['group_gid'])
-
-        # Selinux fixups?
-        # Restorecon of paths?
-        # Bind sockets to our type?
-
-        # Create certdb in sysconfidir
-        if self.verbose:
-            log.info("ACTION: Creating certificate database is %s" % slapd['cert_dir'])
-        # nss_create_new_database(slapd['cert_dir'])
-
-        # Create dse.ldif with a temporary root password.
-        # The template is in slapd['data_dir']/dirsrv/data/template-dse.ldif
-        # Variables are done with %KEY%.
-        # You could cheat and read it in, do a replace of % to { and } then use format?
-        if self.verbose:
-            log.info("ACTION: Creating dse.ldif")
-        dse = ""
-        with open(os.path.join(slapd['data_dir'], 'dirsrv', 'data', 'template-dse.ldif')) as template_dse:
-            for line in template_dse.readlines():
-                dse += line.replace('%', '{', 1).replace('%', '}', 1)
-
-        with open(os.path.join(slapd['config_dir'], 'dse.ldif'), 'w') as file_dse:
-            file_dse.write(dse.format(
-                schema_dir=slapd['schema_dir'],
-                lock_dir=slapd['lock_dir'],
-                tmp_dir=slapd['tmp_dir'],
-                cert_dir=slapd['cert_dir'],
-                ldif_dir=slapd['ldif_dir'],
-                bak_dir=slapd['backup_dir'],
-                run_dir=slapd['run_dir'],
-                inst_dir="",
-                log_dir=slapd['log_dir'],
-                fqdn=general['full_machine_name'],
-                ds_port=slapd['port'],
-                ds_user=slapd['user'],
-                rootdn=slapd['root_dn'],
-                # ds_passwd=slapd['root_password'],
-                ds_passwd=self._secure_password,  # We set our own password here, so we can connect and mod.
-                ds_suffix='',
-                config_dir=slapd['config_dir'],
-                db_dir=slapd['db_dir'],
-            ))
-
-        # open the connection to the instance.
-
-        # Should I move this import? I think this prevents some recursion
-        from lib389 import DirSrv
-        ds_instance = DirSrv(self.verbose)
-        args = {
-            SER_PORT: slapd['port'],
-            SER_SERVERID_PROP: slapd['instance_name'],
-            SER_ROOT_DN: slapd['root_dn'],
-            SER_ROOT_PW: self._raw_secure_password,
-            SER_DEPLOYED_DIR: slapd['prefix']
-        }
-
-        ds_instance.allocate(args)
-        # Does this work?
-        assert(ds_instance.exists())
-        # Start the server
-        ds_instance.start(timeout=60)
-        ds_instance.open()
-
-        # Create the backends as listed
-        # Load example data if needed.
-        for backend in backends:
-            ds_instance.backends.create(properties=backend)
-
-        # Make changes using the temp root
-        # Change the root password finally
-
-        # Complete.
-        ds_instance.config.set('nsslapd-rootpw',
-                               ensure_str(slapd['root_password']))
-
-    def _remove_ds(self):
-        """
-        The opposite of install: Removes an instance from the system.
-        This takes a backup of all relevant data, and removes the paths.
-        """
-        # This probably actually would need to be able to read the ldif, to
-        # know what to remove ...
-        for path in ('backup_dir', 'cert_dir', 'config_dir', 'db_dir',
-                     'ldif_dir', 'lock_dir', 'log_dir', 'run_dir'):
-            print(path)
-- 
1.8.3.1

