From c383ebce74c8e5d161590c06dc4e9ff3805f45eb Mon Sep 17 00:00:00 2001 From: "Thierry bordaz (tbordaz)" Date: Tue, 10 Dec 2013 16:41:36 +0100 Subject: [PATCH] Ticket 47625 - CI lib389: DirSrv not conform to the design Bug Description: Changes to make DirSrv class conform to the design http://port389.org/wiki/Upstream_test_framework Fix Description: The fix introduces several changes - Define SERVER properties with "verb" that could be used in CLI Those properties are used in 'args' dictionary to handle the instance (create/delete...) - Split the previous lib389.DirSrv:__init__, __localinit__ and __initPart2 into __init__/allocate/create/open - Implements lib389.DirSrv:delete/close/start/stop/restart/list/exists - Copy tools functions related to instance backup into instance class lib389.DirSrv clearBackupFS/checkBackupFS/backupFS/restoreFS - Add a new test module for DirSrv: dirsrv_test.py https://fedorahosted.org/389/ticket/47625 Reviewed by: Rich Megginson Platforms tested: F17 Flag Day: no Doc impact: no --- bug_harness.py | 3 +- lib389/__init__.py | 715 +++++++++++++++++++++++++++++++++++++++++++++++---- lib389/_constants.py | 24 ++ lib389/properties.py | 56 ++++ lib389/tools.py | 23 +- lib389/utils.py | 50 ++-- tests/dirsrv_test.py | 208 +++++++++++++++ 7 files changed, 983 insertions(+), 96 deletions(-) create mode 100644 lib389/properties.py create mode 100644 tests/dirsrv_test.py diff --git a/bug_harness.py b/bug_harness.py index 621ef84..b9143d0 100644 --- a/bug_harness.py +++ b/bug_harness.py @@ -1,6 +1,7 @@ from bug_harness import DSAdminHarness as DSAdmin from dsadmin import Entry from dsadmin.tools import DSAdminTools +from lib389.properties import * """ An harness for bug replication. @@ -94,7 +95,7 @@ class DSAdminHarness(DSAdmin, DSAdminTools): def createInstance(args): # eventually set prefix - args.setdefault('prefix', os.environ.get('PREFIX', None)) + args.setdefault(SER_DEPLOYED_DIR, os.environ.get('PREFIX', None)) args.setdefault('sroot', os.environ.get('SERVER_ROOT', None)) DSAdminTools.createInstance(args) diff --git a/lib389/__init__.py b/lib389/__init__.py index b69151f..2a3aa79 100644 --- a/lib389/__init__.py +++ b/lib389/__init__.py @@ -19,6 +19,7 @@ except ImportError: import sys import os +import pwd import os.path import base64 import urllib @@ -34,6 +35,8 @@ import shutil import datetime import logging import decimal +import glob +import tarfile from ldap.ldapobject import SimpleLDAPObject from ldapurl import LDAPUrl @@ -50,8 +53,13 @@ from lib389.utils import ( is_a_dn, normalizeDN, suffixfilt, - escapeDNValue + escapeDNValue, + update_newhost_with_fqdn, + formatInfData, + get_sbin_dir ) +from lib389.properties import * +from lib389.tools import DirSrvTools # mixin #from lib389.tools import DirSrvTools @@ -164,12 +172,21 @@ class DirSrv(SimpleLDAPObject): def __initPart2(self): """Initialize the DirSrv structure filling various fields, like: - - dbdir - - errlog - - confdir + self.errlog -> nsslapd-errorlog + self.accesslog -> nsslapd-accesslog + self.confdir -> nsslapd-certdir + self.inst -> equivalent to self.serverid + self.sroot/self.inst -> nsslapd-instancedir + self.dbdir -> dirname(nsslapd-directory) + + @param - self + + @return - None + + @raise ldap.LDAPError - if failure during initialization """ - if self.binddn and len(self.binddn) and not hasattr(self, 'sroot'): + if self.binddn and len(self.binddn): try: # XXX this fields are stale and not continuously updated # do they have sense? @@ -221,6 +238,16 @@ class DirSrv(SimpleLDAPObject): raise def __localinit__(self): + ''' + Establish a connection to the started instance. It binds with the binddn property, + then it initializes various fields from DirSrv (via __initPart2) + + @param - self + + @return - None + + @raise ldap.LDAPError - if failure during initialization + ''' uri = self.toLDAPURL() SimpleLDAPObject.__init__(self, uri) @@ -234,24 +261,24 @@ class DirSrv(SimpleLDAPObject): if ent: self.binddn = ent.dn else: - log.error("Error: could not find %s under %s" % ( + raise ValueError("Error: could not find %s under %s" % ( self.binddn, CFGSUFFIX)) - if not self.nobind: - needtls = False - while True: + + needtls = False + while True: + try: + if needtls: + self.start_tls_s() try: - if needtls: - self.start_tls_s() - try: - self.simple_bind_s(self.binddn, self.bindpw) - except ldap.SERVER_DOWN, e: - # TODO add server info in exception - log.debug("Cannot connect to %r" % uri) - raise e - break - except ldap.CONFIDENTIALITY_REQUIRED: - needtls = True - self.__initPart2() + self.simple_bind_s(self.binddn, self.bindpw) + except ldap.SERVER_DOWN, e: + # TODO add server info in exception + log.debug("Cannot connect to %r" % uri) + raise e + break + except ldap.CONFIDENTIALITY_REQUIRED: + needtls = True + self.__initPart2() def rebind(self): """Reconnect to the DS @@ -275,46 +302,626 @@ class DirSrv(SimpleLDAPObject): self.config = Config(self) self.index = Index(self) - def __init__(self, host='localhost', port=389, binddn='', bindpw='', serverId=None, nobind=False, sslport=0, verbose=False, offline=False): # default to anon bind - """We just set our instance variables and wrap the methods. - The real work is done in the following methods, reused during - instance creation & co. - * __localinit__ - * __initPart2 - - e.g. when using the start command, we just need to reconnect, - not create a new instance""" - log.info("Initializing %s with %s:%s" % (self.__class__, - host, sslport or port)) - self.__wrapmethods() + def __init__(self, verbose=False): + """ + This method does various initialization of DirSrv object: + parameters: + - 'state' to DIRSRV_STATE_INIT + - 'verbose' flag for debug purpose + - 'log' so that the use the module defined logger + + wrap the methods. + + - from SimpleLDAPObject + - from agreement, backends, suffix... + + It just create a DirSrv object. To use it the user will likely do + the following additional steps: + - allocate + - create + - open + """ + + self.state = DIRSRV_STATE_INIT self.verbose = verbose - self.port = port - self.sslport = sslport - self.host = host - self.binddn = binddn - self.bindpw = bindpw - self.nobind = nobind - self.isLocal = isLocalHost(host) - self.serverId = serverId - - - # - # dict caching DS structure - # - self.suffixes = {} - self.agmt = {} - if not offline: - # the real init - self.__localinit__() self.log = log - # add brookers + + self.__wrapmethods() self.__add_brookers__() - def __str__(self): """XXX and in SSL case?""" return self.host + ":" + str(self.port) + def allocate(self, args): + ''' + Initialize a DirSrv object according to the provided args. + The final state -> DIRSRV_STATE_ALLOCATED + @param args - dictionary that contains the DirSrv properties + properties are + SER_SERVERID_PROP: mandatory server id of the instance -> slapd- + SER_HOST: hostname [LOCALHOST] + SER_PORT: normal ldap port [DEFAULT_PORT] + SER_SECURE_PORT: secure ldap port + SER_ROOT_DN: root DN [DN_DM] + SER_ROOT_PW: password of root DN [PW_DM] + SER_USER_ID: user id of the create instance [DEFAULT_USER] + SER_GROUP_ID: group id of the create instance [SER_USER_ID] + SER_DEPLOYED_DIR: directory where 389-ds is deployed + SER_BACKUP_INST_DIR: directory where instances will be backup + + @return None + + @raise ValueError - if missing mandatory properties or invalid state of DirSrv + ''' + if self.state != DIRSRV_STATE_INIT and self.state != DIRSRV_STATE_ALLOCATED: + raise ValueError("invalid state for calling allocate: %s" % self.state) + + if SER_SERVERID_PROP not in args: + raise ValueError("%s is a mandatory parameter" % SER_SERVERID_PROP) + + # Settings from args of server attributes + self.host = args.get(SER_HOST, LOCALHOST) + self.port = args.get(SER_PORT, DEFAULT_PORT) + self.sslport = args.get(SER_SECURE_PORT) + self.binddn = args.get(SER_ROOT_DN, DN_DM) + self.bindpw = args.get(SER_ROOT_PW, PW_DM) + self.creation_suffix = args.get(SER_CREATION_SUFFIX, DEFAULT_SUFFIX) + self.userid = args.get(SER_USER_ID) + if not self.userid: + if os.getuid() == 0: + # as root run as default user + self.userid = DEFAULT_USER + else: + self.userid = pwd.getpwuid( os.getuid() )[ 0 ] + + + # Settings from args of server attributes + self.serverid = args.get(SER_SERVERID_PROP) + self.groupid = args.get(SER_GROUP_ID, self.userid) + self.backupdir = args.get(SER_BACKUP_INST_DIR, DEFAULT_BACKUPDIR) + self.prefix = args.get(SER_DEPLOYED_DIR, None) + + # Those variables needs to be revisited (sroot for 64 bits) + self.sroot = os.path.join(self.prefix, "lib/dirsrv") + self.errlog = os.path.join(self.prefix, "var/log/dirsrv/slapd-%s/errors" % self.serverid) + + # For compatibility keep self.inst but should be removed + self.inst = self.serverid + + # additional settings + self.isLocal = isLocalHost(self.host) + self.suffixes = {} + self.agmt = {} + + self.state = DIRSRV_STATE_ALLOCATED + self.log.info("Allocate %s with %s:%s" % (self.__class__, + self.host, + self.sslport or self.port)) + + def list(self, all=False): + """ + Returns a list of files containing the environment of the created instances + that are on the local file system (e.g. /etc/dirsrv/slapd-*). + If 'all' is True, it returns all the files else it returns only the + environment file of the current instance. + By default it is False and only returns the current instance file. + + @param all - True or False . default is [False] + + @return list file(s) name(s) + + @raise None + """ + # Retrieves all instances under '/etc/sysconfig' and '/etc/dirsrv' + + # Instances/Environment are + # + # file: /etc/sysconfig/dirsrv- (env) + # inst: /etc/dirsrv/slapd- (conf) + # + # or + # + # file: $HOME/.dirsrv/dirsrv- (env) + # inst: /etc/dirsrv/slapd- (conf) + # + + prefix = self.prefix or '/' + + # configuration is under '/etc/dirsrv/slapd-' + conf_head = os.path.join(os.path.join(prefix, CONF_DIR), DEFAULT_INST_HEAD) + conf_dir_pattern = "%s*" % conf_head + + # environment is under '/etc/sysconfig/dirsrv-' or '$HOME/.dirsrv/dirsrv-' + if prefix: + env_head = "%s/%s/%s" % (os.getenv('HOME'), ENV_LOCAL_DIR, DEFAULT_ENV_HEAD) + else: + # /etc/sysconfig/dirsrv- + env_head = "%s/%s" (ENV_SYSCONFIG_DIR, DEFAULT_ENV_HEAD) + + + if not all: + # easy case we just look for the current instance + + # check the configuration directory exist + if not os.path.isdir(conf_head + self.serverid): + # Ok this instance has not been created + return [] + + # Now check the environment file exist + env_file = env_head + self.serverid + if not os.path.isfile(env_file): + # Ok this instance has no environment file + return [] + + instances = [ env_file ] + else: + # all instances with ENV and configuration will be returned + + # Retrieves all instances under '/etc/dirsrv/slapd-*'. All the created instance. + instances = [] + for instance in glob.glob(conf_dir_pattern): + + serverid = os.path.basename(instance)[len(DEFAULT_INST_HEAD):] + + # skip removed instance + if '.removed' in serverid: + continue + + # Skip instance without environment file + env_file = env_head + serverid + if os.path.isfile(env_file): + instances.append(env_file) + + return instances + + + def _createDirsrv(self, verbose=0): + """Create a new instance of directory server + + @param self - containing the set properties + + SER_HOST (host) + SER_PORT (port) + SER_SECURE_PORT (sslport) + SER_ROOT_DN (binddn) + SER_ROOT_PW (bindpw) + SER_CREATION_SUFFIX (creation_suffix) + SER_USER_ID (userid) + SER_SERVERID_PROP (serverid) + SER_GROUP_ID (groupid) + SER_DEPLOYED_DIR (prefix) + SER_BACKUP_INST_DIR (backupdir) + + @return None + + @raise None + + } + """ + + prog = get_sbin_dir(None, self.prefix) + CMD_PATH_SETUP_DS + + if not os.path.isfile(prog): + log.error("Can't find file: %r, removing extension" % prog) + prog = prog[:-3] + + args = {SER_HOST: self.host, + SER_PORT: self.port, + SER_SECURE_PORT: self.sslport, + SER_ROOT_DN: self.binddn, + SER_ROOT_PW: self.bindpw, + SER_CREATION_SUFFIX: self.creation_suffix, + SER_USER_ID: self.userid, + SER_SERVERID_PROP: self.serverid, + SER_GROUP_ID: self.groupid, + SER_DEPLOYED_DIR: self.prefix, + SER_BACKUP_INST_DIR: self.backupdir} + content = formatInfData(args) + DirSrvTools.runInfProg(prog, content, verbose) + + + def create(self): + """ + Creates an instance with the parameters sets in dirsrv + The state change from DIRSRV_STATE_ALLOCATED -> DIRSRV_STATE_OFFLINE + + @param - self + + @return - None + + @raise ValueError - if it exist an instance with the same 'serverid' + """ + # check that DirSrv was in DIRSRV_STATE_ALLOCATED state + if self.state != DIRSRV_STATE_ALLOCATED: + raise ValueError("invalid state for calling create: %s" % self.state) + + # Check that the instance does not already exist + env_files = self.list() + if len(env_files) != 0: + raise ValueError("Error it already exists the instance (%s)" % env_files[0]) + + # Time to create the instance + self._createDirsrv(verbose=self.verbose) + + # Now the instance is created but DirSrv is not yet connected to it + self.state = DIRSRV_STATE_OFFLINE + + def delete(self): + ''' + Deletes the instance with the parameters sets in dirsrv + The state changes from DIRSRV_STATE_OFFLINE -> DIRSRV_STATE_ALLOCATED + + @param self + + @return None + + @raise ValueError - if it does not exist an instance 'serverid' + ''' + # check that DirSrv was in DIRSRV_STATE_OFFLINE state + if self.state != DIRSRV_STATE_OFFLINE: + raise ValueError("invalid state for calling delete: %s" % self.state) + + # Check that the instance does not already exist + env_files = self.list() + if len(env_files) != 1: + raise ValueError("Error can not find instance %s[%s:%d]" % + (self.serverid, self.host, self.port)) + + # Now time to remove the instance + prog = get_sbin_dir(None, self.prefix) + CMD_PATH_REMOVE_DS + cmd = "%s -i %s%s" % (prog, DEFAULT_INST_HEAD, self.serverid) + self.log.debug("running: %s " % cmd) + try: + os.system(cmd) + except: + log.exception("error executing %r" % cmd) + + self.state = DIRSRV_STATE_ALLOCATED + + def open(self): + ''' + It opens a ldap bound connection to dirsrv so that online administrative tasks are possible. + It binds with the binddn property, then it initializes various fields from DirSrv (via __initPart2) + + The state changes -> DIRSRV_STATE_ONLINE + + @param self + + @return None + + @raise ValueError - if the instance has not the right state or can not find the binddn to bind + ''' + # check that DirSrv was in DIRSRV_STATE_OFFLINE or DIRSRV_STATE_ONLINE state + if self.state != DIRSRV_STATE_OFFLINE and self.state != DIRSRV_STATE_ONLINE: + raise ValueError("invalid state for calling open: %s" % self.state) + + uri = self.toLDAPURL() + + SimpleLDAPObject.__init__(self, uri) + + # see if binddn is a dn or a uid that we need to lookup + if self.binddn and not is_a_dn(self.binddn): + self.simple_bind_s("", "") # anon + ent = self.getEntry(CFGSUFFIX, ldap.SCOPE_SUBTREE, + "(uid=%s)" % self.binddn, + ['uid']) + if ent: + self.binddn = ent.dn + else: + raise ValueError("Error: could not find %s under %s" % ( + self.binddn, CFGSUFFIX)) + + needtls = False + while True: + try: + if needtls: + self.start_tls_s() + try: + self.simple_bind_s(self.binddn, self.bindpw) + except ldap.SERVER_DOWN, e: + # TODO add server info in exception + log.debug("Cannot connect to %r" % uri) + raise e + break + except ldap.CONFIDENTIALITY_REQUIRED: + needtls = True + self.__initPart2() + + self.state = DIRSRV_STATE_ONLINE + + def close(self): + ''' + It closes connection to dirsrv. Online administrative tasks are no longer possible. + + The state changes from DIRSRV_STATE_ONLINE -> DIRSRV_STATE_OFFLINE + + @param self + + @return None + + @raise ValueError - if the instance has not the right state + ''' + + # check that DirSrv was in DIRSRV_STATE_ONLINE state + if self.state != DIRSRV_STATE_ONLINE: + raise ValueError("invalid state for calling close: %s" % self.state) + + SimpleLDAPObject.unbind(self) + + self.state = DIRSRV_STATE_OFFLINE + + def start(self, timeout): + ''' + It starts an instance and rebind it. Its final state after rebind (open) + is DIRSRV_STATE_ONLINE + + @param self + @param timeout (in sec) to wait for successful start + + @return None + + @raise None + ''' + # Default starting timeout (to find 'slapd started' in error log) + if not timeout: + timeout = 120 + + # called with the default timeout + if DirSrvTools.start(self, verbose=False, timeout=timeout): + self.log.error("Probable failure to start the instance") + + self.open() + + def stop(self, timeout): + ''' + It stops an instance. + It changes the state -> DIRSRV_STATE_OFFLINE + + @param self + @param timeout (in sec) to wait for successful stop + + @return None + + @raise None + ''' + + # Default starting timeout (to find 'slapd started' in error log) + if not timeout: + timeout = 120 + + # called with the default timeout + if DirSrvTools.stop(self, verbose=False, timeout=timeout): + self.log.error("Probable failure to stop the instance") + + #whatever the initial state, the instance is now Offline + self.state = DIRSRV_STATE_OFFLINE + + def restart(self, timeout): + ''' + It restarts an instance and rebind it. Its final state after rebind (open) + is DIRSRV_STATE_ONLINE. + + @param self + @param timeout (in sec) to wait for successful stop + + @return None + + @raise None + ''' + self.stop(timeout) + self.start(timeout) + + def _infoBackupFS(self): + """ + Return the information to retrieve the backup file of a given instance + It returns: + - Directory name containing the backup (e.g. /tmp/slapd-standalone.bck) + - The pattern of the backup files (e.g. /tmp/slapd-standalone.bck/backup*.tar.gz) + """ + backup_dir = "%s/slapd-%s.bck" % (self.backupdir, self.serverid) + backup_pattern = os.path.join(backup_dir, "backup*.tar.gz") + return backup_dir, backup_pattern + + def clearBackupFS(self, backup_file=None): + """ + Remove a backup_file or all backup up of a given instance + """ + if backup_file: + if os.path.isfile(backup_file): + try: + os.remove(backup_file) + except: + log.info("clearBackupFS: fail to remove %s" % backup_file) + pass + else: + backup_dir, backup_pattern = self._infoBackupFS() + list_backup_files = glob.glob(backup_pattern) + for f in list_backup_files: + try: + os.remove(f) + except: + log.info("clearBackupFS: fail to remove %s" % backup_file) + pass + + + def checkBackupFS(self): + """ + If it exits a backup file, it returns it + else it returns None + """ + + backup_dir, backup_pattern = self._infoBackupFS() + list_backup_files = glob.glob(backup_pattern) + if not list_backup_files: + return None + else: + # returns the first found backup + return list_backup_files[0] + + + def backupFS(self): + """ + Saves the files of an instance under /tmp/slapd-.bck/backup_HHMMSS.tar.gz + and return the archive file name. + If it already exists a such file, it assums it is a valid backup and + returns its name + + self.sroot : root of the instance (e.g. /usr/lib64/dirsrv) + self.inst : instance name (e.g. standalone for /etc/dirsrv/slapd-standalone) + self.confdir : root of the instance config (e.g. /etc/dirsrv) + self.dbdir: directory where is stored the database (e.g. /var/lib/dirsrv/slapd-standalone/db) + self.changelogdir: directory where is stored the changelog (e.g. /var/lib/dirsrv/slapd-master/changelogdb) + """ + + # First check it if already exists a backup file + backup_dir, backup_pattern = self._infoBackupFS() + backup_file = self.checkBackupFS() + if backup_file is None: + if not os.path.exists(backup_dir): + os.makedirs(backup_dir) + else: + return backup_file + + # goes under the directory where the DS is deployed + listFilesToBackup = [] + here = os.getcwd() + os.chdir(self.prefix) + prefix_pattern = "%s/" % self.prefix + + # build the list of directories to scan + instroot = "%s/slapd-%s" % (self.sroot, self.serverid) + ldir = [ instroot ] + if hasattr(self, 'confdir'): + ldir.append(self.confdir) + if hasattr(self, 'dbdir'): + ldir.append(self.dbdir) + if hasattr(self, 'changelogdir'): + ldir.append(self.changelogdir) + if hasattr(self, 'errlog'): + ldir.append(os.path.dirname(self.errlog)) + if hasattr(self, 'accesslog') and os.path.dirname(self.accesslog) not in ldir: + ldir.append(os.path.dirname(self.accesslog)) + + # now scan the directory list to find the files to backup + for dirToBackup in ldir: + for root, dirs, files in os.walk(dirToBackup): + for file in files: + name = os.path.join(root, file) + name = re.sub(prefix_pattern, '', name) + + if os.path.isfile(name): + listFilesToBackup.append(name) + log.debug("backupFS add = %s (%s)" % (name, self.prefix)) + + + # create the archive + name = "backup_%s.tar.gz" % (time.strftime("%m%d%Y_%H%M%S")) + backup_file = os.path.join(backup_dir, name) + tar = tarfile.open(backup_file, "w:gz") + + + for name in listFilesToBackup: + if os.path.isfile(name): + tar.add(name) + tar.close() + log.info("backupFS: archive done : %s" % backup_file) + + # return to the directory where we were + os.chdir(here) + + return backup_file + + def restoreFS(self, backup_file): + """ + """ + + # First check the archive exists + if backup_file is None: + log.warning("Unable to restore the instance (missing backup)") + return 1 + if not os.path.isfile(backup_file): + log.warning("Unable to restore the instance (%s is not a file)" % backup_file) + return 1 + + # + # Second do some clean up + # + + # previous db (it may exists new db files not in the backup) + log.debug("restoreFS: remove subtree %s/*" % self.dbdir) + for root, dirs, files in os.walk(self.dbdir): + for d in dirs: + if d not in ("bak", "ldif"): + log.debug("restoreFS: before restore remove directory %s/%s" % (root, d)) + shutil.rmtree("%s/%s" % (root, d)) + + # previous error/access logs + log.debug("restoreFS: remove error logs %s" % self.errlog) + for f in glob.glob("%s*" % self.errlog): + log.debug("restoreFS: before restore remove file %s" % (f)) + os.remove(f) + log.debug("restoreFS: remove access logs %s" % self.accesslog) + for f in glob.glob("%s*" % self.accesslog): + log.debug("restoreFS: before restore remove file %s" % (f)) + os.remove(f) + + + # Then restore from the directory where DS was deployed + here = os.getcwd() + os.chdir(self.prefix) + + tar = tarfile.open(backup_file) + for member in tar.getmembers(): + if os.path.isfile(member.name): + # + # restore only writable files + # It could be a bad idea and preferably restore all. + # Now it will be easy to enhance that function. + if os.access(member.name, os.W_OK): + log.debug("restoreFS: restored %s" % member.name) + tar.extract(member.name) + else: + log.debug("restoreFS: not restored %s (no write access)" % member.name) + else: + log.debug("restoreFS: restored %s" % member.name) + tar.extract(member.name) + + tar.close() + + # + # Now be safe, triggers a recovery at restart + # + guardian_file = os.path.join(self.dbdir, "db/guardian") + if os.path.isfile(guardian_file): + try: + log.debug("restoreFS: remove %s" % guardian_file) + os.remove(guardian_file) + except: + log.warning("restoreFS: fail to remove %s" % guardian_file) + pass + + + os.chdir(here) + + + def exists(self): + ''' + Check if an instance exists. + It checks that both exists: + - configuration directory (/etc/dirsrv/slapd-) + - environment file (/etc/sysconfig/dirsrv- or $HOME/.dirsrv/dirsrv-) + + @param None + + @return True of False if the instance exists or not + + @raise None + + ''' + env_files = self.list() + return len(env_files) == 1 + def toLDAPURL(self): """Return the uri ldap[s]://host:[ssl]port.""" if self.sslport: @@ -323,7 +930,7 @@ class DirSrv(SimpleLDAPObject): return "ldap://%s:%d/" % (self.host, self.port) def getServerId(self): - return self.serverId + return self.serverid # # Get entries diff --git a/lib389/_constants.py b/lib389/_constants.py index b0eb23a..c2ce661 100644 --- a/lib389/_constants.py +++ b/lib389/_constants.py @@ -34,6 +34,30 @@ DEFAULT_USER = "nobody" DN_DM = "cn=Directory Manager" DN_CONFIG = "cn=config" DN_LDBM = "cn=ldbm database,cn=plugins,cn=config" + +CMD_PATH_SETUP_DS = "/setup-ds.pl" +CMD_PATH_REMOVE_DS = "/remove-ds.pl" + +# State of an DirSrv object +DIRSRV_STATE_INIT='initial' +DIRSRV_STATE_ALLOCATED='allocated' +DIRSRV_STATE_OFFLINE='offline' +DIRSRV_STATE_ONLINE='online' + +LOCALHOST = "localhost.localdomain" +DEFAULT_PORT = 389 +DEFAULT_SECURE_PORT = 636 +DEFAULT_SUFFIX = 'dc=example,dc=com' +DEFAULT_BACKUPDIR = '/tmp' +DEFAULT_INST_HEAD = 'slapd-' +DEFAULT_ENV_HEAD = 'dirsrv-' + +PW_DM = "password" + +CONF_DIR = 'etc/dirsrv' +ENV_SYSCONFIG_DIR = '/etc/sysconfig' +ENV_LOCAL_DIR = '.dirsrv' + DN_MAPPING_TREE = "cn=mapping tree,cn=config" DN_CHAIN = "cn=chaining database,cn=plugins,cn=config" DN_CHANGELOG = "cn=changelog5,cn=config" diff --git a/lib389/properties.py b/lib389/properties.py new file mode 100644 index 0000000..32a210a --- /dev/null +++ b/lib389/properties.py @@ -0,0 +1,56 @@ +''' +Created on Dec 5, 2013 + +@author: tbordaz +''' + +# +# Properties supported by the server WITH related attribute name +# +SER_HOST='hostname' +SER_PORT='ldap-port' +SER_SECURE_PORT='ldap-secureport' +SER_ROOT_DN='root-dn' +SER_ROOT_PW='root-pw' +SER_USER_ID='user-id' + +SER_PROPNAME_TO_ATTRNAME= {SER_HOST:'nsslapd-localhost', + SER_PORT:'nsslapd-port', + SER_SECURE_PORT:'nsslapd-securePort', + SER_ROOT_DN:'nsslapd-rootdn', + SER_ROOT_PW:'nsslapd-rootpw', + SER_USER_ID:'nsslapd-localuser'} +# +# Properties supported by the server WITHOUT related attribute name +# +SER_SERVERID_PROP='server-id' +SER_GROUP_ID='group-id' +SER_DEPLOYED_DIR='deployed-dir' +SER_BACKUP_INST_DIR='inst-backupdir' +SER_CREATION_SUFFIX='suffix' + +# +# Properties supported by the replica agreement +# +RA_SCHEDULE_PROP='schedule' +RA_TRANSPORT_PROT_PROP='transport-prot' +RA_FRAC_EXCLUDE_PROP='fractional-exclude-attrs-inc' +RA_FRAC_EXCLUDE_TOTAL_UPDATE_PROP='fractional-exclude-attrs-total' +RA_FRAC_STRIP_PROP='fractional-strip-attrs' +RA_CONSUMER_PORT='consumer-port' +RA_CONSUMER_HOST='consumer-host' +RA_CONSUMER_TOTAL_INIT='consumer-total-init' +RA_TIMEOUT='timeout' + +RA_PROPNAME_TO_ATTRNAME = {RA_SCHEDULE_PROP:'nsds5replicaupdateschedule', + RA_TRANSPORT_PROT_PROP:'nsds5replicatransportinfo', + RA_FRAC_EXCLUDE_PROP:'nsDS5ReplicatedAttributeList', + RA_FRAC_EXCLUDE_TOTAL_UPDATE_PROP:'nsDS5ReplicatedAttributeListTotal', + RA_FRAC_STRIP_PROP:'nsds5ReplicaStripAttrs', + RA_CONSUMER_PORT:'nsds5replicaport', + RA_CONSUMER_HOST:'nsds5ReplicaHost', + RA_CONSUMER_TOTAL_INIT:'nsds5BeginReplicaRefresh', + RA_TIMEOUT:'nsds5replicatimeout'} + + + \ No newline at end of file diff --git a/lib389/tools.py b/lib389/tools.py index b4b741e..f03e951 100644 --- a/lib389/tools.py +++ b/lib389/tools.py @@ -27,7 +27,8 @@ import re import glob import lib389 -from lib389 import InvalidArgumentError +from lib389 import * +from lib389.properties import * from lib389.utils import ( getcfgdsuserdn, @@ -49,8 +50,8 @@ log = logging.getLogger(__name__) # Private constants PATH_SETUP_DS_ADMIN = "/setup-ds-admin.pl" -PATH_SETUP_DS = "/setup-ds.pl" -PATH_REMOVE_DS = "/remove-ds.pl" +PATH_SETUP_DS = CMD_PATH_SETUP_DS +PATH_REMOVE_DS = CMD_PATH_REMOVE_DS PATH_ADM_CONF = "/etc/dirsrv/admin-serv/adm.conf" class DirSrvTools(object): @@ -554,7 +555,7 @@ class DirSrvTools(object): prefix = None prog = get_sbin_dir(None, prefix) + PATH_REMOVE_DS - cmd = "%s -i slapd-%s" % (prog, dirsrv.serverId) + cmd = "%s -i slapd-%s" % (prog, dirsrv.serverid) log.debug("running: %s " % cmd) try: os.system(cmd) @@ -570,16 +571,14 @@ class DirSrvTools(object): The properties set are: instance.host instance.port - instance.serverId + instance.serverid instance.inst instance.prefix instance.backup ''' - instance = lib389.DirSrv(host=args['newhost'], port=args['newport'], - serverId=args['newinstance'], offline=True) - instance.prefix = args.get('prefix', '/') - instance.backupdir = args.get('backupdir', '/tmp') - instance.inst = instance.serverId + instance = lib389.DirSrv(verbose=True) + instance.allocate(args) + return instance @staticmethod @@ -602,8 +601,8 @@ class DirSrvTools(object): If it exists it returns a DirSrv instance NOT initialized, else None ''' instance = DirSrvTools._offlineDirsrv(args) - dirname = os.path.join(instance.prefix, "etc/dirsrv/slapd-%s" % instance.serverId) - errorlog = os.path.join(instance.prefix, "var/log/dirsrv/slapd-%s/errors" % instance.serverId) + dirname = os.path.join(instance.prefix, "etc/dirsrv/slapd-%s" % instance.serverid) + errorlog = os.path.join(instance.prefix, "var/log/dirsrv/slapd-%s/errors" % instance.serverid) sroot = os.path.join(instance.prefix, "lib/dirsrv") if os.path.isdir(dirname) and \ os.path.isfile(errorlog) and \ diff --git a/lib389/utils.py b/lib389/utils.py index c7f2755..2983776 100644 --- a/lib389/utils.py +++ b/lib389/utils.py @@ -2,6 +2,8 @@ TODO put them in a module! """ +from lib389.properties import SER_PORT, SER_ROOT_PW, SER_SERVERID_PROP,\ + SER_ROOT_DN try: from subprocess import Popen as my_popen, PIPE except ImportError: @@ -30,6 +32,7 @@ import ldap import lib389 from lib389 import DN_CONFIG from lib389._constants import * +from lib389.properties import * # # Decorator @@ -187,18 +190,18 @@ def get_server_user(args): def update_newhost_with_fqdn(args): - """Replace args['newhost'] with its fqdn and returns True if local. + """Replace args[SER_HOST] with its fqdn and returns True if local. One of the arguments to createInstance is newhost. If this is specified, we need to convert it to the fqdn. If not given, we need to figure out what the fqdn of the local host is. This method sets newhost in args to the appropriate value and returns True if newhost is the localhost, False otherwise""" - if 'newhost' in args: - args['newhost'] = getfqdn(args['newhost']) - isLocal = isLocalHost(args['newhost']) + if SER_HOST in args: + args[SER_HOST] = getfqdn(args[SER_HOST]) + isLocal = isLocalHost(args[SER_HOST]) else: isLocal = True - args['newhost'] = getfqdn() + args[SER_HOST] = getfqdn() return isLocal @@ -346,9 +349,9 @@ def getadminport(cfgconn, cfgdn, args): dn = cfgdn if 'admin_domain' in args: dn = "cn=%s,ou=%s, %s" % ( - args['newhost'], args['admin_domain'], cfgdn) + args[SER_HOST], args['admin_domain'], cfgdn) filt = "(&(objectclass=nsAdminServer)(serverHostName=%s)" % args[ - 'newhost'] + SER_HOST] if 'sroot' in args: filt += "(serverRoot=%s)" % args['sroot'] filt += ")" @@ -378,7 +381,7 @@ def formatInfData(args): args = { # new instance values - newhost, newuserid, newport, newrootdn, newrootpw, newsuffix, + newhost, newuserid, newport, SER_ROOT_DN, newrootpw, newsuffix, # The following parameters require to register the new instance # in the admin server @@ -422,17 +425,10 @@ def formatInfData(args): args = args.copy() args['CFGSUFFIX'] = lib389.CFGSUFFIX - content = ( - "[General]" "\n" - "FullMachineName= %(newhost)s" "\n" - "SuiteSpotUserID= %(newuserid)s" "\n" - ) % args - - # by default, use groupname=username - if 'SuiteSpotGroup' in args: - content += """\nSuiteSpotGroup= %s\n""" % args['SuiteSpotGroup'] - else: - content += """\nSuiteSpotGroup= %(newuserid)s\n""" % args + content = ("[General]" "\n") + content += ("FullMachineName= %s\n" % args[SER_HOST]) + content += ("SuiteSpotUserID= %s\n" % args[SER_USER_ID]) + content += ("nSuiteSpotGroup= %s\n" % args[SER_GROUP_ID]) if args.get('have_admin'): content += ( @@ -442,16 +438,12 @@ def formatInfData(args): "ConfigDirectoryAdminPwd= %(cfgdspwd)s" "\n" ) % args - content += ("\n" "\n" - "[slapd]" "\n" - "ServerPort= %(newport)s" "\n" - "RootDN= %(newrootdn)s" "\n" - "RootDNPwd= %(newrootpw)s" "\n" - "ServerIdentifier= %(newinstance)s" "\n" - "Suffix= %(newsuffix)s" "\n" - ) % args - - + content += ("\n" "\n" "[slapd]" "\n") + content += ("ServerPort= %s\n" % args[SER_PORT]) + content += ("RootDN= %s\n" % args[SER_ROOT_DN]) + content += ("RootDNPwd= %s\n" % args[SER_ROOT_PW]) + content += ("ServerIdentifier= %s\n" % args[SER_SERVERID_PROP]) + content += ("Suffix= %s\n" % args[SER_CREATION_SUFFIX]) # Create admin? if args.get('setup_admin'): diff --git a/tests/dirsrv_test.py b/tests/dirsrv_test.py new file mode 100644 index 0000000..8f70e2b --- /dev/null +++ b/tests/dirsrv_test.py @@ -0,0 +1,208 @@ +''' +Created on Dec 9, 2013 + +@author: tbordaz +''' + +import os +import pwd +import ldap +from random import randint +from lib389.tools import DirSrvTools +from lib389._constants import * +from lib389.properties import * +from lib389 import DirSrv,Entry + +TEST_REPL_DN = "uid=test,%s" % DEFAULT_SUFFIX +INSTANCE_PORT = 54321 +INSTANCE_SERVERID = 'dirsrv' +#INSTANCE_PREFIX = os.environ.get('PREFIX', None) +INSTANCE_PREFIX = '/home/tbordaz/install' +INSTANCE_BACKUP = os.environ.get('BACKUPDIR', DEFAULT_BACKUPDIR) + +class Test_dirsrv(): + def _add_user(self, success=True): + try: + self.instance.add_s(Entry((TEST_REPL_DN, {'objectclass': "top person organizationalPerson inetOrgPerson".split(), + 'uid': 'test', + 'sn': 'test', + 'cn': 'test'}))) + except Exception as e: + if success: + raise + else: + self.instance.log.info('Fail to add (expected): %s' % e.args) + return + + self.instance.log.info('%s added' % TEST_REPL_DN) + + def _mod_user(self, success=True): + try: + replace = [(ldap.MOD_REPLACE, 'description', str(randint(1, 100)))] + self.instance.modify_s(TEST_REPL_DN, replace) + except Exception as e: + if success: + raise + else: + self.instance.log.info('Fail to modify (expected): %s' % e.args) + return + + self.instance.log.info('%s modified' % TEST_REPL_DN) + + def setUp(self): + pass + + + def tearDown(self): + pass + + + def test_allocate(self, verbose=False): + instance = DirSrv(verbose=verbose) + instance.log.debug("Instance allocated") + assert instance.state == DIRSRV_STATE_INIT + + # Check that SER_SERVERID_PROP is a mandatory parameter + args = {SER_HOST: LOCALHOST, + SER_PORT: INSTANCE_PORT, + SER_DEPLOYED_DIR: INSTANCE_PREFIX + } + try: + instance.allocate(args) + except Exception as e: + instance.log.info('Allocate fails (normal): %s' % e.args) + assert type(e) == ValueError + assert e.args[0].find("%s is a mandatory parameter" % SER_SERVERID_PROP) >= 0 + pass + + # Check the state + assert instance.state == DIRSRV_STATE_INIT + + # Now do a successful allocate + args[SER_SERVERID_PROP] = INSTANCE_SERVERID + instance.allocate(args) + + userid = pwd.getpwuid( os.getuid() )[ 0 ] + + + # Now verify the settings + assert instance.state == DIRSRV_STATE_ALLOCATED + assert instance.host == LOCALHOST + assert instance.port == INSTANCE_PORT + assert instance.sslport == None + assert instance.binddn == DN_DM + assert instance.bindpw == PW_DM + assert instance.creation_suffix == DEFAULT_SUFFIX + assert instance.userid == userid + assert instance.serverid == INSTANCE_SERVERID + assert instance.groupid == instance.userid + assert instance.prefix == INSTANCE_PREFIX + assert instance.backupdir == INSTANCE_BACKUP + + # Now check we can change the settings of an allocated DirSrv + args = {SER_SERVERID_PROP:INSTANCE_SERVERID, + SER_HOST: LOCALHOST, + SER_PORT: INSTANCE_PORT, + SER_DEPLOYED_DIR: INSTANCE_PREFIX, + SER_ROOT_DN: "uid=foo"} + instance.allocate(args) + assert instance.state == DIRSRV_STATE_ALLOCATED + assert instance.host == LOCALHOST + assert instance.port == INSTANCE_PORT + assert instance.sslport == None + assert instance.binddn == "uid=foo" + assert instance.bindpw == PW_DM + assert instance.creation_suffix == DEFAULT_SUFFIX + assert instance.userid == userid + assert instance.serverid == INSTANCE_SERVERID + assert instance.groupid == instance.userid + assert instance.prefix == INSTANCE_PREFIX + assert instance.backupdir == INSTANCE_BACKUP + + # OK restore back the valid parameters + args = {SER_SERVERID_PROP:INSTANCE_SERVERID, + SER_HOST: LOCALHOST, + SER_PORT: INSTANCE_PORT, + SER_DEPLOYED_DIR: INSTANCE_PREFIX} + instance.allocate(args) + assert instance.state == DIRSRV_STATE_ALLOCATED + assert instance.host == LOCALHOST + assert instance.port == INSTANCE_PORT + assert instance.sslport == None + assert instance.binddn == DN_DM + assert instance.bindpw == PW_DM + assert instance.creation_suffix == DEFAULT_SUFFIX + assert instance.userid == userid + assert instance.serverid == INSTANCE_SERVERID + assert instance.groupid == instance.userid + assert instance.prefix == INSTANCE_PREFIX + assert instance.backupdir == INSTANCE_BACKUP + + self.instance = instance + + def test_list_init(self): + ''' + Lists the instances on the file system + ''' + for env_file in self.instance.list(): + self.instance.log.info("list instance: %s" % env_file) + + for env_file in self.instance.list(all=True): + self.instance.log.info("list instances: %s" % env_file) + + try: + self._add_user() + except Exception as e: + self.instance.log.info('Fail to add: %s' % e.args) + pass + + + + def test_allocated_to_offline(self): + self.instance.create() + + + def test_offline_to_online(self): + self.instance.open() + + def test_online_to_offline(self): + self.instance.close() + + def test_offline_to_allocated(self): + self.instance.delete() + + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + test = Test_dirsrv() + test.setUp() + + # Allocated the instance, except preparing the instance + test.test_allocate(False) + + clean = False + if not clean: + # Do a listing of the instances + test.test_list_init() + test._add_user(success=False) + + # Create the instance + test.test_allocated_to_offline() + test._add_user(success=False) + + # bind to the instance + test.test_offline_to_online() + test._add_user(success=True) + + test._mod_user(success=True) + + #Unbind to the instance + test.test_online_to_offline() + test._mod_user(success=False) + + test.test_list_init() + else: + test.instance.state = DIRSRV_STATE_OFFLINE + test.test_offline_to_allocated() + test.tearDown() + \ No newline at end of file -- 1.7.11.7