From 4b119b412b273437bf8de884c1d381e108dcaa3a Mon Sep 17 00:00:00 2001 From: "Thierry bordaz (tbordaz)" Date: Tue, 10 Dec 2013 16:41:36 +0100 Subject: [PATCH] Ticket 47625 - CI lib389: DirSrv not conform to the design Bug Description: Changes to make DirSrv class conform to the design http://port389.org/wiki/Upstream_test_framework Fix Description: The fix introduces several changes - Define SERVER properties with "verb" that could be used in CLI Those properties are used in 'args' dictionary to handle the instance (create/delete...) - Split the previous lib389.DirSrv:__init__, __localinit__ and __initPart2 into __init__/allocate/create/open - Implements lib389.DirSrv:delete/close/start/stop/restart/list/exists - Copy tools functions related to instance backup into instance class lib389.DirSrv clearBackupFS/checkBackupFS/backupFS/restoreFS - Add a new test module for DirSrv: dirsrv_test.py - List returns the properties retrieve from the config files (/etc/sysconfig/dirsrv-* or $HOME/.dirsrv/dirsrv-* https://fedorahosted.org/389/ticket/47625 Reviewed by: Rich Megginson Platforms tested: F17 Flag Day: no Doc impact: no --- bug_harness.py | 3 +- lib389/__init__.py | 814 +++++++++++++++++++++++++++++++++++++++++++++++---- lib389/_constants.py | 36 +++ lib389/properties.py | 56 ++++ lib389/tools.py | 37 ++- lib389/utils.py | 50 ++-- tests/dirsrv_test.py | 202 +++++++++++++ 7 files changed, 1101 insertions(+), 97 deletions(-) create mode 100644 lib389/properties.py create mode 100644 tests/dirsrv_test.py diff --git a/bug_harness.py b/bug_harness.py index 621ef84..b9143d0 100644 --- a/bug_harness.py +++ b/bug_harness.py @@ -1,6 +1,7 @@ from bug_harness import DSAdminHarness as DSAdmin from dsadmin import Entry from dsadmin.tools import DSAdminTools +from lib389.properties import * """ An harness for bug replication. @@ -94,7 +95,7 @@ class DSAdminHarness(DSAdmin, DSAdminTools): def createInstance(args): # eventually set prefix - args.setdefault('prefix', os.environ.get('PREFIX', None)) + args.setdefault(SER_DEPLOYED_DIR, os.environ.get('PREFIX', None)) args.setdefault('sroot', os.environ.get('SERVER_ROOT', None)) DSAdminTools.createInstance(args) diff --git a/lib389/__init__.py b/lib389/__init__.py index b69151f..cf14595 100644 --- a/lib389/__init__.py +++ b/lib389/__init__.py @@ -19,6 +19,7 @@ except ImportError: import sys import os +import pwd import os.path import base64 import urllib @@ -34,6 +35,8 @@ import shutil import datetime import logging import decimal +import glob +import tarfile from ldap.ldapobject import SimpleLDAPObject from ldapurl import LDAPUrl @@ -50,8 +53,13 @@ from lib389.utils import ( is_a_dn, normalizeDN, suffixfilt, - escapeDNValue + escapeDNValue, + update_newhost_with_fqdn, + formatInfData, + get_sbin_dir ) +from lib389.properties import * +from lib389.tools import DirSrvTools # mixin #from lib389.tools import DirSrvTools @@ -164,12 +172,21 @@ class DirSrv(SimpleLDAPObject): def __initPart2(self): """Initialize the DirSrv structure filling various fields, like: - - dbdir - - errlog - - confdir + self.errlog -> nsslapd-errorlog + self.accesslog -> nsslapd-accesslog + self.confdir -> nsslapd-certdir + self.inst -> equivalent to self.serverid + self.sroot/self.inst -> nsslapd-instancedir + self.dbdir -> dirname(nsslapd-directory) + + @param - self + + @return - None + + @raise ldap.LDAPError - if failure during initialization """ - if self.binddn and len(self.binddn) and not hasattr(self, 'sroot'): + if self.binddn and len(self.binddn): try: # XXX this fields are stale and not continuously updated # do they have sense? @@ -221,6 +238,16 @@ class DirSrv(SimpleLDAPObject): raise def __localinit__(self): + ''' + Establish a connection to the started instance. It binds with the binddn property, + then it initializes various fields from DirSrv (via __initPart2) + + @param - self + + @return - None + + @raise ldap.LDAPError - if failure during initialization + ''' uri = self.toLDAPURL() SimpleLDAPObject.__init__(self, uri) @@ -234,24 +261,24 @@ class DirSrv(SimpleLDAPObject): if ent: self.binddn = ent.dn else: - log.error("Error: could not find %s under %s" % ( + raise ValueError("Error: could not find %s under %s" % ( self.binddn, CFGSUFFIX)) - if not self.nobind: - needtls = False - while True: + + needtls = False + while True: + try: + if needtls: + self.start_tls_s() try: - if needtls: - self.start_tls_s() - try: - self.simple_bind_s(self.binddn, self.bindpw) - except ldap.SERVER_DOWN, e: - # TODO add server info in exception - log.debug("Cannot connect to %r" % uri) - raise e - break - except ldap.CONFIDENTIALITY_REQUIRED: - needtls = True - self.__initPart2() + self.simple_bind_s(self.binddn, self.bindpw) + except ldap.SERVER_DOWN, e: + # TODO add server info in exception + log.debug("Cannot connect to %r" % uri) + raise e + break + except ldap.CONFIDENTIALITY_REQUIRED: + needtls = True + self.__initPart2() def rebind(self): """Reconnect to the DS @@ -275,46 +302,725 @@ class DirSrv(SimpleLDAPObject): self.config = Config(self) self.index = Index(self) - def __init__(self, host='localhost', port=389, binddn='', bindpw='', serverId=None, nobind=False, sslport=0, verbose=False, offline=False): # default to anon bind - """We just set our instance variables and wrap the methods. - The real work is done in the following methods, reused during - instance creation & co. - * __localinit__ - * __initPart2 - - e.g. when using the start command, we just need to reconnect, - not create a new instance""" - log.info("Initializing %s with %s:%s" % (self.__class__, - host, sslport or port)) - self.__wrapmethods() + def __init__(self, verbose=False): + """ + This method does various initialization of DirSrv object: + parameters: + - 'state' to DIRSRV_STATE_INIT + - 'verbose' flag for debug purpose + - 'log' so that the use the module defined logger + + wrap the methods. + + - from SimpleLDAPObject + - from agreement, backends, suffix... + + It just create a DirSrv object. To use it the user will likely do + the following additional steps: + - allocate + - create + - open + """ + + self.state = DIRSRV_STATE_INIT self.verbose = verbose - self.port = port - self.sslport = sslport - self.host = host - self.binddn = binddn - self.bindpw = bindpw - self.nobind = nobind - self.isLocal = isLocalHost(host) - self.serverId = serverId - - - # - # dict caching DS structure - # - self.suffixes = {} - self.agmt = {} - if not offline: - # the real init - self.__localinit__() self.log = log - # add brookers + + self.__wrapmethods() self.__add_brookers__() - def __str__(self): """XXX and in SSL case?""" return self.host + ":" + str(self.port) + def allocate(self, args): + ''' + Initialize a DirSrv object according to the provided args. + The final state -> DIRSRV_STATE_ALLOCATED + @param args - dictionary that contains the DirSrv properties + properties are + SER_SERVERID_PROP: mandatory server id of the instance -> slapd- + SER_HOST: hostname [LOCALHOST] + SER_PORT: normal ldap port [DEFAULT_PORT] + SER_SECURE_PORT: secure ldap port + SER_ROOT_DN: root DN [DN_DM] + SER_ROOT_PW: password of root DN [PW_DM] + SER_USER_ID: user id of the create instance [DEFAULT_USER] + SER_GROUP_ID: group id of the create instance [SER_USER_ID] + SER_DEPLOYED_DIR: directory where 389-ds is deployed + SER_BACKUP_INST_DIR: directory where instances will be backup + + @return None + + @raise ValueError - if missing mandatory properties or invalid state of DirSrv + ''' + if self.state != DIRSRV_STATE_INIT and self.state != DIRSRV_STATE_ALLOCATED: + raise ValueError("invalid state for calling allocate: %s" % self.state) + + if SER_SERVERID_PROP not in args: + raise ValueError("%s is a mandatory parameter" % SER_SERVERID_PROP) + + # Settings from args of server attributes + self.host = args.get(SER_HOST, LOCALHOST) + self.port = args.get(SER_PORT, DEFAULT_PORT) + self.sslport = args.get(SER_SECURE_PORT) + self.binddn = args.get(SER_ROOT_DN, DN_DM) + self.bindpw = args.get(SER_ROOT_PW, PW_DM) + self.creation_suffix = args.get(SER_CREATION_SUFFIX, DEFAULT_SUFFIX) + self.userid = args.get(SER_USER_ID) + if not self.userid: + if os.getuid() == 0: + # as root run as default user + self.userid = DEFAULT_USER + else: + self.userid = pwd.getpwuid( os.getuid() )[ 0 ] + + + # Settings from args of server attributes + self.serverid = args.get(SER_SERVERID_PROP) + self.groupid = args.get(SER_GROUP_ID, self.userid) + self.backupdir = args.get(SER_BACKUP_INST_DIR, DEFAULT_BACKUPDIR) + self.prefix = args.get(SER_DEPLOYED_DIR, None) + + # Those variables needs to be revisited (sroot for 64 bits) + #self.sroot = os.path.join(self.prefix, "lib/dirsrv") + #self.errlog = os.path.join(self.prefix, "var/log/dirsrv/slapd-%s/errors" % self.serverid) + + # For compatibility keep self.inst but should be removed + self.inst = self.serverid + + # additional settings + self.isLocal = isLocalHost(self.host) + self.suffixes = {} + self.agmt = {} + + self.state = DIRSRV_STATE_ALLOCATED + self.log.info("Allocate %s with %s:%s" % (self.__class__, + self.host, + self.sslport or self.port)) + + + + def list(self, all=False): + """ + Returns a list of files containing the environment of the created instances + that are on the local file system (e.g. /etc/dirsrv/slapd-*). + If 'all' is True, it returns all the files else it returns only the + environment file of the current instance. + By default it is False and only returns the current instance file. + + @param all - True or False . default is [False] + + @return list file(s) name(s) + + @raise None + """ + def test_and_set(prop, propname, variable, value): + ''' + If variable is 'propname' it adds to + 'prop' dictionary the propname:value + ''' + if variable == propname: + prop[propname] = value + return 1 + return 0 + + def _parse_configfile(filename=None): + ''' + This method read 'filename' and build a dictionary with + CONF_* properties + ''' + + if not filename: + raise IOError('filename is mandatory') + if not os.path.isfile(filename) or not os.access(filename, os.R_OK): + raise IOError('invalid file name or rights: %s' % filename) + + prop = {} + file = open(filename, 'r') + for line in file: + # retrieve the value in line:: = [';' export ] + + #skip lines without assignment + if not '=' in line: + continue + value = line.split(';',1)[0] + + #skip lines without assignment + if not '=' in value: + continue + + variable = value.split('=',1)[0] + value = value.split('=',1)[1] + value = value.strip(' \t') # remove heading/ending space/tab + for property in (CONF_SERVER_DIR, + CONF_SERVERBIN_DIR, + CONF_CONFIG_DIR, + CONF_INST_DIR, + CONF_RUN_DIR, + CONF_DS_ROOT, + CONF_PRODUCT_NAME): + if test_and_set(prop, property, variable, value): + break + + return prop + + def search_dir(instances, pattern, stop_value=None): + ''' + It search all the files matching pattern. + It there is not stop_value, it adds the properties found in each file + to the 'instances' + Else it searches the specific stop_value (instance's serverid) to add only + its properties in the 'instances' + + @param instances - list of dictionary containing the instances properties + @param pattern - pattern to find the files containing the properties + @param stop_value - serverid value if we are looking only for one specific instance + + @return True or False - If stop_value is None it returns False. + If stop_value is specified, it returns True if it added + the property dictionary in instances. Or False if it did + not find it. + ''' + added = False + for instance in glob.glob(pattern): + serverid = os.path.basename(instance)[len(DEFAULT_ENV_HEAD):] + + # skip removed instance + if '.removed' in serverid: + continue + + # it is found, store its properties in the list + if stop_value: + if stop_value == serverid: + instances.append(_parse_configfile(instance)) + added = True + break + else: + #this is not the searched value, continue + continue + else: + # we are not looking for a specific value, just add it + instances.append(_parse_configfile(instance)) + + return added + + # Retrieves all instances under '/etc/sysconfig' and '/etc/dirsrv' + + # Instances/Environment are + # + # file: /etc/sysconfig/dirsrv- (env) + # inst: /etc/dirsrv/slapd- (conf) + # + # or + # + # file: $HOME/.dirsrv/dirsrv- (env) + # inst: /etc/dirsrv/slapd- (conf) + # + + prefix = self.prefix or '/' + + # first identify the directories we will scan + confdir = os.getenv('INITCONFIGDIR') + if confdir: + self.log.info("$INITCONFIGDIR set to: %s" % confdir) + if not os.path.isdir(confdir): + raise ValueError("$INITCONFIGDIR incorrect directory (%s)" % confdir) + sysconfig_head = confdir + privconfig_head = None + else: + sysconfig_head = prefix + ENV_SYSCONFIG_DIR + privconfig_head = os.path.join(os.getenv('HOME'), ENV_LOCAL_DIR) + if not os.path.isdir(sysconfig_head): + privconfig_head = None + self.log.info("dir (sys) : %s" % sysconfig_head) + if privconfig_head: + self.log.info("dir (priv): %s" % privconfig_head) + + # list of the found instances + instances = [] + + # now prepare the list of instances properties + if not all: + # easy case we just look for the current instance + + # we have two location to retrieve the self.serverid + # privconfig_head and sysconfig_head + + # first check the private repository + pattern = "%s*" % os.path.join(privconfig_head, DEFAULT_ENV_HEAD) + found = search_dir(instances, pattern, self.serverid) + if found: + assert len(instances) == 1 + else: + assert len(instances) == 0 + + # second, if not already found, search the system repository + if not found: + pattern = "%s*" % os.path.join(sysconfig_head, DEFAULT_ENV_HEAD) + search_dir(instances, pattern, self.serverid) + + else: + # all instances must be retrieved + pattern = "%s*" % os.path.join(privconfig_head, DEFAULT_ENV_HEAD) + search_dir(instances, pattern) + + pattern = "%s*" % os.path.join(sysconfig_head, DEFAULT_ENV_HEAD) + search_dir(instances, pattern) + + return instances + + + def _createDirsrv(self, verbose=0): + """Create a new instance of directory server + + @param self - containing the set properties + + SER_HOST (host) + SER_PORT (port) + SER_SECURE_PORT (sslport) + SER_ROOT_DN (binddn) + SER_ROOT_PW (bindpw) + SER_CREATION_SUFFIX (creation_suffix) + SER_USER_ID (userid) + SER_SERVERID_PROP (serverid) + SER_GROUP_ID (groupid) + SER_DEPLOYED_DIR (prefix) + SER_BACKUP_INST_DIR (backupdir) + + @return None + + @raise None + + } + """ + + prog = get_sbin_dir(None, self.prefix) + CMD_PATH_SETUP_DS + + if not os.path.isfile(prog): + log.error("Can't find file: %r, removing extension" % prog) + prog = prog[:-3] + + args = {SER_HOST: self.host, + SER_PORT: self.port, + SER_SECURE_PORT: self.sslport, + SER_ROOT_DN: self.binddn, + SER_ROOT_PW: self.bindpw, + SER_CREATION_SUFFIX: self.creation_suffix, + SER_USER_ID: self.userid, + SER_SERVERID_PROP: self.serverid, + SER_GROUP_ID: self.groupid, + SER_DEPLOYED_DIR: self.prefix, + SER_BACKUP_INST_DIR: self.backupdir} + content = formatInfData(args) + DirSrvTools.runInfProg(prog, content, verbose) + + + def create(self): + """ + Creates an instance with the parameters sets in dirsrv + The state change from DIRSRV_STATE_ALLOCATED -> DIRSRV_STATE_OFFLINE + + @param - self + + @return - None + + @raise ValueError - if it exist an instance with the same 'serverid' + """ + # check that DirSrv was in DIRSRV_STATE_ALLOCATED state + if self.state != DIRSRV_STATE_ALLOCATED: + raise ValueError("invalid state for calling create: %s" % self.state) + + # Check that the instance does not already exist + props = self.list() + if len(props) != 0: + raise ValueError("Error it already exists the instance (%s)" % props[0][CONF_INST_DIR]) + + # Time to create the instance and retrieve the effective sroot + self._createDirsrv(verbose=self.verbose) + + # Retrieve sroot from the sys/priv config file + props = self.list() + assert len(props) == 1 + self.sroot = props[0][CONF_SERVER_DIR] + + # Now the instance is created but DirSrv is not yet connected to it + self.state = DIRSRV_STATE_OFFLINE + + def delete(self): + ''' + Deletes the instance with the parameters sets in dirsrv + The state changes from DIRSRV_STATE_OFFLINE -> DIRSRV_STATE_ALLOCATED + + @param self + + @return None + + @raise None + ''' + if self.state == DIRSRV_STATE_ONLINE: + self.close() + + # Check that the instance does not already exist + props = self.list() + if len(props) != 1: + raise ValueError("Error can not find instance %s[%s:%d]" % + (self.serverid, self.host, self.port)) + + # Now time to remove the instance + prog = get_sbin_dir(None, self.prefix) + CMD_PATH_REMOVE_DS + cmd = "%s -i %s%s" % (prog, DEFAULT_INST_HEAD, self.serverid) + self.log.debug("running: %s " % cmd) + try: + os.system(cmd) + except: + log.exception("error executing %r" % cmd) + + self.state = DIRSRV_STATE_ALLOCATED + + def open(self): + ''' + It opens a ldap bound connection to dirsrv so that online administrative tasks are possible. + It binds with the binddn property, then it initializes various fields from DirSrv (via __initPart2) + + The state changes -> DIRSRV_STATE_ONLINE + + @param self + + @return None + + @raise ValueError - if the instance has not the right state or can not find the binddn to bind + ''' + # check that DirSrv was in DIRSRV_STATE_OFFLINE or DIRSRV_STATE_ONLINE state + if self.state != DIRSRV_STATE_OFFLINE and self.state != DIRSRV_STATE_ONLINE: + raise ValueError("invalid state for calling open: %s" % self.state) + + uri = self.toLDAPURL() + + SimpleLDAPObject.__init__(self, uri) + + # see if binddn is a dn or a uid that we need to lookup + if self.binddn and not is_a_dn(self.binddn): + self.simple_bind_s("", "") # anon + ent = self.getEntry(CFGSUFFIX, ldap.SCOPE_SUBTREE, + "(uid=%s)" % self.binddn, + ['uid']) + if ent: + self.binddn = ent.dn + else: + raise ValueError("Error: could not find %s under %s" % ( + self.binddn, CFGSUFFIX)) + + needtls = False + while True: + try: + if needtls: + self.start_tls_s() + try: + self.simple_bind_s(self.binddn, self.bindpw) + except ldap.SERVER_DOWN, e: + # TODO add server info in exception + log.debug("Cannot connect to %r" % uri) + raise e + break + except ldap.CONFIDENTIALITY_REQUIRED: + needtls = True + self.__initPart2() + + self.state = DIRSRV_STATE_ONLINE + + def close(self): + ''' + It closes connection to dirsrv. Online administrative tasks are no longer possible. + + The state changes from DIRSRV_STATE_ONLINE -> DIRSRV_STATE_OFFLINE + + @param self + + @return None + + @raise ValueError - if the instance has not the right state + ''' + + # check that DirSrv was in DIRSRV_STATE_ONLINE state + if self.state != DIRSRV_STATE_ONLINE: + raise ValueError("invalid state for calling close: %s" % self.state) + + SimpleLDAPObject.unbind(self) + + self.state = DIRSRV_STATE_OFFLINE + + def start(self, timeout): + ''' + It starts an instance and rebind it. Its final state after rebind (open) + is DIRSRV_STATE_ONLINE + + @param self + @param timeout (in sec) to wait for successful start + + @return None + + @raise None + ''' + # Default starting timeout (to find 'slapd started' in error log) + if not timeout: + timeout = 120 + + # called with the default timeout + if DirSrvTools.start(self, verbose=False, timeout=timeout): + self.log.error("Probable failure to start the instance") + + self.open() + + def stop(self, timeout): + ''' + It stops an instance. + It changes the state -> DIRSRV_STATE_OFFLINE + + @param self + @param timeout (in sec) to wait for successful stop + + @return None + + @raise None + ''' + + # Default starting timeout (to find 'slapd started' in error log) + if not timeout: + timeout = 120 + + # called with the default timeout + if DirSrvTools.stop(self, verbose=False, timeout=timeout): + self.log.error("Probable failure to stop the instance") + + #whatever the initial state, the instance is now Offline + self.state = DIRSRV_STATE_OFFLINE + + def restart(self, timeout): + ''' + It restarts an instance and rebind it. Its final state after rebind (open) + is DIRSRV_STATE_ONLINE. + + @param self + @param timeout (in sec) to wait for successful stop + + @return None + + @raise None + ''' + self.stop(timeout) + self.start(timeout) + + def _infoBackupFS(self): + """ + Return the information to retrieve the backup file of a given instance + It returns: + - Directory name containing the backup (e.g. /tmp/slapd-standalone.bck) + - The pattern of the backup files (e.g. /tmp/slapd-standalone.bck/backup*.tar.gz) + """ + backup_dir = "%s/slapd-%s.bck" % (self.backupdir, self.serverid) + backup_pattern = os.path.join(backup_dir, "backup*.tar.gz") + return backup_dir, backup_pattern + + def clearBackupFS(self, backup_file=None): + """ + Remove a backup_file or all backup up of a given instance + """ + if backup_file: + if os.path.isfile(backup_file): + try: + os.remove(backup_file) + except: + log.info("clearBackupFS: fail to remove %s" % backup_file) + pass + else: + backup_dir, backup_pattern = self._infoBackupFS() + list_backup_files = glob.glob(backup_pattern) + for f in list_backup_files: + try: + os.remove(f) + except: + log.info("clearBackupFS: fail to remove %s" % backup_file) + pass + + + def checkBackupFS(self): + """ + If it exits a backup file, it returns it + else it returns None + """ + + backup_dir, backup_pattern = self._infoBackupFS() + list_backup_files = glob.glob(backup_pattern) + if not list_backup_files: + return None + else: + # returns the first found backup + return list_backup_files[0] + + + def backupFS(self): + """ + Saves the files of an instance under /tmp/slapd-.bck/backup_HHMMSS.tar.gz + and return the archive file name. + If it already exists a such file, it assums it is a valid backup and + returns its name + + self.sroot : root of the instance (e.g. /usr/lib64/dirsrv) + self.inst : instance name (e.g. standalone for /etc/dirsrv/slapd-standalone) + self.confdir : root of the instance config (e.g. /etc/dirsrv) + self.dbdir: directory where is stored the database (e.g. /var/lib/dirsrv/slapd-standalone/db) + self.changelogdir: directory where is stored the changelog (e.g. /var/lib/dirsrv/slapd-master/changelogdb) + """ + + # First check it if already exists a backup file + backup_dir, backup_pattern = self._infoBackupFS() + backup_file = self.checkBackupFS() + if backup_file is None: + if not os.path.exists(backup_dir): + os.makedirs(backup_dir) + else: + return backup_file + + # goes under the directory where the DS is deployed + listFilesToBackup = [] + here = os.getcwd() + os.chdir(self.prefix) + prefix_pattern = "%s/" % self.prefix + + # build the list of directories to scan + instroot = "%s/slapd-%s" % (self.sroot, self.serverid) + ldir = [ instroot ] + if hasattr(self, 'confdir'): + ldir.append(self.confdir) + if hasattr(self, 'dbdir'): + ldir.append(self.dbdir) + if hasattr(self, 'changelogdir'): + ldir.append(self.changelogdir) + if hasattr(self, 'errlog'): + ldir.append(os.path.dirname(self.errlog)) + if hasattr(self, 'accesslog') and os.path.dirname(self.accesslog) not in ldir: + ldir.append(os.path.dirname(self.accesslog)) + + # now scan the directory list to find the files to backup + for dirToBackup in ldir: + for root, dirs, files in os.walk(dirToBackup): + for file in files: + name = os.path.join(root, file) + name = re.sub(prefix_pattern, '', name) + + if os.path.isfile(name): + listFilesToBackup.append(name) + log.debug("backupFS add = %s (%s)" % (name, self.prefix)) + + + # create the archive + name = "backup_%s.tar.gz" % (time.strftime("%m%d%Y_%H%M%S")) + backup_file = os.path.join(backup_dir, name) + tar = tarfile.open(backup_file, "w:gz") + + + for name in listFilesToBackup: + if os.path.isfile(name): + tar.add(name) + tar.close() + log.info("backupFS: archive done : %s" % backup_file) + + # return to the directory where we were + os.chdir(here) + + return backup_file + + def restoreFS(self, backup_file): + """ + """ + + # First check the archive exists + if backup_file is None: + log.warning("Unable to restore the instance (missing backup)") + return 1 + if not os.path.isfile(backup_file): + log.warning("Unable to restore the instance (%s is not a file)" % backup_file) + return 1 + + # + # Second do some clean up + # + + # previous db (it may exists new db files not in the backup) + log.debug("restoreFS: remove subtree %s/*" % self.dbdir) + for root, dirs, files in os.walk(self.dbdir): + for d in dirs: + if d not in ("bak", "ldif"): + log.debug("restoreFS: before restore remove directory %s/%s" % (root, d)) + shutil.rmtree("%s/%s" % (root, d)) + + # previous error/access logs + log.debug("restoreFS: remove error logs %s" % self.errlog) + for f in glob.glob("%s*" % self.errlog): + log.debug("restoreFS: before restore remove file %s" % (f)) + os.remove(f) + log.debug("restoreFS: remove access logs %s" % self.accesslog) + for f in glob.glob("%s*" % self.accesslog): + log.debug("restoreFS: before restore remove file %s" % (f)) + os.remove(f) + + + # Then restore from the directory where DS was deployed + here = os.getcwd() + os.chdir(self.prefix) + + tar = tarfile.open(backup_file) + for member in tar.getmembers(): + if os.path.isfile(member.name): + # + # restore only writable files + # It could be a bad idea and preferably restore all. + # Now it will be easy to enhance that function. + if os.access(member.name, os.W_OK): + log.debug("restoreFS: restored %s" % member.name) + tar.extract(member.name) + else: + log.debug("restoreFS: not restored %s (no write access)" % member.name) + else: + log.debug("restoreFS: restored %s" % member.name) + tar.extract(member.name) + + tar.close() + + # + # Now be safe, triggers a recovery at restart + # + guardian_file = os.path.join(self.dbdir, "db/guardian") + if os.path.isfile(guardian_file): + try: + log.debug("restoreFS: remove %s" % guardian_file) + os.remove(guardian_file) + except: + log.warning("restoreFS: fail to remove %s" % guardian_file) + pass + + + os.chdir(here) + + + def exists(self): + ''' + Check if an instance exists. + It checks that both exists: + - configuration directory (/etc/dirsrv/slapd-) + - environment file (/etc/sysconfig/dirsrv- or $HOME/.dirsrv/dirsrv-) + + @param None + + @return True of False if the instance exists or not + + @raise None + + ''' + props = self.list() + return len(props) == 1 + def toLDAPURL(self): """Return the uri ldap[s]://host:[ssl]port.""" if self.sslport: @@ -323,7 +1029,7 @@ class DirSrv(SimpleLDAPObject): return "ldap://%s:%d/" % (self.host, self.port) def getServerId(self): - return self.serverId + return self.serverid # # Get entries diff --git a/lib389/_constants.py b/lib389/_constants.py index b0eb23a..704d6ac 100644 --- a/lib389/_constants.py +++ b/lib389/_constants.py @@ -34,6 +34,42 @@ DEFAULT_USER = "nobody" DN_DM = "cn=Directory Manager" DN_CONFIG = "cn=config" DN_LDBM = "cn=ldbm database,cn=plugins,cn=config" + +CMD_PATH_SETUP_DS = "/setup-ds.pl" +CMD_PATH_REMOVE_DS = "/remove-ds.pl" + +# State of an DirSrv object +DIRSRV_STATE_INIT='initial' +DIRSRV_STATE_ALLOCATED='allocated' +DIRSRV_STATE_OFFLINE='offline' +DIRSRV_STATE_ONLINE='online' + +LOCALHOST = "localhost.localdomain" +DEFAULT_PORT = 389 +DEFAULT_SECURE_PORT = 636 +DEFAULT_SUFFIX = 'dc=example,dc=com' +DEFAULT_BACKUPDIR = '/tmp' +DEFAULT_INST_HEAD = 'slapd-' +DEFAULT_ENV_HEAD = 'dirsrv-' + +PW_DM = "password" + +CONF_DIR = 'etc/dirsrv' +ENV_SYSCONFIG_DIR = '/etc/sysconfig' +ENV_LOCAL_DIR = '.dirsrv' + + +# CONFIG file (/etc/sysconfig/dirsrv-* or $HOME/.dirsrv/dirsrv-*) keywords +CONF_SERVER_DIR = 'SERVER_DIR' +CONF_SERVERBIN_DIR = 'SERVERBIN_DIR' +CONF_CONFIG_DIR = 'CONFIG_DIR' +CONF_INST_DIR = 'INST_DIR' +CONF_RUN_DIR = 'RUN_DIR' +CONF_DS_ROOT = 'DS_ROOT' +CONF_PRODUCT_NAME = 'PRODUCT_NAME' + + + DN_MAPPING_TREE = "cn=mapping tree,cn=config" DN_CHAIN = "cn=chaining database,cn=plugins,cn=config" DN_CHANGELOG = "cn=changelog5,cn=config" diff --git a/lib389/properties.py b/lib389/properties.py new file mode 100644 index 0000000..32a210a --- /dev/null +++ b/lib389/properties.py @@ -0,0 +1,56 @@ +''' +Created on Dec 5, 2013 + +@author: tbordaz +''' + +# +# Properties supported by the server WITH related attribute name +# +SER_HOST='hostname' +SER_PORT='ldap-port' +SER_SECURE_PORT='ldap-secureport' +SER_ROOT_DN='root-dn' +SER_ROOT_PW='root-pw' +SER_USER_ID='user-id' + +SER_PROPNAME_TO_ATTRNAME= {SER_HOST:'nsslapd-localhost', + SER_PORT:'nsslapd-port', + SER_SECURE_PORT:'nsslapd-securePort', + SER_ROOT_DN:'nsslapd-rootdn', + SER_ROOT_PW:'nsslapd-rootpw', + SER_USER_ID:'nsslapd-localuser'} +# +# Properties supported by the server WITHOUT related attribute name +# +SER_SERVERID_PROP='server-id' +SER_GROUP_ID='group-id' +SER_DEPLOYED_DIR='deployed-dir' +SER_BACKUP_INST_DIR='inst-backupdir' +SER_CREATION_SUFFIX='suffix' + +# +# Properties supported by the replica agreement +# +RA_SCHEDULE_PROP='schedule' +RA_TRANSPORT_PROT_PROP='transport-prot' +RA_FRAC_EXCLUDE_PROP='fractional-exclude-attrs-inc' +RA_FRAC_EXCLUDE_TOTAL_UPDATE_PROP='fractional-exclude-attrs-total' +RA_FRAC_STRIP_PROP='fractional-strip-attrs' +RA_CONSUMER_PORT='consumer-port' +RA_CONSUMER_HOST='consumer-host' +RA_CONSUMER_TOTAL_INIT='consumer-total-init' +RA_TIMEOUT='timeout' + +RA_PROPNAME_TO_ATTRNAME = {RA_SCHEDULE_PROP:'nsds5replicaupdateschedule', + RA_TRANSPORT_PROT_PROP:'nsds5replicatransportinfo', + RA_FRAC_EXCLUDE_PROP:'nsDS5ReplicatedAttributeList', + RA_FRAC_EXCLUDE_TOTAL_UPDATE_PROP:'nsDS5ReplicatedAttributeListTotal', + RA_FRAC_STRIP_PROP:'nsds5ReplicaStripAttrs', + RA_CONSUMER_PORT:'nsds5replicaport', + RA_CONSUMER_HOST:'nsds5ReplicaHost', + RA_CONSUMER_TOTAL_INIT:'nsds5BeginReplicaRefresh', + RA_TIMEOUT:'nsds5replicatimeout'} + + + \ No newline at end of file diff --git a/lib389/tools.py b/lib389/tools.py index b4b741e..4b83fea 100644 --- a/lib389/tools.py +++ b/lib389/tools.py @@ -27,7 +27,8 @@ import re import glob import lib389 -from lib389 import InvalidArgumentError +from lib389 import * +from lib389.properties import * from lib389.utils import ( getcfgdsuserdn, @@ -49,8 +50,8 @@ log = logging.getLogger(__name__) # Private constants PATH_SETUP_DS_ADMIN = "/setup-ds-admin.pl" -PATH_SETUP_DS = "/setup-ds.pl" -PATH_REMOVE_DS = "/remove-ds.pl" +PATH_SETUP_DS = CMD_PATH_SETUP_DS +PATH_REMOVE_DS = CMD_PATH_REMOVE_DS PATH_ADM_CONF = "/etc/dirsrv/admin-serv/adm.conf" class DirSrvTools(object): @@ -152,11 +153,23 @@ class DirSrvTools(object): def serverCmd(self, cmd, verbose, timeout=120): """NOTE: this tries to open the log! """ + if not hasattr(self, 'sroot'): + # If the instance was previously create, retrieve 'sroot' from + # sys/priv config file (e.g /etc/sysconfig/dirsrv- + props = self.list() + if len(props) == 0: + # the instance has not yet been created, just return + return + else: + assert len(props) == 1 + self.sroot = props[0][CONF_SERVER_DIR] + instanceDir = os.path.join(self.sroot, "slapd-" + self.inst) - errLog = instanceDir + '/logs/errors' if hasattr(self, 'errlog'): errLog = self.errlog + else: + errLog = os.path.join(self.prefix, "var/log/dirsrv/slapd-%s/errors" % self.serverid) done = False started = True lastLine = "" @@ -554,7 +567,7 @@ class DirSrvTools(object): prefix = None prog = get_sbin_dir(None, prefix) + PATH_REMOVE_DS - cmd = "%s -i slapd-%s" % (prog, dirsrv.serverId) + cmd = "%s -i slapd-%s" % (prog, dirsrv.serverid) log.debug("running: %s " % cmd) try: os.system(cmd) @@ -570,16 +583,14 @@ class DirSrvTools(object): The properties set are: instance.host instance.port - instance.serverId + instance.serverid instance.inst instance.prefix instance.backup ''' - instance = lib389.DirSrv(host=args['newhost'], port=args['newport'], - serverId=args['newinstance'], offline=True) - instance.prefix = args.get('prefix', '/') - instance.backupdir = args.get('backupdir', '/tmp') - instance.inst = instance.serverId + instance = lib389.DirSrv(verbose=True) + instance.allocate(args) + return instance @staticmethod @@ -602,8 +613,8 @@ class DirSrvTools(object): If it exists it returns a DirSrv instance NOT initialized, else None ''' instance = DirSrvTools._offlineDirsrv(args) - dirname = os.path.join(instance.prefix, "etc/dirsrv/slapd-%s" % instance.serverId) - errorlog = os.path.join(instance.prefix, "var/log/dirsrv/slapd-%s/errors" % instance.serverId) + dirname = os.path.join(instance.prefix, "etc/dirsrv/slapd-%s" % instance.serverid) + errorlog = os.path.join(instance.prefix, "var/log/dirsrv/slapd-%s/errors" % instance.serverid) sroot = os.path.join(instance.prefix, "lib/dirsrv") if os.path.isdir(dirname) and \ os.path.isfile(errorlog) and \ diff --git a/lib389/utils.py b/lib389/utils.py index c7f2755..2983776 100644 --- a/lib389/utils.py +++ b/lib389/utils.py @@ -2,6 +2,8 @@ TODO put them in a module! """ +from lib389.properties import SER_PORT, SER_ROOT_PW, SER_SERVERID_PROP,\ + SER_ROOT_DN try: from subprocess import Popen as my_popen, PIPE except ImportError: @@ -30,6 +32,7 @@ import ldap import lib389 from lib389 import DN_CONFIG from lib389._constants import * +from lib389.properties import * # # Decorator @@ -187,18 +190,18 @@ def get_server_user(args): def update_newhost_with_fqdn(args): - """Replace args['newhost'] with its fqdn and returns True if local. + """Replace args[SER_HOST] with its fqdn and returns True if local. One of the arguments to createInstance is newhost. If this is specified, we need to convert it to the fqdn. If not given, we need to figure out what the fqdn of the local host is. This method sets newhost in args to the appropriate value and returns True if newhost is the localhost, False otherwise""" - if 'newhost' in args: - args['newhost'] = getfqdn(args['newhost']) - isLocal = isLocalHost(args['newhost']) + if SER_HOST in args: + args[SER_HOST] = getfqdn(args[SER_HOST]) + isLocal = isLocalHost(args[SER_HOST]) else: isLocal = True - args['newhost'] = getfqdn() + args[SER_HOST] = getfqdn() return isLocal @@ -346,9 +349,9 @@ def getadminport(cfgconn, cfgdn, args): dn = cfgdn if 'admin_domain' in args: dn = "cn=%s,ou=%s, %s" % ( - args['newhost'], args['admin_domain'], cfgdn) + args[SER_HOST], args['admin_domain'], cfgdn) filt = "(&(objectclass=nsAdminServer)(serverHostName=%s)" % args[ - 'newhost'] + SER_HOST] if 'sroot' in args: filt += "(serverRoot=%s)" % args['sroot'] filt += ")" @@ -378,7 +381,7 @@ def formatInfData(args): args = { # new instance values - newhost, newuserid, newport, newrootdn, newrootpw, newsuffix, + newhost, newuserid, newport, SER_ROOT_DN, newrootpw, newsuffix, # The following parameters require to register the new instance # in the admin server @@ -422,17 +425,10 @@ def formatInfData(args): args = args.copy() args['CFGSUFFIX'] = lib389.CFGSUFFIX - content = ( - "[General]" "\n" - "FullMachineName= %(newhost)s" "\n" - "SuiteSpotUserID= %(newuserid)s" "\n" - ) % args - - # by default, use groupname=username - if 'SuiteSpotGroup' in args: - content += """\nSuiteSpotGroup= %s\n""" % args['SuiteSpotGroup'] - else: - content += """\nSuiteSpotGroup= %(newuserid)s\n""" % args + content = ("[General]" "\n") + content += ("FullMachineName= %s\n" % args[SER_HOST]) + content += ("SuiteSpotUserID= %s\n" % args[SER_USER_ID]) + content += ("nSuiteSpotGroup= %s\n" % args[SER_GROUP_ID]) if args.get('have_admin'): content += ( @@ -442,16 +438,12 @@ def formatInfData(args): "ConfigDirectoryAdminPwd= %(cfgdspwd)s" "\n" ) % args - content += ("\n" "\n" - "[slapd]" "\n" - "ServerPort= %(newport)s" "\n" - "RootDN= %(newrootdn)s" "\n" - "RootDNPwd= %(newrootpw)s" "\n" - "ServerIdentifier= %(newinstance)s" "\n" - "Suffix= %(newsuffix)s" "\n" - ) % args - - + content += ("\n" "\n" "[slapd]" "\n") + content += ("ServerPort= %s\n" % args[SER_PORT]) + content += ("RootDN= %s\n" % args[SER_ROOT_DN]) + content += ("RootDNPwd= %s\n" % args[SER_ROOT_PW]) + content += ("ServerIdentifier= %s\n" % args[SER_SERVERID_PROP]) + content += ("Suffix= %s\n" % args[SER_CREATION_SUFFIX]) # Create admin? if args.get('setup_admin'): diff --git a/tests/dirsrv_test.py b/tests/dirsrv_test.py new file mode 100644 index 0000000..afe4e53 --- /dev/null +++ b/tests/dirsrv_test.py @@ -0,0 +1,202 @@ +''' +Created on Dec 9, 2013 + +@author: tbordaz +''' + +import os +import pwd +import ldap +from random import randint +from lib389.tools import DirSrvTools +from lib389._constants import * +from lib389.properties import * +from lib389 import DirSrv,Entry + +TEST_REPL_DN = "uid=test,%s" % DEFAULT_SUFFIX +INSTANCE_PORT = 54321 +INSTANCE_SERVERID = 'dirsrv' +#INSTANCE_PREFIX = os.environ.get('PREFIX', None) +INSTANCE_PREFIX = '/home/tbordaz/install' +INSTANCE_BACKUP = os.environ.get('BACKUPDIR', DEFAULT_BACKUPDIR) + +class Test_dirsrv(): + def _add_user(self, success=True): + try: + self.instance.add_s(Entry((TEST_REPL_DN, {'objectclass': "top person organizationalPerson inetOrgPerson".split(), + 'uid': 'test', + 'sn': 'test', + 'cn': 'test'}))) + except Exception as e: + if success: + raise + else: + self.instance.log.info('Fail to add (expected): %s' % e.args) + return + + self.instance.log.info('%s added' % TEST_REPL_DN) + + def _mod_user(self, success=True): + try: + replace = [(ldap.MOD_REPLACE, 'description', str(randint(1, 100)))] + self.instance.modify_s(TEST_REPL_DN, replace) + except Exception as e: + if success: + raise + else: + self.instance.log.info('Fail to modify (expected): %s' % e.args) + return + + self.instance.log.info('%s modified' % TEST_REPL_DN) + + def setUp(self): + pass + + + def tearDown(self): + pass + + + def test_allocate(self, verbose=False): + instance = DirSrv(verbose=verbose) + instance.log.debug("Instance allocated") + assert instance.state == DIRSRV_STATE_INIT + + # Check that SER_SERVERID_PROP is a mandatory parameter + args = {SER_HOST: LOCALHOST, + SER_PORT: INSTANCE_PORT, + SER_DEPLOYED_DIR: INSTANCE_PREFIX + } + try: + instance.allocate(args) + except Exception as e: + instance.log.info('Allocate fails (normal): %s' % e.args) + assert type(e) == ValueError + assert e.args[0].find("%s is a mandatory parameter" % SER_SERVERID_PROP) >= 0 + pass + + # Check the state + assert instance.state == DIRSRV_STATE_INIT + + # Now do a successful allocate + args[SER_SERVERID_PROP] = INSTANCE_SERVERID + instance.allocate(args) + + userid = pwd.getpwuid( os.getuid() )[ 0 ] + + + # Now verify the settings + assert instance.state == DIRSRV_STATE_ALLOCATED + assert instance.host == LOCALHOST + assert instance.port == INSTANCE_PORT + assert instance.sslport == None + assert instance.binddn == DN_DM + assert instance.bindpw == PW_DM + assert instance.creation_suffix == DEFAULT_SUFFIX + assert instance.userid == userid + assert instance.serverid == INSTANCE_SERVERID + assert instance.groupid == instance.userid + assert instance.prefix == INSTANCE_PREFIX + assert instance.backupdir == INSTANCE_BACKUP + + # Now check we can change the settings of an allocated DirSrv + args = {SER_SERVERID_PROP:INSTANCE_SERVERID, + SER_HOST: LOCALHOST, + SER_PORT: INSTANCE_PORT, + SER_DEPLOYED_DIR: INSTANCE_PREFIX, + SER_ROOT_DN: "uid=foo"} + instance.allocate(args) + assert instance.state == DIRSRV_STATE_ALLOCATED + assert instance.host == LOCALHOST + assert instance.port == INSTANCE_PORT + assert instance.sslport == None + assert instance.binddn == "uid=foo" + assert instance.bindpw == PW_DM + assert instance.creation_suffix == DEFAULT_SUFFIX + assert instance.userid == userid + assert instance.serverid == INSTANCE_SERVERID + assert instance.groupid == instance.userid + assert instance.prefix == INSTANCE_PREFIX + assert instance.backupdir == INSTANCE_BACKUP + + # OK restore back the valid parameters + args = {SER_SERVERID_PROP:INSTANCE_SERVERID, + SER_HOST: LOCALHOST, + SER_PORT: INSTANCE_PORT, + SER_DEPLOYED_DIR: INSTANCE_PREFIX} + instance.allocate(args) + assert instance.state == DIRSRV_STATE_ALLOCATED + assert instance.host == LOCALHOST + assert instance.port == INSTANCE_PORT + assert instance.sslport == None + assert instance.binddn == DN_DM + assert instance.bindpw == PW_DM + assert instance.creation_suffix == DEFAULT_SUFFIX + assert instance.userid == userid + assert instance.serverid == INSTANCE_SERVERID + assert instance.groupid == instance.userid + assert instance.prefix == INSTANCE_PREFIX + assert instance.backupdir == INSTANCE_BACKUP + + self.instance = instance + + def test_list_init(self): + ''' + Lists the instances on the file system + ''' + for properties in self.instance.list(): + self.instance.log.info("properties: %r" % properties) + + for properties in self.instance.list(all=True): + self.instance.log.info("properties (all): %r" % properties) + + def test_allocated_to_offline(self): + self.instance.create() + + + def test_offline_to_online(self): + self.instance.open() + + def test_online_to_offline(self): + self.instance.close() + + def test_offline_to_allocated(self): + self.instance.delete() + + +if __name__ == "__main__": + #import sys;sys.argv = ['', 'Test.testName'] + test = Test_dirsrv() + test.setUp() + + # Allocated the instance, except preparing the instance + test.test_allocate(False) + + #clean = True + clean = False + + if not clean: + # Do a listing of the instances + test.test_list_init() + test._add_user(success=False) + + # Create the instance + test.test_allocated_to_offline() + test._add_user(success=False) + + # bind to the instance + test.test_offline_to_online() + test._add_user(success=True) + + test._mod_user(success=True) + + #Unbind to the instance + test.test_online_to_offline() + test._mod_user(success=False) + + test.test_list_init() + else: + test.instance.state = DIRSRV_STATE_OFFLINE + test.test_offline_to_allocated() + test.tearDown() + \ No newline at end of file -- 1.7.11.7