From 4acb33b09bf2fc8a618acdd558ed002c041eaf6c Mon Sep 17 00:00:00 2001 From: "Thierry bordaz (tbordaz)" Date: Wed, 13 Nov 2013 16:41:49 +0100 Subject: [PATCH] Ticket 47590: CI tests: add/split functions around replication Bug Description: Functions to setup replication are a bit too complex. They do a lot of things that should be split in several simpler functions. The parameters are also complex with mandatory/optional. Fix Description: The fix implements: - delSchema(attr, val) - getSchemaCSN(instance) - New class Agreement (in brooker) - status - schedule - create - init - reorganise routine from lib389 and brooker.replica to brooker.Agreement (init, status, create, schedule) - Creation of a default replica Mgr moved from lib389 to brooker.replica.create_repl_manager - Add a fix into instancebackupFS to backup changelog directory https://fedorahosted.org/389/ticket/47590? Reviewed by: Roberto Polli Flag Day: no Doc impact: no --- bug_harness.py | 2 +- lib389/__init__.py | 127 +++++++++++- lib389/_constants.py | 16 ++ lib389/brooker.py | 561 +++++++++++++++++++++++++++++++------------------- lib389/tools.py | 15 +- tests/dsadmin_test.py | 2 +- tests/replica_test.py | 2 +- 7 files changed, 500 insertions(+), 225 deletions(-) diff --git a/bug_harness.py b/bug_harness.py index 1d523bc..621ef84 100644 --- a/bug_harness.py +++ b/bug_harness.py @@ -39,7 +39,7 @@ class DSAdminHarness(DSAdmin, DSAdminTools): args.setdefault('binddn', REPLBINDDN) args.setdefault('bindpw', REPLBINDPW) - return DSAdmin.setupAgreement(self, repoth, args) + return DSAdmin.createAgreement(self, repoth, args) def setupReplica(self, args): """Set default replia credentials """ diff --git a/lib389/__init__.py b/lib389/__init__.py index 2f980ff..d034d07 100644 --- a/lib389/__init__.py +++ b/lib389/__init__.py @@ -33,6 +33,7 @@ import operator import shutil import datetime import logging +import decimal from ldap.ldapobject import SimpleLDAPObject from ldapurl import LDAPUrl @@ -261,10 +262,12 @@ class DirSrv(SimpleLDAPObject): def __add_brookers__(self): from lib389.brooker import ( + Agreement, Replica, Backend, Config, Index) + self.agreement = Agreement(self) self.replica = Replica(self) self.backend = Backend(self) self.config = Config(self) @@ -886,16 +889,21 @@ class DirSrv(SimpleLDAPObject): def addSchema(self, attr, val): dn = "cn=schema" self.modify_s(dn, [(ldap.MOD_ADD, attr, val)]) + + def delSchema(self, attr, val): + dn = "cn=schema" + self.modify_s(dn, [(ldap.MOD_DELETE, attr, val)]) def addAttr(self, *attributes): return self.addSchema('attributeTypes', attributes) def addObjClass(self, *objectclasses): return self.addSchema('objectClasses', objectclasses) - - - - + + def getSchemaCSN(self): + ents = self.search_s("cn=schema", ldap.SCOPE_BASE, "objectclass=*", ['nsSchemaCSN']) + ent = ents[0] + return ent.getValue('nsSchemaCSN') def setupChainingIntermediate(self): confdn = ','.join(("cn=config", DN_CHAIN)) @@ -1044,7 +1052,7 @@ class DirSrv(SimpleLDAPObject): # args - DirSrv consumer (repoth), suffix, binddn, bindpw, timeout # also need an auto_init argument - def setupAgreement(self, consumer, args, cn_format=r'meTo_%s:%s', description_format=r'me to %s:%s'): + def createAgreement(self, consumer, args, cn_format=r'meTo_%s:%s', description_format=r'me to %s:%s'): """Create (and return) a replication agreement from self to consumer. - self is the supplier, - consumer is a DirSrv object (consumer can be a master) @@ -1056,9 +1064,7 @@ class DirSrv(SimpleLDAPObject): args = { 'suffix': "dc=example,dc=com", - 'bename': "userRoot", 'binddn': "cn=replrepl,cn=config", - 'bindcn': "replrepl", # so I need it? 'bindpw': "replrepl", 'bindmethod': 'simple', 'log' : True. @@ -1070,11 +1076,46 @@ class DirSrv(SimpleLDAPObject): 'o=suffix2': 'ldap://consumer.example.net:3890' } """ - assert args.get('binddn') and args.get('bindpw') suffix = args['suffix'] + if not suffix: + # This is a mandatory parameter of the command... it fails + log.warning("createAgreement: suffix is missing") + return None + + # get the RA binddn binddn = args.get('binddn') + if not binddn: + binddn = defaultProperties.get(REPLICATION_BIND_DN, None) + if not binddn: + # weird, internal error we do not retrieve the default replication bind DN + # this replica agreement will fail to update the consumer until the + # property will be set + log.warning("createAgreement: binddn not provided and default value unavailable") + pass + + + # get the RA binddn password bindpw = args.get('bindpw') - + if not bindpw: + bindpw = defaultProperties.get(REPLICATION_BIND_PW, None) + if not bindpw: + # weird, internal error we do not retrieve the default replication bind DN password + # this replica agreement will fail to update the consumer until the + # property will be set + log.warning("createAgreement: bindpw not provided and default value unavailable") + pass + + # get the RA bind method + bindmethod = args.get('bindmethod') + if not bindmethod: + bindmethod = defaultProperties.get(REPLICATION_BIND_METHOD, None) + if not bindmethod: + # weird, internal error we do not retrieve the default replication bind method + # this replica agreement will fail to update the consumer until the + # property will be set + log.warning("createAgreement: bindmethod not provided and default value unavailable") + pass + nsuffix = normalizeDN(suffix) othhost, othport, othsslport = ( consumer.host, consumer.port, consumer.sslport) @@ -1092,6 +1133,7 @@ class DirSrv(SimpleLDAPObject): 'dn': replent.dn, 'type': int(replent.nsds5replicatype) } + # define agreement entry cn = cn_format % (othhost, othport) dn_agreement = "cn=%s,%s" % (cn, self.suffixes[nsuffix]['dn']) @@ -1116,7 +1158,7 @@ class DirSrv(SimpleLDAPObject): 'nsds5replicatimeout': str(args.get('timeout', 120)), 'nsds5replicabinddn': binddn, 'nsds5replicacredentials': bindpw, - 'nsds5replicabindmethod': args.get('bindmethod', 'simple'), + 'nsds5replicabindmethod': bindmethod, 'nsds5replicaroot': nsuffix, 'nsds5replicaupdateschedule': '0000-2359 0123456', 'description': description_format % (othhost, othport) @@ -1201,6 +1243,71 @@ class DirSrv(SimpleLDAPObject): def startReplication(self, agmtdn): return self.replica.start_and_wait(agmtdn) + def enableReplication(self, suffix=None, role=None, replicaId=CONSUMER_REPLICAID, binddn=None): + if not suffix: + log.fatal("enableReplication: suffix not specified") + return 1 + + if not role: + log.fatal("enableReplication: replica role not specify (REPLICAROLE_*)") + return 1 + + # + # Check the validity of the parameters + # + + # First role and replicaID + if role == REPLICAROLE_MASTER: + # master + replica_type = REPLICA_RDWR_TYPE + else: + # hub or consumer + replica_type = REPLICA_RDONLY_TYPE + + if replica_type == REPLICA_RDWR_TYPE: + # check the replicaId [1..CONSUMER_REPLICAID[ + if not decimal.Decimal(replicaId) or (replicaId <= 0) or (replicaId >=CONSUMER_REPLICAID): + log.fatal("enableReplication: invalid replicaId (%s) for a RW replica" % replicaId) + return 1 + elif replicaId != CONSUMER_REPLICAID: + # check the replicaId is CONSUMER_REPLICAID + log.fatal("enableReplication: invalid replicaId (%s) for a Read replica (expected %d)" % (replicaId, CONSUMER_REPLICAID)) + return 1 + + # Now check we have a suffix + entries_backend = self.getBackendsForSuffix(suffix, ['nsslapd-suffix']) + if not entries_backend: + log.fatal("enableReplication: enable to retrieve the backend for %s" % suffix) + return 1 + + ent = entries_backend[0] + if normalizeDN(suffix) != normalizeDN(ent.getValue('nsslapd-suffix')): + log.warning("enableReplication: suffix (%s) and backend suffix (%s) differs" % (suffix, entries_backend[0].nsslapd-suffix)) + pass + + # Now prepare the bindDN property + if not binddn: + binddn = defaultProperties.get(REPLICATION_BIND_DN, None) + if not binddn: + # weird, internal error we do not retrieve the default replication bind DN + # this replica will not be updatable through replication until the binddn + # property will be set + log.warning("enableReplication: binddn not provided and default value unavailable") + pass + + # Now do the effectif job + # First add the changelog if master/hub + if (role == REPLICAROLE_MASTER) or (role == REPLICAROLE_HUB): + self.replica.changelog() + + # Second create the default replica manager entry if it does not exist + # it should not be called from here but for the moment I am unsure when to create it elsewhere + self.replica.create_repl_manager() + + # then enable replication + ret = self.replica.add(suffix=suffix, binddn=binddn, rtype=replica_type, rid=replicaId) + + return ret def replicaSetupAll(self, repArgs): """setup everything needed to enable replication for a given suffix. diff --git a/lib389/_constants.py b/lib389/_constants.py index 3103587..b0eb23a 100644 --- a/lib389/_constants.py +++ b/lib389/_constants.py @@ -5,11 +5,27 @@ (MASTER_TYPE, HUB_TYPE, LEAF_TYPE) = range(3) + +REPLICAROLE_MASTER = "master" +REPLICAROLE_HUB = "hub" +REPLICAROLE_CONSUMER = "consumer" + +CONSUMER_REPLICAID = 65535 REPLICA_RDONLY_TYPE = 2 # CONSUMER and HUB REPLICA_WRONLY_TYPE = 1 # SINGLE and MULTI MASTER REPLICA_RDWR_TYPE = REPLICA_RDONLY_TYPE | REPLICA_WRONLY_TYPE +REPLICATION_BIND_DN = 'replication_bind_dn' +REPLICATION_BIND_PW = 'replication_bind_pw' +REPLICATION_BIND_METHOD = 'replication_bind_method' + +defaultProperties = { + REPLICATION_BIND_DN: "cn=replrepl,cn=config", + REPLICATION_BIND_PW: "password", + REPLICATION_BIND_METHOD: "simple" +} + CFGSUFFIX = "o=NetscapeRoot" DEFAULT_USER = "nobody" diff --git a/lib389/brooker.py b/lib389/brooker.py index b3a1b6b..c681a29 100644 --- a/lib389/brooker.py +++ b/lib389/brooker.py @@ -14,32 +14,321 @@ import time from lib389._constants import * -from lib389 import Entry, DirSrv +from lib389 import Entry, DirSrv, InvalidArgumentError from lib389.utils import normalizeDN, escapeDNValue, suffixfilt from lib389 import ( NoSuchEntryError ) -from lib389._constants import ( - DN_CHANGELOG, - DN_MAPPING_TREE, - DN_CHAIN, DN_LDBM, - MASTER_TYPE, - HUB_TYPE, - LEAF_TYPE, - REPLICA_RDONLY_TYPE, - REPLICA_RDWR_TYPE -) -from lib389._replication import RUV, CSN +from lib389._replication import RUV from lib389._entry import FormatDict +class Agreement(object): + ALWAYS = None + + proxied_methods = 'search_s getEntry'.split() + + def __init__(self, conn): + """@param conn - a DirSrv instance""" + self.conn = conn + self.log = conn.log + + def __getattr__(self, name): + if name in Agreement.proxied_methods: + return DirSrv.__getattr__(self.conn, name) + + + def status(self, agreement_dn): + """Return a formatted string with the replica status. + @param agreement_dn - + """ + + attrlist = ['cn', 'nsds5BeginReplicaRefresh', 'nsds5replicaUpdateInProgress', + 'nsds5ReplicaLastInitStatus', 'nsds5ReplicaLastInitStart', + 'nsds5ReplicaLastInitEnd', 'nsds5replicaReapActive', + 'nsds5replicaLastUpdateStart', 'nsds5replicaLastUpdateEnd', + 'nsds5replicaChangesSentSinceStartup', 'nsds5replicaLastUpdateStatus', + 'nsds5replicaChangesSkippedSinceStartup', 'nsds5ReplicaHost', + 'nsds5ReplicaPort'] + try: + ent = self.conn.getEntry( + agreement_dn, ldap.SCOPE_BASE, "(objectclass=*)", attrlist) + except NoSuchEntryError: + raise NoSuchEntryError( + "Error reading status from agreement", agreement_dn) + else: + retstr = ( + "Status for %(cn)s agmt %(nsDS5ReplicaHost)s:%(nsDS5ReplicaPort)s" "\n" + "Update in progress: %(nsds5replicaUpdateInProgress)s" "\n" + "Last Update Start: %(nsds5replicaLastUpdateStart)s" "\n" + "Last Update End: %(nsds5replicaLastUpdateEnd)s" "\n" + "Num. Changes Sent: %(nsds5replicaChangesSentSinceStartup)s" "\n" + "Num. changes Skipped: %(nsds5replicaChangesSkippedSinceStartup)s" "\n" + "Last update Status: %(nsds5replicaLastUpdateStatus)s" "\n" + "Init in progress: %(nsds5BeginReplicaRefresh)s" "\n" + "Last Init Start: %(nsds5ReplicaLastInitStart)s" "\n" + "Last Init End: %(nsds5ReplicaLastInitEnd)s" "\n" + "Last Init Status: %(nsds5ReplicaLastInitStatus)s" "\n" + "Reap Active: %(nsds5ReplicaReapActive)s" "\n" + ) + # FormatDict manages missing fields in string formatting + return retstr % FormatDict(ent.data) + + def schedule(self, agmtdn, interval='start'): + """Schedule the replication agreement + @param agmtdn - DN of the replica agreement + @param interval - in the form + - 'ALWAYS' + - 'NEVER' + - or 'HHMM-HHMM D+' With D=[0123456]+ + @raise InvalidArgumentError - if interval is not valid + """ + + # check the validity of the interval + if str(interval).lower() == 'start': + interval = '0000-2359 0123456' + elif str(interval).lower == 'never': + interval = '2358-2359 0' + else: + c = re.compile(re.compile('^([0-9][0-9])([0-9][0-9])-([0-9][0-9])([0-9][0-9]) ([0-6]+)$')) + if not c.match(interval): + raise InvalidArgumentError + + + + # Check if the replica agreement exists + try: + self.conn.getEntry(agmtdn, ldap.SCOPE_BASE) + except ldap.NO_SUCH_OBJECT: + raise + + # update it + self.log.info("Schedule replication agreement %s" % agmtdn) + mod = [( + ldap.MOD_REPLACE, 'nsds5replicaupdateschedule', [interval])] + self.conn.modify_s(agmtdn, mod) + + def create(self, consumer, suffix=None, binddn=None, bindpw=None, cn_format=r'meTo_$host:$port', description_format=r'me to $host:$port', timeout=120, auto_init=False, bindmethod='simple', starttls=False, schedule=ALWAYS, args=None): + """Create (and return) a replication agreement from self to consumer. + - self is the supplier, + + @param consumer: one of the following (consumer can be a master) + * a DirSrv object if chaining + * an object with attributes: host, port, sslport, __str__ + @param suffix - eg. 'dc=babel,dc=it' + @param binddn - + @param bindpw - + @param cn_format - string.Template to format the agreement name + @param timeout - replica timeout in seconds + @param auto_init - start replication immediately + @param bindmethod- 'simple' + @param starttls - True or False + @param schedule - when to schedule the replication. default: ALWAYS + @param args - further args dict. Allowed keys: + 'fractional', + 'stripattrs', + 'winsync' + + @return dn_agreement - DN of the created agreement + + @raise InvalidArgumentError - If the suffix is missing + @raise NosuchEntryError - if a replica doesn't exist for that suffix + @raise ALREADY_EXISTS - If the replica agreement already exists + @raise UNWILLING_TO_PERFORM if the database was previously + in read-only state. To create new agreements you + need to *restart* the directory server + + NOTE: this method doesn't cache connection entries + + TODO: test winsync + TODO: test chain + + """ + import string + + # Check we have a suffix [ mandatory ] + if not suffix: + self.log.warning("create: suffix is missing") + raise InvalidArgumentError('suffix is mandatory') + + # Check we have a bindDN [optional] can be set later + try: + binddn = binddn or defaultProperties[REPLICATION_BIND_DN] + if not binddn: + raise KeyError + except KeyError: + self.log.warning("create: bind DN not specified") + pass + + # Check we have a bindDN [optional] can be set later + try: + bindpw = bindpw or defaultProperties[REPLICATION_BIND_PW] + if not bindpw: + raise KeyError + except KeyError: + self.log.warning("create: %s password not specified" % (binddn)) + pass + + # Set the connection info HOST:PORT + othhost, othport, othsslport = ( + consumer.host, consumer.port, consumer.sslport) + othport = othsslport or othport + + # Compute the normalized suffix to be set in RA entry + nsuffix = normalizeDN(suffix) + + # adding agreement under the replica entry + replica_entries = self.conn.replica.list(suffix) + if not replica_entries: + raise NoSuchEntryError( + "Error: no replica set up for suffix " + suffix) + replica = replica_entries[0] + + # define agreement entry + cn = string.Template(cn_format).substitute({'host': othhost, 'port': othport}) + dn_agreement = ','.join(["cn=%s" % cn, replica.dn]) + + # This is probably unnecessary because + # we can just raise ALREADY_EXISTS + try: + + entry = self.conn.getEntry(dn_agreement, ldap.SCOPE_BASE) + self.log.warn("Agreement exists: %r" % dn_agreement) + raise ldap.ALREADY_EXISTS + except ldap.NO_SUCH_OBJECT: + entry = None + + # In a separate function in this scope? + entry = Entry(dn_agreement) + entry.update({ + 'objectclass': ["top", "nsds5replicationagreement"], + 'cn': cn, + 'nsds5replicahost': consumer.host, + 'nsds5replicatimeout': str(timeout), + 'nsds5replicabinddn': binddn, + 'nsds5replicacredentials': bindpw, + 'nsds5replicabindmethod': bindmethod, + 'nsds5replicaroot': nsuffix, + 'description': string.Template(description_format).substitute({'host': othhost, 'port': othport}) + }) + if schedule: + if not re.match(r'\d{4}-\d{4} [0-6]{1,7}', schedule): # TODO put the regexp in a separate variable + raise ValueError("Bad schedule format %r" % schedule) + entry.update({'nsds5replicaupdateschedule': schedule}) + if starttls: + entry.setValues('nsds5replicatransportinfo', 'TLS') + entry.setValues('nsds5replicaport', str(othport)) + elif othsslport: + entry.setValues('nsds5replicatransportinfo', 'SSL') + entry.setValues('nsds5replicaport', str(othsslport)) + else: + entry.setValues('nsds5replicatransportinfo', 'LDAP') + entry.setValues('nsds5replicaport', str(othport)) + + if auto_init: + entry.setValues('nsds5BeginReplicaRefresh', 'start') + + # further arguments + args = args or {} + if 'fractional' in args: + entry.setValues('nsDS5ReplicatedAttributeList', args['fractional']) + if 'stripattrs' in args: + entry.setValues('nsds5ReplicaStripAttrs', args['stripattrs']) + if 'winsync' in args: # state it clearly! + self.conn.setupWinSyncAgmt(args, entry) + + try: + self.log.debug("Adding replica agreement: [%s]" % entry) + self.conn.add_s(entry) + except: + # FIXME check please! + raise + + entry = self.conn.waitForEntry(dn_agreement) + if entry: + # More verbose but shows what's going on + if 'chain' in args: + chain_args = { + 'suffix': suffix, + 'binddn': binddn, + 'bindpw': bindpw + } + # Work on `self` aka producer + if replica.nsds5replicatype == MASTER_TYPE: + self.setupChainingFarm(**chain_args) + # Work on `consumer` + # TODO - is it really required? + if replica.nsds5replicatype == LEAF_TYPE: + chain_args.update({ + 'isIntermediate': 0, + 'urls': self.conn.toLDAPURL(), + 'args': args['chainargs'] + }) + consumer.setupConsumerChainOnUpdate(**chain_args) + elif replica.nsds5replicatype == HUB_TYPE: + chain_args.update({ + 'isIntermediate': 1, + 'urls': self.conn.toLDAPURL(), + 'args': args['chainargs'] + }) + consumer.setupConsumerChainOnUpdate(**chain_args) + + return dn_agreement + + def init(self, suffix=None, consumer_host=None, consumer_port=None): + """Trigger a total update of the consumer replica + - self is the supplier, + - consumer is a DirSrv object (consumer can be a master) + - cn_format - use this string to format the agreement name + @param - suffix is the suffix targeted by the total update [mandatory] + @param - consumer_host hostname of the consumer [mandatory] + @param - consumer_port port of the consumer [mandatory] + @raise InvalidArgument: if missing mandatory argurment (suffix/host/port) + """ + # + # check the required parameters are set + # + if not suffix: + self.log.fatal("initAgreement: suffix is missing") + raise InvalidArgumentError('suffix is mandatory argument') + + nsuffix = normalizeDN(suffix) + + if not consumer_host: + self.log.fatal("initAgreement: host is missing") + raise InvalidArgumentError('host is mandatory argument') + + if not consumer_port: + self.log.fatal("initAgreement: port is missing") + raise InvalidArgumentError('port is mandatory argument') + # + # check the replica agreement already exist + # + replica_entries = self.conn.replica.list(suffix) + if not replica_entries: + raise NoSuchEntryError( + "Error: no replica set up for suffix " + suffix) + replica_entry = replica_entries[0] + self.log.debug("initAgreement: looking for replica agreements under %s" % replica_entry.dn) + try: + filt = "(&(objectclass=nsds5replicationagreement)(nsds5replicahost=%s)(nsds5replicaport=%d)(nsds5replicaroot=%s))" % (consumer_host, consumer_port, nsuffix) + entry = self.conn.getEntry(replica_entry.dn, ldap.SCOPE_ONELEVEL, filt) + except ldap.NO_SUCH_OBJECT: + self.log.fatal("initAgreement: No replica agreement to %s:%d for suffix %s" % (consumer_host, consumer_port, nsuffix)) + raise + + # + # trigger the total init + # + self.log.info("Starting total init %s" % entry.dn) + mod = [(ldap.MOD_ADD, 'nsds5BeginReplicaRefresh', 'start')] + self.conn.modify_s(entry.dn, mod) + + class Replica(object): proxied_methods = 'search_s getEntry'.split() - STOP = '2358-2359 0' - START = '0000-2359 0123456' - ALWAYS = None def __init__(self, conn): """@param conn - a DirSrv instance""" @@ -54,6 +343,49 @@ class Replica(object): """Return the replica dn of the given suffix.""" mtent = self.conn.getMTEntry(suffix) return ','.join(("cn=replica", mtent.dn)) + + def create_repl_manager(self, repl_manager_dn=None, repl_manager_pw=None): + ''' + Create an entry that will be used to bind as replica manager. + + @param repl_manager_dn - DN of the bind entry. If not provided use the default one + @param repl_manager_pw - Password of the entry. If not provide use the default one + + @raise - KeyError if can not find valid values of Bind DN and Pwd + ''' + + # check the DN and PW + try: + repl_manager_dn = repl_manager_dn or defaultProperties[REPLICATION_BIND_DN] + repl_manager_pw = repl_manager_pw or defaultProperties[REPLICATION_BIND_PW] + if not repl_manager_dn or not repl_manager_pw: + raise KeyError + except KeyError: + if not repl_manager_pw: + self.log.warning("replica_createReplMgr: bind DN password not specified") + if not repl_manager_dn: + self.log.warning("replica_createReplMgr: bind DN not specified") + raise + + # if the replication manager entry already exists, ust return + try: + entries = self.search_s(repl_manager_dn, ldap.SCOPE_BASE, "objectclass=*") + if entries: + #it already exist, fine + return + except ldap.NO_SUCH_OBJECT: + pass + + # ok it does not exist, create it + try: + attrs = { + 'nsIdleTimeout': '0', + 'passwordExpirationTime': '20381010000000Z' + } + self.conn.setupBindDN(repl_manager_dn, repl_manager_pw, attrs) + except ldap.ALREADY_EXISTS: + self.log.warn("User already exists (weird we just checked: %s " % repl_manager_dn) + def changelog(self, dbname='changelogdb'): """Add and return the replication changelog entry. @@ -155,26 +487,6 @@ class Replica(object): mod = [(ldap.MOD_ADD, 'nsds5BeginReplicaRefresh', 'start')] self.conn.modify_s(agmtdn, mod) - def stop(self, agmtdn): - """Stop replication. - @param agmtdn - agreement dn - """ - self.log.info("Stopping replication %s" % agmtdn) - mod = [( - ldap.MOD_REPLACE, 'nsds5replicaupdateschedule', [Replica.STOP])] - self.conn.modify_s(agmtdn, mod) - - def restart(self, agmtdn, schedule=START): - """Schedules a new replication. - @param agmtdn - - @param schedule - default START - `schedule` allows to customize the replication instant. - see 389 documentation for further info - """ - self.log.info("Restarting replication %s" % agmtdn) - mod = [(ldap.MOD_REPLACE, 'nsds5replicaupdateschedule', [ - schedule])] - self.modify_s(agmtdn, mod) def keep_in_sync(self, agmtdn): """ @@ -183,50 +495,16 @@ class Replica(object): self.log.info("Setting agreement for continuous replication") raise NotImplementedError("Check nsds5replicaupdateschedule before writing!") - def status(self, agreement_dn): - """Return a formatted string with the replica status. - @param agreement_dn - - """ - attrlist = ['cn', 'nsds5BeginReplicaRefresh', 'nsds5replicaUpdateInProgress', - 'nsds5ReplicaLastInitStatus', 'nsds5ReplicaLastInitStart', - 'nsds5ReplicaLastInitEnd', 'nsds5replicaReapActive', - 'nsds5replicaLastUpdateStart', 'nsds5replicaLastUpdateEnd', - 'nsds5replicaChangesSentSinceStartup', 'nsds5replicaLastUpdateStatus', - 'nsds5replicaChangesSkippedSinceStartup', 'nsds5ReplicaHost', - 'nsds5ReplicaPort'] - try: - ent = self.conn.getEntry( - agreement_dn, ldap.SCOPE_BASE, "(objectclass=*)", attrlist) - except NoSuchEntryError: - raise NoSuchEntryError( - "Error reading status from agreement", agreement_dn) - else: - retstr = ( - "Status for %(cn)s agmt %(nsDS5ReplicaHost)s:%(nsDS5ReplicaPort)s" "\n" - "Update in progress: %(nsds5replicaUpdateInProgress)s" "\n" - "Last Update Start: %(nsds5replicaLastUpdateStart)s" "\n" - "Last Update End: %(nsds5replicaLastUpdateEnd)s" "\n" - "Num. Changes Sent: %(nsds5replicaChangesSentSinceStartup)s" "\n" - "Num. changes Skipped: %(nsds5replicaChangesSkippedSinceStartup)s" "\n" - "Last update Status: %(nsds5replicaLastUpdateStatus)s" "\n" - "Init in progress: %(nsds5BeginReplicaRefresh)s" "\n" - "Last Init Start: %(nsds5ReplicaLastInitStart)s" "\n" - "Last Init End: %(nsds5ReplicaLastInitEnd)s" "\n" - "Last Init Status: %(nsds5ReplicaLastInitStatus)s" "\n" - "Reap Active: %(nsds5ReplicaReapActive)s" "\n" - ) - # FormatDict manages missing fields in string formatting - return retstr % FormatDict(ent.data) - def add(self, suffix, binddn, bindpw, rtype=MASTER_TYPE, rid=None, tombstone_purgedelay=None, purgedelay=None, referrals=None, legacy=False): + def add(self, suffix, binddn, bindpw=None, rtype=REPLICA_RDONLY_TYPE, rid=None, tombstone_purgedelay=None, purgedelay=None, referrals=None, legacy=False): """Setup a replica entry on an existing suffix. @param suffix - dn of suffix @param binddn - the replication bind dn for this replica can also be a list ["cn=r1,cn=config","cn=r2,cn=config"] @param bindpw - used to eventually provision the replication entry - @param rtype - master, hub, leaf (see above for values) - default is master + @param rtype - REPLICA_RDWR_TYPE (master) or REPLICA_RDONLY_TYPE (hub/consumer) @param rid - replica id or - if not given - an internal sequence number will be assigned # further args @@ -244,10 +522,6 @@ class Replica(object): TODO: this method does not update replica type """ # set default values - if rtype == MASTER_TYPE: - rtype = REPLICA_RDWR_TYPE - else: - rtype = REPLICA_RDONLY_TYPE if legacy: legacy = 'on' @@ -351,143 +625,10 @@ class Replica(object): return [ent.dn for ent in ents] return ents - def agreement_add(self, consumer, suffix=None, binddn=None, bindpw=None, cn_format=r'meTo_$host:$port', description_format=r'me to $host:$port', timeout=120, auto_init=False, bindmethod='simple', starttls=False, schedule=ALWAYS, args=None): - """Create (and return) a replication agreement from self to consumer. - - self is the supplier, - @param consumer: one of the following (consumer can be a master) - * a DirSrv object if chaining - * an object with attributes: host, port, sslport, __str__ - @param suffix - eg. 'dc=babel,dc=it' - @param binddn - - @param bindpw - - @param cn_format - string.Template to format the agreement name - @param timeout - replica timeout in seconds - @param auto_init - start replication immediately - @param bindmethod- 'simple' - @param starttls - True or False - @param schedule - when to schedule the replication. default: ALWAYS - @param args - further args dict. Allowed keys: - 'fractional', - 'stripattrs', - 'winsync' - - @raise NosuchEntryError - if a replica doesn't exist for that suffix - @raise ALREADY_EXISTS - @raise UNWILLING_TO_PERFORM if the database was previously - in read-only state. To create new agreements you - need to *restart* the directory server - - NOTE: this method doesn't cache connection entries - - TODO: test winsync - TODO: test chain - - """ - import string - assert binddn and bindpw and suffix - args = args or {} - - othhost, othport, othsslport = ( - consumer.host, consumer.port, consumer.sslport) - othport = othsslport or othport - nsuffix = normalizeDN(suffix) - - # adding agreement to previously created replica - replica_entries = self.list(suffix) - if not replica_entries: - raise NoSuchEntryError( - "Error: no replica set up for suffix " + suffix) - replica = replica_entries[0] - - # define agreement entry - cn = string.Template(cn_format).substitute({'host': othhost, 'port': othport}) - dn_agreement = ','.join(["cn=%s" % cn, replica.dn]) - - # This is probably unnecessary because - # we can just raise ALREADY_EXISTS - try: - entry = self.conn.getEntry(dn_agreement, ldap.SCOPE_BASE) - self.log.warn("Agreement exists: %r" % dn_agreement) - raise ldap.ALREADY_EXISTS - except ldap.NO_SUCH_OBJECT: - entry = None - - # In a separate function in this scope? - entry = Entry(dn_agreement) - entry.update({ - 'objectclass': ["top", "nsds5replicationagreement"], - 'cn': cn, - 'nsds5replicahost': consumer.host, - 'nsds5replicatimeout': str(timeout), - 'nsds5replicabinddn': binddn, - 'nsds5replicacredentials': bindpw, - 'nsds5replicabindmethod': bindmethod, - 'nsds5replicaroot': nsuffix, - 'description': string.Template(description_format).substitute({'host': othhost, 'port': othport}) - }) - if schedule: - if not re.match(r'\d{4}-\d{4} [0-6]{1,7}', schedule): # TODO put the regexp in a separate variable - raise ValueError("Bad schedule format %r" % schedule) - entry.update({'nsds5replicaupdateschedule': schedule}) - if starttls: - entry.setValues('nsds5replicatransportinfo', 'TLS') - entry.setValues('nsds5replicaport', str(othport)) - elif othsslport: - entry.setValues('nsds5replicatransportinfo', 'SSL') - entry.setValues('nsds5replicaport', str(othsslport)) - else: - entry.setValues('nsds5replicatransportinfo', 'LDAP') - entry.setValues('nsds5replicaport', str(othport)) - - if auto_init: - entry.setValues('nsds5BeginReplicaRefresh', 'start') - - # further arguments - if 'fractional' in args: - entry.setValues('nsDS5ReplicatedAttributeList', args['fractional']) - if 'stripattrs' in args: - entry.setValues('nsds5ReplicaStripAttrs', args['stripattrs']) - if 'winsync' in args: # state it clearly! - self.conn.setupWinSyncAgmt(args, entry) - - try: - self.log.debug("Adding replica agreement: [%s]" % entry) - self.conn.add_s(entry) - except: - # FIXME check please! - raise - - entry = self.conn.waitForEntry(dn_agreement) - if entry: - # More verbose but shows what's going on - if 'chain' in args: - chain_args = { - 'suffix': suffix, - 'binddn': binddn, - 'bindpw': bindpw - } - # Work on `self` aka producer - if replica.nsds5replicatype == MASTER_TYPE: - self.setupChainingFarm(**chain_args) - # Work on `consumer` - # TODO - is it really required? - if replica.nsds5replicatype == LEAF_TYPE: - chain_args.update({ - 'isIntermediate': 0, - 'urls': self.conn.toLDAPURL(), - 'args': args['chainargs'] - }) - consumer.setupConsumerChainOnUpdate(**chain_args) - elif replica.nsds5replicatype == HUB_TYPE: - chain_args.update({ - 'isIntermediate': 1, - 'urls': self.conn.toLDAPURL(), - 'args': args['chainargs'] - }) - consumer.setupConsumerChainOnUpdate(**chain_args) + + - return dn_agreement def agreement_changes(self, agmtdn): """Return a list of changes sent by this agreement.""" diff --git a/lib389/tools.py b/lib389/tools.py index b2f4cba..3fef1e4 100644 --- a/lib389/tools.py +++ b/lib389/tools.py @@ -322,7 +322,8 @@ class DirSrvTools(object): dirsrv.sroot : root of the instance (e.g. /usr/lib64/dirsrv) dirsrv.inst : instance name (e.g. standalone for /etc/dirsrv/slapd-standalone) dirsrv.confdir : root of the instance config (e.g. /etc/dirsrv) - dirsrv.dbdir: directory where is stored the database (e.g. /var/lib/dirsrv/slapd-standalon/db + dirsrv.dbdir: directory where is stored the database (e.g. /var/lib/dirsrv/slapd-standalone/db) + dirsrv.changelogdir: directory where is stored the changelog (e.g. /var/lib/dirsrv/slapd-master/changelogdb) """ # First check it if already exists a backup file @@ -340,8 +341,18 @@ class DirSrvTools(object): os.chdir(dirsrv.prefix) prefix_pattern = "%s/" % dirsrv.prefix + # build the list of directories to scan instroot = "%s/slapd-%s" % (dirsrv.sroot, dirsrv.inst) - for dirToBackup in [ instroot, dirsrv.confdir, dirsrv.dbdir]: + ldir = [ instroot ] + if hasattr(dirsrv, 'confir'): + ldir.append(dirsrv.confdir) + if hasattr(dirsrv, 'dbdir'): + ldir.append(dirsrv.dbdir) + if hasattr(dirsrv, 'changelogdb'): + ldir.append(dirsrv.changelogdb) + + # now scan the directory list to find the files to backup + for dirToBackup in ldir: for root, dirs, files in os.walk(dirToBackup): for file in files: name = os.path.join(root, file) diff --git a/tests/dsadmin_test.py b/tests/dsadmin_test.py index 285d661..251617d 100644 --- a/tests/dsadmin_test.py +++ b/tests/dsadmin_test.py @@ -207,7 +207,7 @@ def setupAgreement_test(): conn.replica.add(**args) conn.added_entries.append(args['binddn']) - dn_replica = conn.setupAgreement(consumer, args) + dn_replica = conn.createAgreement(consumer, args) def stop_start_test(): diff --git a/tests/replica_test.py b/tests/replica_test.py index c0e7feb..c358296 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -174,7 +174,7 @@ def enable_logging_test(): def status_test(): - status = conn.replica.status(conn.agreement_dn) + status = conn.agreement.status(conn.agreement_dn) log.info(status) assert status -- 1.7.11.7