From 4e60f7efe24755ce544d19059da9dff8d1495d4b Mon Sep 17 00:00:00 2001 From: "Thierry bordaz (tbordaz)" Date: Wed, 13 Nov 2013 16:41:49 +0100 Subject: [PATCH] Ticket 47590: CI tests: add/split functions around replication Bug Description: Functions to setup replication are a bit too complex. They do a lot of things that should be split in several simpler functions. The parameters are also complex with mandatory/optional. Fix Description: The fix implements: - enableReplication(suffix, role, replicaId, [binddn]) - createAgreement(suffix, host, port, [binddn], [bindpw], [bindmethod]) - initAgreement(suffix, host, port) - _createDefaultReplMgr() - getSchemaCSN(instance) https://fedorahosted.org/389/ticket/47590? Reviewed by: Flag Day: no Doc impact: no --- bug_harness.py | 2 +- lib389/__init__.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++---- lib389/_constants.py | 16 ++++++ lib389/brooker.py | 68 +++++++++++++++++++++-- tests/dsadmin_test.py | 2 +- 5 files changed, 221 insertions(+), 18 deletions(-) diff --git a/bug_harness.py b/bug_harness.py index 1d523bc..621ef84 100644 --- a/bug_harness.py +++ b/bug_harness.py @@ -39,7 +39,7 @@ class DSAdminHarness(DSAdmin, DSAdminTools): args.setdefault('binddn', REPLBINDDN) args.setdefault('bindpw', REPLBINDPW) - return DSAdmin.setupAgreement(self, repoth, args) + return DSAdmin.createAgreement(self, repoth, args) def setupReplica(self, args): """Set default replia credentials """ diff --git a/lib389/__init__.py b/lib389/__init__.py index 2f980ff..8448031 100644 --- a/lib389/__init__.py +++ b/lib389/__init__.py @@ -33,6 +33,7 @@ import operator import shutil import datetime import logging +import decimal from ldap.ldapobject import SimpleLDAPObject from ldapurl import LDAPUrl @@ -892,10 +893,11 @@ class DirSrv(SimpleLDAPObject): def addObjClass(self, *objectclasses): return self.addSchema('objectClasses', objectclasses) - - - - + + def getSchemaCSN(self): + ents = self.search_s("cn=schema", ldap.SCOPE_BASE, "objectclass=*", ['nsSchemaCSN']) + ent = ents[0] + return ent.getValue('nsSchemaCSN') def setupChainingIntermediate(self): confdn = ','.join(("cn=config", DN_CHAIN)) @@ -1044,7 +1046,7 @@ class DirSrv(SimpleLDAPObject): # args - DirSrv consumer (repoth), suffix, binddn, bindpw, timeout # also need an auto_init argument - def setupAgreement(self, consumer, args, cn_format=r'meTo_%s:%s', description_format=r'me to %s:%s'): + def createAgreement(self, consumer, args, cn_format=r'meTo_%s:%s', description_format=r'me to %s:%s'): """Create (and return) a replication agreement from self to consumer. - self is the supplier, - consumer is a DirSrv object (consumer can be a master) @@ -1056,9 +1058,7 @@ class DirSrv(SimpleLDAPObject): args = { 'suffix': "dc=example,dc=com", - 'bename': "userRoot", 'binddn': "cn=replrepl,cn=config", - 'bindcn': "replrepl", # so I need it? 'bindpw': "replrepl", 'bindmethod': 'simple', 'log' : True. @@ -1070,11 +1070,46 @@ class DirSrv(SimpleLDAPObject): 'o=suffix2': 'ldap://consumer.example.net:3890' } """ - assert args.get('binddn') and args.get('bindpw') suffix = args['suffix'] + if not suffix: + # This is a mandatory parameter of the command... it fails + log.warning("createAgreement: suffix is missing") + return None + + # get the RA binddn binddn = args.get('binddn') + if not binddn: + binddn = defaultProperties.get(REPLICATION_BIND_DN, None) + if not binddn: + # weird, internal error we do not retrieve the default replication bind DN + # this replica agreement will fail to update the consumer until the + # property will be set + log.warning("createAgreement: binddn not provided and default value unavailable") + pass + + + # get the RA binddn password bindpw = args.get('bindpw') - + if not bindpw: + bindpw = defaultProperties.get(REPLICATION_BIND_PW, None) + if not bindpw: + # weird, internal error we do not retrieve the default replication bind DN password + # this replica agreement will fail to update the consumer until the + # property will be set + log.warning("createAgreement: bindpw not provided and default value unavailable") + pass + + # get the RA bind method + bindmethod = args.get('bindmethod') + if not bindmethod: + bindmethod = defaultProperties.get(REPLICATION_BIND_METHOD, None) + if not bindmethod: + # weird, internal error we do not retrieve the default replication bind method + # this replica agreement will fail to update the consumer until the + # property will be set + log.warning("createAgreement: bindmethod not provided and default value unavailable") + pass + nsuffix = normalizeDN(suffix) othhost, othport, othsslport = ( consumer.host, consumer.port, consumer.sslport) @@ -1092,6 +1127,7 @@ class DirSrv(SimpleLDAPObject): 'dn': replent.dn, 'type': int(replent.nsds5replicatype) } + # define agreement entry cn = cn_format % (othhost, othport) dn_agreement = "cn=%s,%s" % (cn, self.suffixes[nsuffix]['dn']) @@ -1116,7 +1152,7 @@ class DirSrv(SimpleLDAPObject): 'nsds5replicatimeout': str(args.get('timeout', 120)), 'nsds5replicabinddn': binddn, 'nsds5replicacredentials': bindpw, - 'nsds5replicabindmethod': args.get('bindmethod', 'simple'), + 'nsds5replicabindmethod': bindmethod, 'nsds5replicaroot': nsuffix, 'nsds5replicaupdateschedule': '0000-2359 0123456', 'description': description_format % (othhost, othport) @@ -1200,7 +1236,102 @@ class DirSrv(SimpleLDAPObject): def startReplication(self, agmtdn): return self.replica.start_and_wait(agmtdn) + + def _createDefaultReplMgr(self): + # create replication Manager entry without timeout and expiration issues + + # check if the default replication manager entry already exists + repl_manager_dn = defaultProperties.get(REPLICATION_BIND_DN, None) + if not repl_manager_dn: + log.fatal("_createDefaultReplMgr: Default replication manager DN unavailable") + return + + attrs = [ 'dn' ] + try: + entries = self.search_s(repl_manager_dn, ldap.SCOPE_BASE, "objectclass=*") + if entries: + #it already exist, fine + return + except ldap.NO_SUCH_OBJECT: + pass + + # ok it does not exist, create it + repl_manager_pw = defaultProperties.get(REPLICATION_BIND_PW, None) + try: + attrs = { + 'nsIdleTimeout': '0', + 'passwordExpirationTime': '20381010000000Z' + } + self.setupBindDN(repl_manager_dn, repl_manager_pw, attrs) + except ldap.ALREADY_EXISTS: + log.warn("User already exists: %r " % repl_manager_dn) + + def enableReplication(self, suffix=None, role=None, replicaId=CONSUMER_REPLICAID, binddn=None): + if not suffix: + log.fatal("enableReplication: suffix not specified") + return 1 + + if not role: + log.fatal("enableReplication: replica role not specify (REPLICAROLE_*)") + return 1 + + # + # Check the validity of the parameters + # + + # First role and replicaID + if role == REPLICAROLE_MASTER: + # master + replica_type = REPLICA_RDWR_TYPE + else: + # hub or consumer + replica_type = REPLICA_RDONLY_TYPE + + if replica_type == REPLICA_RDWR_TYPE: + # check the replicaId [1..CONSUMER_REPLICAID[ + if not decimal.Decimal(replicaId) or (replicaId <= 0) or (replicaId >=CONSUMER_REPLICAID): + log.fatal("enableReplication: invalid replicaId (%s) for a RW replica" % replicaId) + return 1 + elif replicaId != CONSUMER_REPLICAID: + # check the replicaId is CONSUMER_REPLICAID + log.fatal("enableReplication: invalid replicaId (%s) for a Read replica (expected %d)" % (replicaId, CONSUMER_REPLICAID)) + return 1 + + # Now check we have a suffix + entries_backend = self.getBackendsForSuffix(suffix, ['nsslapd-suffix']) + if not entries_backend: + log.fatal("enableReplication: enable to retrieve the backend for %s" % suffix) + return 1 + + ent = entries_backend[0] + if normalizeDN(suffix) != normalizeDN(ent.getValue('nsslapd-suffix')): + log.warning("enableReplication: suffix (%s) and backend suffix (%s) differs" % (suffix, entries_backend[0].nsslapd-suffix)) + pass + + # Now prepare the bindDN property + if not binddn: + binddn = defaultProperties.get(REPLICATION_BIND_DN, None) + if not binddn: + # weird, internal error we do not retrieve the default replication bind DN + # this replica will not be updatable through replication until the binddn + # property will be set + log.warning("enableReplication: binddn not provided and default value unavailable") + pass + + # Now do the effectif job + # First add the changelog if master/hub + if (role == REPLICAROLE_MASTER) or (role == REPLICAROLE_HUB): + self.replica.changelog() + + # Second create the default replica manager entry if it does not exist + # it should not be called from here but for the moment I am unsure when to create it elsewhere + self._createDefaultReplMgr() + + # then enable replication + ret = self.replica.add(suffix=suffix, binddn=binddn, rtype=replica_type, rid=replicaId) + + return ret def replicaSetupAll(self, repArgs): """setup everything needed to enable replication for a given suffix. diff --git a/lib389/_constants.py b/lib389/_constants.py index 3103587..b0eb23a 100644 --- a/lib389/_constants.py +++ b/lib389/_constants.py @@ -5,11 +5,27 @@ (MASTER_TYPE, HUB_TYPE, LEAF_TYPE) = range(3) + +REPLICAROLE_MASTER = "master" +REPLICAROLE_HUB = "hub" +REPLICAROLE_CONSUMER = "consumer" + +CONSUMER_REPLICAID = 65535 REPLICA_RDONLY_TYPE = 2 # CONSUMER and HUB REPLICA_WRONLY_TYPE = 1 # SINGLE and MULTI MASTER REPLICA_RDWR_TYPE = REPLICA_RDONLY_TYPE | REPLICA_WRONLY_TYPE +REPLICATION_BIND_DN = 'replication_bind_dn' +REPLICATION_BIND_PW = 'replication_bind_pw' +REPLICATION_BIND_METHOD = 'replication_bind_method' + +defaultProperties = { + REPLICATION_BIND_DN: "cn=replrepl,cn=config", + REPLICATION_BIND_PW: "password", + REPLICATION_BIND_METHOD: "simple" +} + CFGSUFFIX = "o=NetscapeRoot" DEFAULT_USER = "nobody" diff --git a/lib389/brooker.py b/lib389/brooker.py index b3a1b6b..a4c4a37 100644 --- a/lib389/brooker.py +++ b/lib389/brooker.py @@ -219,14 +219,14 @@ class Replica(object): # FormatDict manages missing fields in string formatting return retstr % FormatDict(ent.data) - def add(self, suffix, binddn, bindpw, rtype=MASTER_TYPE, rid=None, tombstone_purgedelay=None, purgedelay=None, referrals=None, legacy=False): + def add(self, suffix, binddn, bindpw=None, rtype=REPLICA_RDONLY_TYPE, rid=None, tombstone_purgedelay=None, purgedelay=None, referrals=None, legacy=False): """Setup a replica entry on an existing suffix. @param suffix - dn of suffix @param binddn - the replication bind dn for this replica can also be a list ["cn=r1,cn=config","cn=r2,cn=config"] @param bindpw - used to eventually provision the replication entry - @param rtype - master, hub, leaf (see above for values) - default is master + @param rtype - REPLICA_RDWR_TYPE (master) or REPLICA_RDONLY_TYPE (hub/consumer) @param rid - replica id or - if not given - an internal sequence number will be assigned # further args @@ -244,10 +244,6 @@ class Replica(object): TODO: this method does not update replica type """ # set default values - if rtype == MASTER_TYPE: - rtype = REPLICA_RDWR_TYPE - else: - rtype = REPLICA_RDONLY_TYPE if legacy: legacy = 'on' @@ -351,6 +347,66 @@ class Replica(object): return [ent.dn for ent in ents] return ents + def initAgreement(self, suffix=None, consumer_host=None, consumer_port=None): + """Trigger a total update of the consumer replica + - self is the supplier, + - consumer is a DirSrv object (consumer can be a master) + - cn_format - use this string to format the agreement name + + consumer: + * a DirSrv object if chaining + * an object with attributes: host, port, sslport, __str__ + + args = { + 'suffix': "dc=example,dc=com", + 'binddn': "cn=replrepl,cn=config", + 'bindpw': "replrepl", + 'bindmethod': 'simple', + 'log' : True. + 'timeout': 120 + } + """ + # + # check the required parameters are set + # + if not suffix: + self.log.fatal("initAgreement: suffix is missing") + return -1 + nsuffix = normalizeDN(suffix) + + if not consumer_host: + self.log.fatal("initAgreement: host is missing") + return -1 + + if not consumer_port: + self.log.fatal("initAgreement: port is missing") + return -1 + # + # check the replica agreement already exist + # + replica_entries = self.list(suffix) + if not replica_entries: + raise NoSuchEntryError( + "Error: no replica set up for suffix " + suffix) + replica_entry = replica_entries[0] + self.log.debug("initAgreement: looking for replica agreements under %s" % replica_entry.dn) + try: + filt = "(&(objectclass=nsds5replicationagreement)(nsds5replicahost=%s)(nsds5replicaport=%d)(nsds5replicaroot=%s))" % (consumer_host, consumer_port, nsuffix) + entry = self.conn.getEntry(replica_entry.dn, ldap.SCOPE_ONELEVEL, filt) + except ldap.NO_SUCH_OBJECT: + self.log.fatal("initAgreement: No replica agreement to %s:%d for suffix %s" % (consumer_host, consumer_port, nsuffix)) + return -1 + + # + # trigger the total init + # + self.log.info("Starting total init %s" % entry.dn) + mod = [(ldap.MOD_ADD, 'nsds5BeginReplicaRefresh', 'start')] + self.conn.modify_s(entry.dn, mod) + + return 0 + + def agreement_add(self, consumer, suffix=None, binddn=None, bindpw=None, cn_format=r'meTo_$host:$port', description_format=r'me to $host:$port', timeout=120, auto_init=False, bindmethod='simple', starttls=False, schedule=ALWAYS, args=None): """Create (and return) a replication agreement from self to consumer. - self is the supplier, diff --git a/tests/dsadmin_test.py b/tests/dsadmin_test.py index 285d661..251617d 100644 --- a/tests/dsadmin_test.py +++ b/tests/dsadmin_test.py @@ -207,7 +207,7 @@ def setupAgreement_test(): conn.replica.add(**args) conn.added_entries.append(args['binddn']) - dn_replica = conn.setupAgreement(consumer, args) + dn_replica = conn.createAgreement(consumer, args) def stop_start_test(): -- 1.7.11.7