From ad257d63059a619cf83929aee4d165f012a98f3a Mon Sep 17 00:00:00 2001 From: Sankar Ramalingam Date: Mon, 11 Sep 2017 21:11:09 +0530 Subject: [PATCH] Ticket #48085 - CI tests - replication cl5 Description: Adding changelog5 tests for replication. Test cases will check for changelog contains all the ldap operations are present in the changelog dump file created using nsds5task. https://pagure.io/389-ds-base/issue/48085 Reviewed by: ? --- .../tests/suites/replication/changelog_test.py | 271 +++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100755 dirsrvtests/tests/suites/replication/changelog_test.py diff --git a/dirsrvtests/tests/suites/replication/changelog_test.py b/dirsrvtests/tests/suites/replication/changelog_test.py new file mode 100755 index 0000000..3281c82 --- /dev/null +++ b/dirsrvtests/tests/suites/replication/changelog_test.py @@ -0,0 +1,271 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2017 Red Hat, Inc. +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- +# +import os, ldap, logging, pytest, re +from lib389.replica import Replicas +from lib389.properties import TASK_WAIT +from lib389.idm.user import UserAccounts +from lib389.topologies import topology_st as topo +from lib389._constants import DEFAULT_SUFFIX, PASSWORD, DN_MAPPING_TREE, RDN_REPLICA, ReplicaRole + +TEST_ENTRY_NAME = 'replusr' +NEW_RDN_NAME = 'cl5usr' +ATTRIBUTES = ('replgen', 'csn', 'nsuniqueid', 'dn', 'change', 'newrdn', 'parentuniqueid', 'newrdn', 'deleteoldrdn') +CHANGE_TYPES = {'add', 'modify', 'modrdn', 'delete'} +DN_REPLICA = '{},cn="{}",{}'.format(RDN_REPLICA, DEFAULT_SUFFIX, DN_MAPPING_TREE) +CHANGELOG_NAME = 'changelog5' + +DEBUGGING = os.getenv('DEBUGGING', default=False) +if DEBUGGING: + logging.getLogger(__name__).setLevel(logging.DEBUG) +else: + logging.getLogger(__name__).setLevel(logging.INFO) +log = logging.getLogger(__name__) + + +@pytest.fixture(scope='module') +def replica_changelog(topo, request): + """Configure replica and changelog for standalone instance""" + + log.info('Configure replica for standalone instance') + changelog_dir = os.path.join(topo.standalone.dbdir, CHANGELOG_NAME) + topo.standalone.changelog.create(dbname=changelog_dir) + replica_dn = topo.standalone.replica.create(suffix=DEFAULT_SUFFIX, role=ReplicaRole.MASTER, rid=1) + + def fin(): + log.info('Delete replica entry-{} for standalone instance'.format(replica_dn)) + try: + topo.standalone.delete_s(replica_dn) + except ldap.LDAPError as e: + log.fatal('Failed to delete replica entry-{}'.format(replica_dn)) + log.info('Unconfigure changelog for standalone instance') + topo.standalone.changelog.delete() + + request.addfinalizer(fin) + return replica_dn + + +def ldap_operations(topo): + """Add test users using UserAccounts""" + + log.info('Adding user {}'.format(TEST_ENTRY_NAME)) + users = UserAccounts(topo.standalone, DEFAULT_SUFFIX) + user_properties = { + 'uid': TEST_ENTRY_NAME, + 'cn': TEST_ENTRY_NAME, + 'sn': TEST_ENTRY_NAME, + 'uidNumber': '1001', + 'gidNumber': '2001', + 'userpassword': PASSWORD, + 'description': 'userdesc', + 'homeDirectory': '/home/{}'.format(TEST_ENTRY_NAME)} + tuser = users.create(properties=user_properties) + tuser.replace('description', 'newdesc') + log.info('Modify RDN of user {}'.format(tuser.dn)) + try: + topo.standalone.modrdn_s(tuser.dn, 'uid={}'.format(NEW_RDN_NAME), 0) + except ldap.LDAPError as e: + log.fatal('Failed to modrdn entry {}'.format(tuser.dn)) + raise e + tuser = users.get(NEW_RDN_NAME) + log.info('Deleting user-{}'.format(tuser.dn)) + tuser.delete() + + +def _create_changelog_dump(topo): + """Dump changelog using nss5task and check if ldap operations are logged""" + + log.info('Dump changelog using nss5task and check if ldap operations are logged') + changelog_dir = os.path.join(topo.standalone.dbdir, CHANGELOG_NAME) + if len(os.listdir(changelog_dir)) != 0: + for files in os.listdir(changelog_dir): + if files.endswith('.ldif'): + log.info('Remove LDIF files present in {}'.format(changelog_dir)) + os.remove(os.path.join(changelog_dir, files)) + try: + topo.standalone.modify_s(DN_REPLICA, [(ldap.MOD_REPLACE, 'nsds5task', 'cl2ldif')]) + except ldap.LDAPError as e: + log.fatal('Failed to dump changelog to ldif file') + raise e + + log.info('List the changelog dump file') + for files in os.listdir(changelog_dir): + if files.endswith('.ldif'): + log.info('Changelog LDIF file exists-{}'.format(changelog_dir)) + change_log = os.path.join(changelog_dir, files) + return change_log + + +def _check_changelog_ldif(topo, change_log): + """Check changelog ldif file for ldap operations""" + + log.info('Checking changelog ldif file for ldap operations') + change_types = set() + change_type_flag = False + ldap_operation = '' + if not os.stat(change_log).st_size > 0: + log.fatal('Changelog file-{} has no contents'.format(change_log)) + assert False + with open(change_log, 'r') as fh: + attr_list = list(ATTRIBUTES) + for line in fh: + if not line.strip(): + continue + elif re.match('^changetype: ["add", "modify", "modrdn", "delete"]', line): + ldap_operation = line.split(':')[-1].strip() + log.info('Found ldap operation type-{}'.format(ldap_operation)) + change_types.add(ldap_operation) + change_type_flag = True + attr_list = list(ATTRIBUTES) + elif len(attr_list) != 0 and change_type_flag: + if ldap_operation == 'delete' and 'change' in attr_list: + attr_list.remove('change') + attr = line.split(':')[0] + if attr in attr_list: + attr_list.remove(attr) + else: + log.fatal('Missing some of the LDAP operations-{}'.format(CHANGE_TYPES)) + assert False + + if CHANGE_TYPES.issubset(change_types): + log.info('All LDAP operation types-{} are present'.format(', '.join(change_types))) + assert True + else: + log.fatal('NO LDAP operations type found') + assert False + + +def test_verify_changelog(topo, replica_changelog): + """Check if changelog is logging ldap operations + + :id: 16347170-46cf-44f5-878e-16974ed3c394 + :feature: Changelog5 + :setup: Two master two consumer setup + :steps: 1. Add user to server. + 2. Perform ldap modify, modrdn and delete operations. + 3. Dump the changelog to a file using nsds5task. + 4. Check if changelog is updated with ldap operations. + :expectedresults: + 1. Add user should PASS. + 2. Ldap operations should PASS. + 3. Changelog should be dumped successfully. + 4. Changelog dump file should contain ldap operations + """ + + log.info('LDAP operations add, modify, modrdn and delete') + ldap_operations(topo) + change_log = _create_changelog_dump(topo) + _check_changelog_ldif(topo, change_log) + + +def test_verify_changelog_online_backup(topo, replica_changelog): + """Check if changelog is logging ldap operations + + :id: 196747c9-8a4b-4f76-a383-42e4310b0dbb + :feature: Changelog5 + :setup: Two master two consumer setup + :steps: 1. Add user to server. + 2. Take online backup using db2bak task. + 3. Restore the database using bak2db task. + 4. Perform ldap modify, modrdn and delete operations. + 5. Dump the changelog to a file using nsds5task. + 6. Check if changelog is updated with ldap operations. + :expectedresults: + 1. Add user should PASS. + 2. Backup of database should PASS. + 3. Restore of database should PASS. + 4. Ldap operations should PASS. + 5. Changelog should be dumped successfully. + 6. Changelog dump file should contain ldap operations + """ + + backup_dir = os.path.join(topo.standalone.get_bak_dir(), 'online_backup') + log.info('Run db2bak script to take database backup') + try: + topo.standalone.tasks.db2bak(backup_dir=backup_dir, args={TASK_WAIT: True}) + except ValueError: + log.fatal('test_changelog5: Online backup failed') + assert False + + backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup/{}'.format(CHANGELOG_NAME)) + if os.path.exists(backup_checkdir): + log.info('Database backup is created successfully') + else: + log.error('test_changelog5: backup directory does not exist -{}'.format(backup_checkdir)) + assert False + + log.info('Run bak2db to restore directory server') + try: + topo.standalone.tasks.bak2db(backup_dir=backup_dir, args={TASK_WAIT: True}) + except ValueError: + log.fatal('test_changelog5: Online restore failed') + assert False + + log.info('LDAP operations add, modify, modrdn and delete') + ldap_operations(topo) + change_log = _create_changelog_dump(topo) + _check_changelog_ldif(topo, change_log) + + +def test_verify_changelog_offline_backup(topo, replica_changelog): + """Check if changelog is logging ldap operations + + :id: cbf84dcc-2baa-4563-bb54-07645339faca + :feature: Changelog5 + :setup: Two master two consumer setup + :steps: 1. Add user to server. + 2. Stop server and take offline backup using db2bak. + 3. Restore the database using bak2db. + 4. Perform ldap modify, modrdn and delete operations. + 5. Start the server and dump the changelog using nsds5task. + 6. Check if changelog is updated with ldap operations. + :expectedresults: + 1. Add user should PASS. + 2. Backup of database should PASS. + 3. Restore of database should PASS. + 4. Ldap operations should PASS. + 5. Changelog should be dumped successfully. + 6. Changelog dump file should contain ldap operations + """ + + backup_dir = os.path.join(topo.standalone.get_bak_dir(), 'offline_backup') + + topo.standalone.stop() + log.info('Run db2bak to take database backup') + try: + topo.standalone.db2bak(backup_dir) + except ValueError: + log.fatal('test_changelog5: Offline backup failed') + assert False + + log.info('Run bak2db to restore directory server') + try: + topo.standalone.bak2db(backup_dir) + except ValueError: + log.fatal('test_changelog5: Offline restore failed') + assert False + topo.standalone.start() + + backup_checkdir = os.path.join(backup_dir, '.repl_changelog_backup/{}'.format(CHANGELOG_NAME)) + if os.path.exists(backup_checkdir): + log.info('Database backup is created successfully') + else: + log.fatal('test_changelog5: backup directory does not exist -{}'.format(backup_checkdir)) + assert False + + log.info('LDAP operations add, modify, modrdn and delete') + ldap_operations(topo) + change_log = _create_changelog_dump(topo) + _check_changelog_ldif(topo, change_log) + + +if __name__ == '__main__': + # Run isolated + # -s for DEBUG mode + CURRENT_FILE = os.path.realpath(__file__) + pytest.main('-s {}'.format(CURRENT_FILE)) -- 2.7.4