From 1c5f0605e2d3be4a56b4d3aede4c02e11256b15b Mon Sep 17 00:00:00 2001 From: William Brown Date: Feb 01 2019 00:50:00 +0000 Subject: Ticket 50184 - Add cli tool parity to dsconf/dsctl Bug Description: As we are removing the shell/perl tools, we need to have functional parity with the existing tools. This adds the final tools needed to make that equivalent. Fix Description: Add support for dbverify, linkedattr fixup and a monitoring tool. https://pagure.io/389-ds-base/issue/50184 Author: William Brown Review by: mreynolds (thanks!) --- diff --git a/src/lib389/cli/dsconf b/src/lib389/cli/dsconf index b22736e..a844fca 100755 --- a/src/lib389/cli/dsconf +++ b/src/lib389/cli/dsconf @@ -23,6 +23,7 @@ from lib389.cli_conf import directory_manager as cli_directory_manager from lib389.cli_conf import plugin as cli_plugin from lib389.cli_conf import schema as cli_schema from lib389.cli_conf import health as cli_health +from lib389.cli_conf import monitor as cli_monitor from lib389.cli_conf import saslmappings as cli_sasl from lib389.cli_conf import pwpolicy as cli_pwpolicy from lib389.cli_conf import backup as cli_backup @@ -78,6 +79,7 @@ cli_chaining.create_parser(subparsers) cli_config.create_parser(subparsers) cli_directory_manager.create_parsers(subparsers) cli_health.create_parser(subparsers) +cli_monitor.create_parser(subparsers) cli_plugin.create_parser(subparsers) cli_pwpolicy.create_parser(subparsers) cli_replication.create_parser(subparsers) diff --git a/src/lib389/lib389/__init__.py b/src/lib389/lib389/__init__.py index 974e218..fb987e0 100644 --- a/src/lib389/lib389/__init__.py +++ b/src/lib389/lib389/__init__.py @@ -3091,6 +3091,32 @@ class DirSrv(SimpleLDAPObject, object): return output + def dbverify(self, bename): + """ + @param bename - the backend name to verify + @return - True if the verify succeded + """ + prog = os.path.join(self.ds_paths.sbin_dir, 'ns-slapd') + + if self.status(): + self.log.error("dbverify: Can not operate while directory server is running") + return False + + cmd = [ + prog, + 'dbverify', + '-D', self.get_config_dir(), + '-n', bename + ] + + try: + result = subprocess.check_output(cmd, encoding='utf-8') + except subprocess.CalledProcessError as e: + self.log.debug("Command: %s failed with the return code %s and the error %s", + format_cmd_list(cmd), e.returncode, e.output) + return False + return True + def searchAccessLog(self, pattern): """ Search all the access logs diff --git a/src/lib389/lib389/_mapped_object.py b/src/lib389/lib389/_mapped_object.py index eae3154..1dd8825 100644 --- a/src/lib389/lib389/_mapped_object.py +++ b/src/lib389/lib389/_mapped_object.py @@ -491,6 +491,18 @@ class DSLdapObject(DSLogging): entry = self._instance.search_ext_s(self._dn, ldap.SCOPE_BASE, self._object_filter, attrlist=keys, serverctrls=self._server_controls, clientctrls=self._client_controls)[0] return entry.getValuesSet(keys) + def get_attrs_vals_utf8(self, keys, use_json=False): + self._log.debug("%s get_attrs_vals_utf8(%r)" % (self._dn, keys)) + if self._instance.state != DIRSRV_STATE_ONLINE: + raise ValueError("Invalid state. Cannot get properties on instance that is not ONLINE") + entry = self._instance.search_ext_s(self._dn, ldap.SCOPE_BASE, self._object_filter, attrlist=keys, serverctrls=self._server_controls, clientctrls=self._client_controls)[0] + vset = entry.getValuesSet(keys) + r = {} + for (k, vo) in vset.items(): + r[k] = ensure_list_str(vo) + return r + + def get_attr_vals(self, key, use_json=False): self._log.debug("%s get_attr_vals(%r)" % (self._dn, key)) # We might need to add a state check for NONE dn. diff --git a/src/lib389/lib389/cli_base/__init__.py b/src/lib389/lib389/cli_base/__init__.py index 14cb999..6175cc3 100644 --- a/src/lib389/lib389/cli_base/__init__.py +++ b/src/lib389/lib389/cli_base/__init__.py @@ -366,7 +366,7 @@ def setup_script_logger(name, verbose=False): """ root = logging.getLogger() log = logging.getLogger(name) - log_handler = logging.StreamHandler() + log_handler = logging.StreamHandler(sys.stdout) if verbose: log.setLevel(logging.DEBUG) diff --git a/src/lib389/lib389/cli_conf/__init__.py b/src/lib389/lib389/cli_conf/__init__.py index 1ba3b4a..836e05d 100644 --- a/src/lib389/lib389/cli_conf/__init__.py +++ b/src/lib389/lib389/cli_conf/__init__.py @@ -70,33 +70,33 @@ def generic_object_edit(dsldap_object, log, args, arg_to_attr): def generic_show(inst, basedn, log, args): """Display plugin configuration.""" plugin = args.plugin_cls(inst) - print(plugin.display()) + log.info(plugin.display()) def generic_enable(inst, basedn, log, args): plugin = args.plugin_cls(inst) if plugin.status(): - print("Plugin '%s' already enabled", plugin.rdn) + log.info("Plugin '%s' already enabled" % plugin.rdn) else: plugin.enable() - print("Enabled plugin '%s'", plugin.rdn) + log.info("Enabled plugin '%s'" % plugin.rdn) def generic_disable(inst, basedn, log, args): plugin = args.plugin_cls(inst) if not plugin.status(): - print("Plugin '%s' already disabled", plugin.rdn) + log.info("Plugin '%s' already disabled" % plugin.rdn) else: plugin.disable() - print("Disabled plugin '%s'", plugin.rdn) + log.info("Disabled plugin '%s'" % plugin.rdn) def generic_status(inst, basedn, log, args): plugin = args.plugin_cls(inst) if plugin.status() is True: - print("Plugin '%s' is enabled", plugin.rdn) + log.info("Plugin '%s' is enabled" % plugin.rdn) else: - print("Plugin '%s' is disabled", plugin.rdn) + log.info("Plugin '%s' is disabled" % plugin.rdn) def add_generic_plugin_parsers(subparser, plugin_cls): diff --git a/src/lib389/lib389/cli_conf/monitor.py b/src/lib389/lib389/cli_conf/monitor.py new file mode 100644 index 0000000..9e89381 --- /dev/null +++ b/src/lib389/lib389/cli_conf/monitor.py @@ -0,0 +1,64 @@ +# --- BEGIN COPYRIGHT BLOCK --- +# Copyright (C) 2019 William Brown +# All rights reserved. +# +# License: GPL (version 3 or any later version). +# See LICENSE for details. +# --- END COPYRIGHT BLOCK --- + +from lib389.monitor import Monitor, MonitorLDBM +from lib389.backend import Backends + +from lib389.utils import ensure_str + +def _format_status(log, mtype, json=False): + if json: + print(mtype.get_status_json()) + else: + status_dict = mtype.get_status() + log.info('dn: ' + mtype._dn) + for k, v in list(status_dict.items()): + # For each value in the multivalue attr + for vi in v: + log.info('{}: {}'.format(k, vi)) + + +def monitor(inst, basedn, log, args): + monitor = Monitor(inst) + _format_status(log, monitor, args.json) + + +def backend_monitor(inst, basedn, log, args): + bes = Backends(inst) + if args.backend: + be = bes.get(args.backend) + be_monitor = be.get_monitor() + be_monitor.get_status() + else: + for be in bes.list(): + be_monitor = be.get_monitor() + be_monitor.get_status() + # Inejct a new line for now ... see https://pagure.io/389-ds-base/issue/50189 + print("") + + +def ldbm_monitor(inst, basedn, log, args): + ldbm_monitor = MonitorLDBM(inst) + ldbm_monitor.get_status() + + +def create_parser(subparsers): + monitor_parser = subparsers.add_parser('monitor', help="Monitor the state of the instance") + + subcommands = monitor_parser.add_subparsers(help='action') + + server_parser = subcommands.add_parser('server', help="Monitor the server statistics, connectinos and operations") + server_parser.set_defaults(func=monitor) + + ldbm_parser = subcommands.add_parser('ldbm', help="Monitor the ldbm statistics, such as dbcache") + ldbm_parser.set_defaults(func=ldbm_monitor) + + backend_parser = subcommands.add_parser('backend', help="Monitor the behaviour of a backend database") + backend_parser.add_argument('backend', nargs='?', help="Optional name of the backend to monitor") + backend_parser.set_defaults(func=backend_monitor) + diff --git a/src/lib389/lib389/cli_conf/plugins/linkedattr.py b/src/lib389/lib389/cli_conf/plugins/linkedattr.py index 7581e80..3f9a6ac 100644 --- a/src/lib389/lib389/cli_conf/plugins/linkedattr.py +++ b/src/lib389/lib389/cli_conf/plugins/linkedattr.py @@ -1,5 +1,5 @@ # --- BEGIN COPYRIGHT BLOCK --- -# Copyright (C) 2018 Red Hat, Inc. +# Copyright (C) 2019 William Brown # All rights reserved. # # License: GPL (version 3 or any later version). @@ -10,7 +10,26 @@ from lib389.plugins import LinkedAttributesPlugin from lib389.cli_conf import add_generic_plugin_parsers +def fixup(inst, basedn, log, args): + plugin = LinkedAttributesPlugin(inst) + log.info('Attempting to add task entry... This will fail if LinkedAttributes plug-in is not enabled.') + if not plugin.status(): + log.error("'%s' is disabled. Fix up task can't be executed" % plugin.rdn) + fixup_task = plugin.fixup(args.basedn, args.filter) + fixup_task.wait() + exitcode = fixup_task.get_exit_code() + if exitcode != 0: + log.error('LinkedAttributes fixup task for %s has failed. Please, check logs') + else: + log.info('Successfully added fixup task') + + def create_parser(subparsers): linkedattr_parser = subparsers.add_parser('linkedattr', help='Manage and configure Linked Attributes plugin') subcommands = linkedattr_parser.add_subparsers(help='action') add_generic_plugin_parsers(subcommands, LinkedAttributesPlugin) + + fixup_parser = subcommands.add_parser('fixup', help='Run the fix-up task for linked attributes plugin') + fixup_parser.add_argument('basedn', help="basedn that contains entries to fix up") + fixup_parser.add_argument('-f', '--filter', help='Filter for entries to fix up linked attributes.') + fixup_parser.set_defaults(func=fixup) diff --git a/src/lib389/lib389/cli_conf/plugins/memberof.py b/src/lib389/lib389/cli_conf/plugins/memberof.py index 0ccbed3..90fb774 100644 --- a/src/lib389/lib389/cli_conf/plugins/memberof.py +++ b/src/lib389/lib389/cli_conf/plugins/memberof.py @@ -1,5 +1,6 @@ # --- BEGIN COPYRIGHT BLOCK --- # Copyright (C) 2018 Red Hat, Inc. +# Copyright (C) 2019 William Brown # All rights reserved. # # License: GPL (version 3 or any later version). @@ -74,12 +75,15 @@ def memberof_del_config(inst, basedn, log, args): def fixup(inst, basedn, log, args): plugin = MemberOfPlugin(inst) log.info('Attempting to add task entry... This will fail if MemberOf plug-in is not enabled.') - assert plugin.status(), "'%s' is disabled. Fix up task can't be executed" % plugin.rdn + if not plugin.status(): + log.error("'%s' is disabled. Fix up task can't be executed" % plugin.rdn) fixup_task = plugin.fixup(args.DN, args.filter) fixup_task.wait() exitcode = fixup_task.get_exit_code() - assert exitcode == 0, 'MemberOf fixup task for %s has failed. Please, check logs' - log.info('Successfully added task entry for %s', args.DN) + if exitcode != 0: + log.error('MemberOf fixup task for %s has failed. Please, check logs') + else: + log.info('Successfully added task entry') def _add_parser_args(parser): diff --git a/src/lib389/lib389/cli_ctl/dbtasks.py b/src/lib389/lib389/cli_ctl/dbtasks.py index 7854fbe..0b6e948 100644 --- a/src/lib389/lib389/cli_ctl/dbtasks.py +++ b/src/lib389/lib389/cli_ctl/dbtasks.py @@ -1,5 +1,6 @@ # --- BEGIN COPYRIGHT BLOCK --- # Copyright (C) 2016 Red Hat, Inc. +# Copyright (C) 2019 William Brown # All rights reserved. # # License: GPL (version 3 or any later version). @@ -65,6 +66,13 @@ def dbtasks_backups(inst, log, args): log.info("backups successful") +def dbtasks_verify(inst, log, args): + if not inst.dbverify(bename=args.backend): + log.fatal("dbverify failed") + return False + else: + log.info("dbverify successful") + def create_parser(subcommands): db2index_parser = subcommands.add_parser('db2index', help="Initialise a reindex of the server database. The server must be stopped for this to proceed.") @@ -85,6 +93,11 @@ def create_parser(subcommands): db2ldif_parser.add_argument('--encrypted', help="Export encrypted attributes", default=False, action='store_true') db2ldif_parser.set_defaults(func=dbtasks_db2ldif) + dbverify_parser = subcommands.add_parser('dbverify', help="Perform a db verification. You should only do this at direction of support") + dbverify_parser.add_argument('backend', help="The backend to verify. IE userRoot") + dbverify_parser.set_defaults(func=dbtasks_verify) + + bak2db_parser = subcommands.add_parser('bak2db', help="Restore a BDB backup of the database. The server must be stopped for this to proceed.") bak2db_parser.add_argument('archive', help="The archive to restore. This will erase all current server databases.") bak2db_parser.set_defaults(func=dbtasks_bak2db) diff --git a/src/lib389/lib389/monitor.py b/src/lib389/lib389/monitor.py index 0102630..325261c 100644 --- a/src/lib389/lib389/monitor.py +++ b/src/lib389/lib389/monitor.py @@ -85,6 +85,26 @@ class Monitor(DSLdapObject): starttime = self.get_attr_vals_utf8('starttime') return (dtablesize, readwaiters, entriessent, bytessent, currenttime, starttime) + def get_status(self, use_json=False): + return self.get_attrs_vals_utf8([ + 'version', + 'threads', + 'connection', + 'currentconnections', + 'totalconnections', + 'currentconnectionsatmaxthreads', + 'maxthreadsperconnhits', + 'dtablesize', + 'readwaiters', + 'opsinitiated', + 'opscompleted', + 'entriessent', + 'bytessent', + 'currenttime', + 'starttime', + 'nbackends', + ]) + class MonitorLDBM(DSLdapObject): def __init__(self, instance, dn=None): @@ -155,6 +175,8 @@ class MonitorBackend(DSLdapObject): 'currentnormalizeddncachecount' ]) + # Issue: get status should return a dict and the called should be + # formatting it. See: https://pagure.io/389-ds-base/issue/50189 def get_status(self, use_json=False): if use_json: print(self.get_attrs_vals_json(self._backend_keys))