Integration tests: Port the BeakerLib plugin and log collection to pytest

Move the IPA-specific log collection out of the Beakerlib plugin.
Add the --logfile-dir option to tests and ipa-test-task, so that logs
can be collected even if BeakerLib is not used.

https://fedorahosted.org/freeipa/ticket/4610

Reviewed-By: Tomas Babej <tbabej@redhat.com>

2 files changed, 368 insertions, 3 deletions

diff --git a/ipatests/pytest_plugins/beakerlib.py b/ipatests/pytest_plugins/beakerlib.py
new file mode 100644
index 000000000..45bbb0539
--- /dev/null
+++ b/ipatests/pytest_plugins/beakerlib.py
@@ -0,0 +1,234 @@
+#!/usr/bin/python2
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""pytest integration with BeakerLib
+
+Runs a Bash process on the side, and feeds BeakerLib commands to it
+(rlPhaseStart, rlPhaseEnd, rlPass, rlFail, ...)
+
+Other plugins may integrate with this using pytest's
+config.pluginmanager.getplugin('BeakerLibPlugin'). If this is None,
+BeakerLib integration is not active, otherwise the result's
+run_beakerlib_command method can be used to run additional commands.
+
+IPA logging is also redirected to the Bash process.
+"""
+
+import os
+import re
+import logging
+import subprocess
+
+import pytest
+
+from ipapython import ipautil
+from ipapython.ipa_log_manager import log_mgr
+
+
+@pytest.fixture
+def log_files_to_collect():
+    return []
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--with-beakerlib', action="store_true",
+        dest="with_beakerlib", default=None,
+        help="Report test results via beakerlib")
+
+
+@pytest.mark.tryfirst
+def pytest_load_initial_conftests(args, early_config, parser):
+    ns = early_config.known_args_namespace
+    if ns.with_beakerlib:
+        if 'BEAKERLIB' not in os.environ:
+            raise exit('$BEAKERLIB not set, cannot use --with-beakerlib')
+
+        plugin = BeakerLibPlugin()
+        pluginmanager = early_config.pluginmanager.register(
+            plugin, 'BeakerLibPlugin')
+
+
+class BeakerLibLogHandler(logging.Handler):
+    def __init__(self, beakerlib_command):
+        super(BeakerLibLogHandler, self).__init__()
+        self.beakerlib_command = beakerlib_command
+
+    def emit(self, record):
+        command = {
+            'DEBUG': 'rlLogDebug',
+            'INFO': 'rlLogInfo',
+            'WARNING': 'rlLogWarning',
+            'ERROR': 'rlLogError',
+            'CRITICAL': 'rlLogFatal',
+        }.get(record.levelname, 'rlLog')
+        self.beakerlib_command([command, self.format(record)])
+
+
+class BeakerLibProcess(object):
+    """Manager of a Bash process that is being fed beakerlib commands
+    """
+    def __init__(self, env=os.environ):
+        self.log = log_mgr.get_logger(self)
+
+        if 'BEAKERLIB' not in env:
+            raise RuntimeError('$BEAKERLIB not set, cannot use BeakerLib')
+
+        self.env = env
+        # Set up the Bash process
+        self.bash = subprocess.Popen(['bash'],
+                                     stdin=subprocess.PIPE,
+                                     stdout=open(os.devnull, 'w'),
+                                     stderr=open(os.devnull, 'w'))
+        source_path = os.path.join(self.env['BEAKERLIB'], 'beakerlib.sh')
+        self.run_beakerlib_command(['.', source_path])
+
+        # Redirect logging to our own handlers
+        self.setup_log_handler(BeakerLibLogHandler(self.run_beakerlib_command))
+
+    def setup_log_handler(self, handler):
+        log_mgr.configure(
+            {
+                'default_level': 'DEBUG',
+                'handlers': [{'log_handler': handler,
+                              'format': '[%(name)s] %(message)s',
+                              'level': 'info'}]},
+            configure_state='beakerlib_plugin')
+
+    def run_beakerlib_command(self, cmd):
+        """Given a command as a Popen-style list, run it in the Bash process"""
+        if not self.bash:
+            return
+        for word in cmd:
+            self.bash.stdin.write(ipautil.shell_quote(word))
+            self.bash.stdin.write(' ')
+        self.bash.stdin.write('\n')
+        self.bash.stdin.flush()
+        assert self.bash.returncode is None, "BeakerLib Bash process exited"
+
+    def log_links(self, docstring):
+        for match in LINK_RE.finditer(docstring or ''):
+            self.log.info('Link: %s', match.group())
+
+    def end(self):
+        """End the Bash process"""
+        self.run_beakerlib_command(['exit'])
+        bash = self.bash
+        self.bash = None
+        bash.communicate()
+
+    def log_exception(self, err=None):
+        """Log an exception
+
+        err is a 3-tuple as returned from sys.exc_info(); if not given,
+        sys.exc_info() is used.
+        """
+        if err is None:
+            err = sys.exc_info()
+        message = ''.join(traceback.format_exception(*err)).rstrip()
+        self.run_beakerlib_command(['rlLogError', message])
+
+
+class BeakerLibPlugin(object):
+    def __init__(self):
+        self.log = log_mgr.get_logger(self)
+
+        self.process = BeakerLibProcess(env=os.environ)
+
+        self._current_item = None
+
+    def run_beakerlib_command(self, cmd):
+        """Given a command as a Popen-style list, run it in the Bash process"""
+        self.process.run_beakerlib_command(cmd)
+
+    def get_item_name(self, item):
+        """Return a "identifier-style" name for the given item
+
+        The name only contains the characters [^a-zA-Z0-9_].
+        """
+        bad_char_re = re.compile('[^a-zA-Z0-9_]')
+        parts = []
+        current = item
+        while current:
+            if isinstance(current, pytest.Module):
+                name = current.name
+                if name.endswith('.py'):
+                    name = name[:-3]
+                name = bad_char_re.sub('-', name)
+                parts.append(name)
+                break
+            if isinstance(current, pytest.Instance):
+                pass
+            else:
+                name = current.name
+                name = bad_char_re.sub('-', name)
+                parts.append(name)
+            current = current.parent
+        return '-'.join(reversed(parts))
+
+    def set_current_item(self, item):
+        """Set the item that is currently being processed
+
+        No-op if the same item is already being processed.
+        Ends the phase for the previous item, if any.
+        """
+        if item != self._current_item:
+            item_name = self.get_item_name(item)
+            if self._current_item:
+                self.run_beakerlib_command(['rlPhaseEnd'])
+            if item:
+                self.run_beakerlib_command(['rlPhaseStart', 'FAIL', item_name])
+            self._current_item = item
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        """Log all collected items at start of test"""
+        self.run_beakerlib_command(['rlLogInfo', 'Collected pytest tests:'])
+        for item in items:
+            self.run_beakerlib_command(['rlLogInfo',
+                                        '  - ' + self.get_item_name(item)])
+
+    def pytest_runtest_setup(self, item):
+        """Log item before running it"""
+        self.set_current_item(item)
+
+    def pytest_runtest_makereport(self, item, call):
+        """Report pass/fail for setup/call/teardown of an item"""
+        self.set_current_item(item)
+        desc = '%s: %s' % (call.when, item)
+
+        if not call.excinfo:
+            self.run_beakerlib_command(['rlPass', 'PASS %s' % desc])
+        else:
+            self.run_beakerlib_command(['rlLogError', call.excinfo.exconly()])
+            short_repr = str(call.excinfo.getrepr(style='short'))
+            self.run_beakerlib_command(['rlLogInfo', short_repr])
+
+            # Give super-detailed traceback for DEBUG=1
+            long_repr = str(call.excinfo.getrepr(
+                showlocals=True, funcargs=True))
+            self.run_beakerlib_command(['rlLogDebug', long_repr])
+
+            if call.excinfo.errisinstance(pytest.skip.Exception):
+                self.run_beakerlib_command(['rlPass', 'SKIP %s' % desc])
+            else:
+                self.run_beakerlib_command(['rlFail', 'FAIL %s' % desc])
+
+    def pytest_unconfigure(self, config):
+        """Clean up and exit"""
+        self.set_current_item(None)
+        self.process.end()
diff --git a/ipatests/pytest_plugins/integration.py b/ipatests/pytest_plugins/integration.py
index 5a0e46845..5329e5190 100644
--- a/ipatests/pytest_plugins/integration.py
+++ b/ipatests/pytest_plugins/integration.py
@@ -19,13 +19,137 @@
 
 """Pytest plugin for IPA Integration tests"""
 
+import os
+import tempfile
+import shutil
+
 import pytest
 
+from ipapython import ipautil
+from ipapython.ipa_log_manager import log_mgr
 from ipatests.test_integration.config import get_global_config
 
 
+log = log_mgr.get_logger(__name__)
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("IPA integration tests")
+
+    group.addoption(
+        '--logfile-dir', dest="logfile_dir", default=None,
+        help="Directory to store integration test logs in.")
+
+
+def collect_test_logs(node, logs_dict, test_config):
+    """Collect logs from a test
+
+    Calls collect_logs
+
+    :param node: The pytest collection node (request.node)
+    :param logs_dict: Mapping of host to list of log filnames to collect
+    :param test_config: Pytest configuration
+    """
+    collect_logs(
+        name=node.nodeid.replace('/', '-').replace('::', '-'),
+        logs_dict=logs_dict,
+        logfile_dir=test_config.getoption('logfile_dir'),
+        beakerlib_plugin=test_config.pluginmanager.getplugin('BeakerLibPlugin'),
+    )
+
+
+def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None):
+    """Collect logs from remote hosts
+
+    Calls collect_logs
+
+    :param name: Name under which logs arecollected, e.g. name of the test
+    :param logs_dict: Mapping of host to list of log filnames to collect
+    :param logfile_dir: Directory to log to
+    :param beakerlib_plugin:
+        BeakerLibProcess or BeakerLibPlugin used to collect tests for BeakerLib
+
+    If neither logfile_dir nor beakerlib_plugin is given, no tests are
+    collected.
+    """
+    if logs_dict and (logfile_dir or beakerlib_plugin):
+
+        if logfile_dir:
+            remove_dir = False
+        else:
+            logfile_dir = tempfile.mkdtemp()
+            remove_dir = True
+
+        topdirname = os.path.join(logfile_dir, name)
+
+        for host, logs in logs_dict.items():
+            log.info('Collecting logs from: %s', host.hostname)
+
+            # Tar up the logs on the remote server
+            cmd = host.run_command(['tar', 'cJv'] + logs, log_stdout=False,
+                                   raiseonerr=False)
+            if cmd.returncode:
+                log.warn('Could not collect all requested logs')
+
+            # Unpack on the local side
+            dirname = os.path.join(topdirname, host.hostname)
+            try:
+                os.makedirs(dirname)
+            except OSError:
+                pass
+            tarname = os.path.join(dirname, 'logs.tar.xz')
+            with open(tarname, 'w') as f:
+                f.write(cmd.stdout_text)
+            ipautil.run(['tar', 'xJvf', 'logs.tar.xz'], cwd=dirname,
+                        raiseonerr=False)
+            os.unlink(tarname)
+
+        if beakerlib_plugin:
+            # Use BeakerLib's rlFileSubmit on the indifidual files
+            # The resulting submitted filename will be
+            # $HOSTNAME-$FILENAME (with '/' replaced by '-')
+            beakerlib_plugin.run_beakerlib_command(['pushd', topdirname])
+            try:
+                for dirpath, dirnames, filenames in os.walk(topdirname):
+                    for filename in filenames:
+                        fullname = os.path.relpath(
+                            os.path.join(dirpath, filename), topdirname)
+                        log.debug('Submitting file: %s', fullname)
+                        beakerlib_plugin.run_beakerlib_command(
+                            ['rlFileSubmit', fullname])
+            finally:
+                beakerlib_plugin.run_beakerlib_command(['popd'])
+
+        if remove_dir:
+            if beakerlib_plugin:
+                # The BeakerLib process runs asynchronously, let it clean up
+                # after it's done with the directory
+                beakerlib_plugin.run_beakerlib_command(
+                    ['rm', '-rvf', topdirname])
+            else:
+                shutil.rmtree(topdirname)
+
+        logs_dict.clear()
+
+
+@pytest.fixture(scope='class')
+def class_integration_logs():
+    """Internal fixture providing class-level logs_dict"""
+    return {}
+
+
+@pytest.yield_fixture
+def integration_logs(class_integration_logs, request):
+    """Provides access to test integration logs, and collects after each test
+    """
+    yield class_integration_logs
+    collect_test_logs(request.node, class_integration_logs, request.config)
+
+
 @pytest.yield_fixture(scope='class')
-def integration_config(request):
+def integration_config(request, class_integration_logs):
+    """Integration test Config object
+    """
     cls = request.cls
 
     def get_resources(resource_container, resource_str, num_needed):
@@ -39,7 +163,7 @@ def integration_config(request):
     if not config.domains:
         raise pytest.skip('Integration testing not configured')
 
-    cls.logs_to_collect = {}
+    cls.logs_to_collect = class_integration_logs
 
     cls.domain = config.domains[0]
 
@@ -64,8 +188,13 @@ def integration_config(request):
                           % (missing_extra_roles,
                              available_extra_roles))
 
+    def collect_log(host, filename):
+        log.info('Adding %s:%s to list of logs to collect' %
+                 (host.external_hostname, filename))
+        class_integration_logs.setdefault(host, []).append(filename)
+
     for host in cls.get_all_hosts():
-        host.add_log_collector(cls.collect_log)
+        host.add_log_collector(collect_log)
         cls.prepare_host(host)
 
     try:
@@ -79,6 +208,8 @@ def integration_config(request):
     for host in cls.get_all_hosts():
         host.remove_log_collector(collect_log)
 
+    collect_test_logs(request.node, class_integration_logs, request.config)
+
     try:
         cls.uninstall()
     finally: