Integration tests: Port the BeakerLib plugin and log collection to pytest

Move the IPA-specific log collection out of the Beakerlib plugin.
Add the --logfile-dir option to tests and ipa-test-task, so that logs
can be collected even if BeakerLib is not used.

https://fedorahosted.org/freeipa/ticket/4610

Reviewed-By: Tomas Babej <tbabej@redhat.com>

6 files changed, 378 insertions, 295 deletions

diff --git a/ipatests/beakerlib_plugin.py b/ipatests/beakerlib_plugin.py
deleted file mode 100644
index 1f7811a68..000000000
--- a/ipatests/beakerlib_plugin.py
+++ /dev/null
@@ -1,285 +0,0 @@
-# Authors:
-#   Petr Viktorin <pviktori@redhat.com>
-#
-# Copyright (C) 2013  Red Hat
-# see file 'COPYING' for use and warranty information
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""A Nose plugin that integrates with BeakerLib"""
-
-import os
-import sys
-import subprocess
-import traceback
-import logging
-import tempfile
-import re
-
-import nose
-from nose.plugins import Plugin
-
-from ipapython import ipautil
-from ipaplatform.paths import paths
-from ipapython.ipa_log_manager import log_mgr
-
-LINK_RE = re.compile(r'https?://[^\s]+')
-
-
-class BeakerLibLogHandler(logging.Handler):
-    def __init__(self, beakerlib_command):
-        super(BeakerLibLogHandler, self).__init__()
-        self.beakerlib_command = beakerlib_command
-
-    def emit(self, record):
-        command = {
-            'DEBUG': 'rlLogDebug',
-            'INFO': 'rlLogInfo',
-            'WARNING': 'rlLogWarning',
-            'ERROR': 'rlLogError',
-            'CRITICAL': 'rlLogFatal',
-        }.get(record.levelname, 'rlLog')
-        self.beakerlib_command([command, self.format(record)])
-
-
-class BeakerLibProcess(object):
-    def __init__(self, env=os.environ):
-        self.log = log_mgr.get_logger(self)
-
-        if 'BEAKERLIB' not in env:
-            raise RuntimeError('$BEAKERLIB not set, cannot use BeakerLib')
-
-        self.env = env
-        # Set up the Bash process
-        self.bash = subprocess.Popen(['bash'],
-                                     stdin=subprocess.PIPE,
-                                     stdout=open(paths.DEV_NULL, 'w'),
-                                     stderr=open(paths.DEV_NULL, 'w'))
-        source_path = os.path.join(self.env['BEAKERLIB'], 'beakerlib.sh')
-        self.run_beakerlib_command(['.', source_path])
-
-        # _in_class_setup is set when we are in setup_class, so logs can be
-        # collected just before the first test starts
-        self._in_class_setup = False
-
-        # Redirect logging to our own handlers
-        self.setup_log_handler(BeakerLibLogHandler(self.run_beakerlib_command))
-
-    def setup_log_handler(self, handler):
-        log_mgr.configure(
-            {
-                'default_level': 'DEBUG',
-                'handlers': [{'log_handler': handler,
-                              'format': '[%(name)s] %(message)s',
-                              'level': 'info'}]},
-            configure_state='beakerlib_plugin')
-
-    def run_beakerlib_command(self, cmd):
-        """Given a command as a Popen-style list, run it in the Bash process"""
-        if not self.bash:
-            return
-        for word in cmd:
-            self.bash.stdin.write(ipautil.shell_quote(word))
-            self.bash.stdin.write(' ')
-        self.bash.stdin.write('\n')
-        self.bash.stdin.flush()
-        assert self.bash.returncode is None, "BeakerLib Bash process exited"
-
-    def log_links(self, docstring):
-        for match in LINK_RE.finditer(docstring or ''):
-            self.log.info('Link: %s', match.group())
-
-    def end(self):
-        """End the Bash process"""
-        self.run_beakerlib_command(['exit'])
-        bash = self.bash
-        self.bash = None
-        bash.communicate()
-
-    def collect_logs(self, logs_to_collect):
-        """Collect specified logs"""
-        for host, logs in logs_to_collect.items():
-            self.log.info('Collecting logs from: %s', host.hostname)
-
-            # Tar up the logs on the remote server
-            cmd = host.run_command(['tar', 'cJv'] + logs, log_stdout=False,
-                                    raiseonerr=False)
-            if cmd.returncode:
-                self.log.warn('Could not collect all requested logs')
-                return
-
-            # Copy and unpack on the local side
-            topdirname = tempfile.mkdtemp()
-            dirname = os.path.join(topdirname, host.hostname)
-            os.mkdir(dirname)
-            tarname = os.path.join(dirname, 'logs.tar.xz')
-            with open(tarname, 'w') as f:
-                f.write(cmd.stdout_text)
-            ipautil.run(['tar', 'xJvf', 'logs.tar.xz'], cwd=dirname)
-            os.unlink(tarname)
-
-            # Use BeakerLib's rlFileSubmit on the indifidual files
-            # The resulting submitted filename will be
-            # $HOSTNAME-$FILENAME (with '/' replaced by '-')
-            self.run_beakerlib_command(['pushd', topdirname])
-            for dirpath, dirnames, filenames in os.walk(topdirname):
-                for filename in filenames:
-                    fullname = os.path.relpath(
-                        os.path.join(dirpath, filename), topdirname)
-                    self.log.debug('Submitting file: %s', fullname)
-                    self.run_beakerlib_command(['rlFileSubmit', fullname])
-            self.run_beakerlib_command(['popd'])
-
-            # The BeakerLib process runs asynchronously, let it clean up
-            # after it's done with the directory
-            self.run_beakerlib_command(['rm', '-rvf', topdirname])
-
-        logs_to_collect.clear()
-
-    def log_exception(self, err=None):
-        """Log an exception
-
-        err is a 3-tuple as returned from sys.exc_info(); if not given,
-        sys.exc_info() is used.
-        """
-        if err is None:
-            err = sys.exc_info()
-        message = ''.join(traceback.format_exception(*err)).rstrip()
-        self.run_beakerlib_command(['rlLogError', message])
-
-
-class BeakerLibPlugin(Plugin):
-    """A Nose plugin that integrates with BeakerLib"""
-    # Since BeakerLib is a Bash library, we need to run it in Bash.
-    # The plugin maintains a Bash process and feeds it with commands
-    # on events like test start/end, logging, etc.
-    # See nose.plugins.base.IPluginInterface for Nose plugin interface docs
-    name = 'beakerlib'
-
-    def __init__(self):
-        super(BeakerLibPlugin, self).__init__()
-        self.log = log_mgr.get_logger(self)
-        self._in_class_setup = False
-
-    def options(self, parser, env=os.environ):
-        super(BeakerLibPlugin, self).options(parser, env=env)
-        self.env = env
-        self.parser = parser
-
-    def configure(self, options, conf):
-        super(BeakerLibPlugin, self).configure(options, conf)
-        if not self.enabled:
-            return
-
-        if 'BEAKERLIB' not in self.env:
-            self.parser.error(
-                '$BEAKERLIB not set, cannot use --with-beakerlib')
-
-        self.process = BeakerLibProcess(env=self.env)
-
-    def run_beakerlib_command(self, cmd):
-        """Given a command as a Popen-style list, run it in the Bash process"""
-        self.process.run_beakerlib_command(cmd)
-
-    def report(self, stream):
-        self.process.end()
-
-    def log_exception(self, err):
-        self.process.log_exception(err)
-
-    def log_links(self, docstring):
-        self.process.log_links(docstring)
-
-    def startContext(self, context):
-        """Start a test context (module, class)
-
-        For test classes, this starts a BeakerLib phase
-        """
-        if not isinstance(context, type):
-            return
-        try:
-            docstring = context.__doc__
-            caption = docstring.strip().partition('\n')[0]
-        except AttributeError:
-            docstring = ''
-            caption = 'Nose class (no docstring)'
-        phase_name = "%s-%s: %s" % (context.__module__.replace('.', '-'),
-                                    context.__name__, caption)
-        self.run_beakerlib_command(['rlPhaseStart', 'FAIL', phase_name])
-        self._in_class_setup = True
-        self.log_links(docstring)
-
-    def stopContext(self, context):
-        """End a test context"""
-        if not isinstance(context, type):
-            return
-        self.collect_logs(context)
-        self.run_beakerlib_command(['rlPhaseEnd'])
-
-    def startTest(self, test):
-        """Start a test phase"""
-        if self._in_class_setup:
-            self.collect_logs(test.context)
-        self.log.info('Running test: %s', test.id())
-        caption = test.shortDescription()
-        if not caption:
-            caption = 'Nose method (no docstring)'
-        phase_name = test.id().replace('.', '-')
-        method = test
-        while hasattr(method, 'test'):
-            method = method.test
-        argument = getattr(method, 'test_argument', None)
-        if argument:
-            phase_name += '-%s' % re.sub('[^-a-zA-Z0-9]+', '_', str(argument))
-        phase_name += ": %s" % caption
-        self.run_beakerlib_command(['rlPhaseStart', 'FAIL', phase_name])
-
-        while hasattr(test, 'test'):
-            # Un-wrap Nose test cases to get at the actual test method
-            test = test.test
-        self.log_links(getattr(test, '__doc__', ''))
-
-    def stopTest(self, test):
-        """End a test phase"""
-        self.collect_logs(test.context)
-        self.run_beakerlib_command(['rlPhaseEnd'])
-
-    def addSuccess(self, test):
-        self.run_beakerlib_command(['rlPass', 'Test succeeded'])
-
-    def addError(self, test, err):
-        if issubclass(err[0], nose.SkipTest):
-            # Log skipped test.
-            # Unfortunately we only get to see this if the built-in skip
-            # plugin is disabled (--no-skip)
-            self.run_beakerlib_command(['rlPass', 'Test skipped: %s' % err[1]])
-        else:
-            self.log_exception(err)
-            self.run_beakerlib_command(
-                ['rlFail', 'Test failed: unhandled exception'])
-        self.collect_logs(test.context)
-
-    def addFailure(self, test, err):
-        self.log_exception(err)
-        self.run_beakerlib_command(['rlFail', 'Test failed'])
-
-    def collect_logs(self, test):
-        """Collect logs specified in test's logs_to_collect attribute
-        """
-        try:
-            logs_to_collect = test.logs_to_collect
-        except AttributeError:
-            self.log.debug('No logs to collect')
-        else:
-            self.process.collect_logs(logs_to_collect)
diff --git a/ipatests/ipa-test-task b/ipatests/ipa-test-task
index 02a04b6ab..612974549 100755
--- a/ipatests/ipa-test-task
+++ b/ipatests/ipa-test-task
@@ -28,7 +28,8 @@ from ipapython.ipa_log_manager import log_mgr, standard_logging_setup
 from ipatests.test_integration import config
 from ipatests.test_integration import tasks
 from ipatests.test_integration.host import Host
-from ipatests.beakerlib_plugin import BeakerLibProcess
+from ipatests.pytest_plugins.beakerlib import BeakerLibProcess
+from ipatests.pytest_plugins.integration import collect_logs
 
 
 log = log_mgr.get_logger(__name__)
@@ -50,6 +51,9 @@ class TaskRunner(object):
                             help="""Issue BeakerLib commands for logging
                                     and log collection""")
 
+        parser.add_argument('--logfile-dir', dest='logfile_dir',
+                            help="""Directory to collect logs in""")
+
         subparsers = parser.add_subparsers(
             metavar='SUBCOMMAND',
             help='The action to perform (* indicates an idempotent operation)')
@@ -282,8 +286,10 @@ class TaskRunner(object):
             raise
         finally:
             if args.with_beakerlib:
+                collect_logs('ipa-test-task', logs_to_collect,
+                             logfile_dir=args.logfile_dir,
+                             beakerlib_plugin=beakerlib_process)
                 beakerlib_process.end()
-                beakerlib_process.collect_logs(logs_to_collect)
             for host in self._prepared_hosts:
                 host.remove_log_collector(self.collect_log)
 
diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index 38b0484c0..3531482d2 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -12,6 +12,7 @@ addopts = --doctest-modules
           -p ipatests.pytest_plugins.declarative
           -p ipatests.pytest_plugins.ordering
           -p ipatests.pytest_plugins.integration
+          -p ipatests.pytest_plugins.beakerlib
             # Ignore files for doc tests.
             # TODO: ideally, these should all use __name__=='__main__' guards
           --ignore=setup.py
diff --git a/ipatests/pytest_plugins/beakerlib.py b/ipatests/pytest_plugins/beakerlib.py
new file mode 100644
index 000000000..45bbb0539
--- /dev/null
+++ b/ipatests/pytest_plugins/beakerlib.py
@@ -0,0 +1,234 @@
+#!/usr/bin/python2
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""pytest integration with BeakerLib
+
+Runs a Bash process on the side, and feeds BeakerLib commands to it
+(rlPhaseStart, rlPhaseEnd, rlPass, rlFail, ...)
+
+Other plugins may integrate with this using pytest's
+config.pluginmanager.getplugin('BeakerLibPlugin'). If this is None,
+BeakerLib integration is not active, otherwise the result's
+run_beakerlib_command method can be used to run additional commands.
+
+IPA logging is also redirected to the Bash process.
+"""
+
+import os
+import re
+import logging
+import subprocess
+
+import pytest
+
+from ipapython import ipautil
+from ipapython.ipa_log_manager import log_mgr
+
+
+@pytest.fixture
+def log_files_to_collect():
+    return []
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--with-beakerlib', action="store_true",
+        dest="with_beakerlib", default=None,
+        help="Report test results via beakerlib")
+
+
+@pytest.mark.tryfirst
+def pytest_load_initial_conftests(args, early_config, parser):
+    ns = early_config.known_args_namespace
+    if ns.with_beakerlib:
+        if 'BEAKERLIB' not in os.environ:
+            raise exit('$BEAKERLIB not set, cannot use --with-beakerlib')
+
+        plugin = BeakerLibPlugin()
+        pluginmanager = early_config.pluginmanager.register(
+            plugin, 'BeakerLibPlugin')
+
+
+class BeakerLibLogHandler(logging.Handler):
+    def __init__(self, beakerlib_command):
+        super(BeakerLibLogHandler, self).__init__()
+        self.beakerlib_command = beakerlib_command
+
+    def emit(self, record):
+        command = {
+            'DEBUG': 'rlLogDebug',
+            'INFO': 'rlLogInfo',
+            'WARNING': 'rlLogWarning',
+            'ERROR': 'rlLogError',
+            'CRITICAL': 'rlLogFatal',
+        }.get(record.levelname, 'rlLog')
+        self.beakerlib_command([command, self.format(record)])
+
+
+class BeakerLibProcess(object):
+    """Manager of a Bash process that is being fed beakerlib commands
+    """
+    def __init__(self, env=os.environ):
+        self.log = log_mgr.get_logger(self)
+
+        if 'BEAKERLIB' not in env:
+            raise RuntimeError('$BEAKERLIB not set, cannot use BeakerLib')
+
+        self.env = env
+        # Set up the Bash process
+        self.bash = subprocess.Popen(['bash'],
+                                     stdin=subprocess.PIPE,
+                                     stdout=open(os.devnull, 'w'),
+                                     stderr=open(os.devnull, 'w'))
+        source_path = os.path.join(self.env['BEAKERLIB'], 'beakerlib.sh')
+        self.run_beakerlib_command(['.', source_path])
+
+        # Redirect logging to our own handlers
+        self.setup_log_handler(BeakerLibLogHandler(self.run_beakerlib_command))
+
+    def setup_log_handler(self, handler):
+        log_mgr.configure(
+            {
+                'default_level': 'DEBUG',
+                'handlers': [{'log_handler': handler,
+                              'format': '[%(name)s] %(message)s',
+                              'level': 'info'}]},
+            configure_state='beakerlib_plugin')
+
+    def run_beakerlib_command(self, cmd):
+        """Given a command as a Popen-style list, run it in the Bash process"""
+        if not self.bash:
+            return
+        for word in cmd:
+            self.bash.stdin.write(ipautil.shell_quote(word))
+            self.bash.stdin.write(' ')
+        self.bash.stdin.write('\n')
+        self.bash.stdin.flush()
+        assert self.bash.returncode is None, "BeakerLib Bash process exited"
+
+    def log_links(self, docstring):
+        for match in LINK_RE.finditer(docstring or ''):
+            self.log.info('Link: %s', match.group())
+
+    def end(self):
+        """End the Bash process"""
+        self.run_beakerlib_command(['exit'])
+        bash = self.bash
+        self.bash = None
+        bash.communicate()
+
+    def log_exception(self, err=None):
+        """Log an exception
+
+        err is a 3-tuple as returned from sys.exc_info(); if not given,
+        sys.exc_info() is used.
+        """
+        if err is None:
+            err = sys.exc_info()
+        message = ''.join(traceback.format_exception(*err)).rstrip()
+        self.run_beakerlib_command(['rlLogError', message])
+
+
+class BeakerLibPlugin(object):
+    def __init__(self):
+        self.log = log_mgr.get_logger(self)
+
+        self.process = BeakerLibProcess(env=os.environ)
+
+        self._current_item = None
+
+    def run_beakerlib_command(self, cmd):
+        """Given a command as a Popen-style list, run it in the Bash process"""
+        self.process.run_beakerlib_command(cmd)
+
+    def get_item_name(self, item):
+        """Return a "identifier-style" name for the given item
+
+        The name only contains the characters [^a-zA-Z0-9_].
+        """
+        bad_char_re = re.compile('[^a-zA-Z0-9_]')
+        parts = []
+        current = item
+        while current:
+            if isinstance(current, pytest.Module):
+                name = current.name
+                if name.endswith('.py'):
+                    name = name[:-3]
+                name = bad_char_re.sub('-', name)
+                parts.append(name)
+                break
+            if isinstance(current, pytest.Instance):
+                pass
+            else:
+                name = current.name
+                name = bad_char_re.sub('-', name)
+                parts.append(name)
+            current = current.parent
+        return '-'.join(reversed(parts))
+
+    def set_current_item(self, item):
+        """Set the item that is currently being processed
+
+        No-op if the same item is already being processed.
+        Ends the phase for the previous item, if any.
+        """
+        if item != self._current_item:
+            item_name = self.get_item_name(item)
+            if self._current_item:
+                self.run_beakerlib_command(['rlPhaseEnd'])
+            if item:
+                self.run_beakerlib_command(['rlPhaseStart', 'FAIL', item_name])
+            self._current_item = item
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        """Log all collected items at start of test"""
+        self.run_beakerlib_command(['rlLogInfo', 'Collected pytest tests:'])
+        for item in items:
+            self.run_beakerlib_command(['rlLogInfo',
+                                        '  - ' + self.get_item_name(item)])
+
+    def pytest_runtest_setup(self, item):
+        """Log item before running it"""
+        self.set_current_item(item)
+
+    def pytest_runtest_makereport(self, item, call):
+        """Report pass/fail for setup/call/teardown of an item"""
+        self.set_current_item(item)
+        desc = '%s: %s' % (call.when, item)
+
+        if not call.excinfo:
+            self.run_beakerlib_command(['rlPass', 'PASS %s' % desc])
+        else:
+            self.run_beakerlib_command(['rlLogError', call.excinfo.exconly()])
+            short_repr = str(call.excinfo.getrepr(style='short'))
+            self.run_beakerlib_command(['rlLogInfo', short_repr])
+
+            # Give super-detailed traceback for DEBUG=1
+            long_repr = str(call.excinfo.getrepr(
+                showlocals=True, funcargs=True))
+            self.run_beakerlib_command(['rlLogDebug', long_repr])
+
+            if call.excinfo.errisinstance(pytest.skip.Exception):
+                self.run_beakerlib_command(['rlPass', 'SKIP %s' % desc])
+            else:
+                self.run_beakerlib_command(['rlFail', 'FAIL %s' % desc])
+
+    def pytest_unconfigure(self, config):
+        """Clean up and exit"""
+        self.set_current_item(None)
+        self.process.end()
diff --git a/ipatests/pytest_plugins/integration.py b/ipatests/pytest_plugins/integration.py
index 5a0e46845..5329e5190 100644
--- a/ipatests/pytest_plugins/integration.py
+++ b/ipatests/pytest_plugins/integration.py
@@ -19,13 +19,137 @@
 
 """Pytest plugin for IPA Integration tests"""
 
+import os
+import tempfile
+import shutil
+
 import pytest
 
+from ipapython import ipautil
+from ipapython.ipa_log_manager import log_mgr
 from ipatests.test_integration.config import get_global_config
 
 
+log = log_mgr.get_logger(__name__)
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("IPA integration tests")
+
+    group.addoption(
+        '--logfile-dir', dest="logfile_dir", default=None,
+        help="Directory to store integration test logs in.")
+
+
+def collect_test_logs(node, logs_dict, test_config):
+    """Collect logs from a test
+
+    Calls collect_logs
+
+    :param node: The pytest collection node (request.node)
+    :param logs_dict: Mapping of host to list of log filnames to collect
+    :param test_config: Pytest configuration
+    """
+    collect_logs(
+        name=node.nodeid.replace('/', '-').replace('::', '-'),
+        logs_dict=logs_dict,
+        logfile_dir=test_config.getoption('logfile_dir'),
+        beakerlib_plugin=test_config.pluginmanager.getplugin('BeakerLibPlugin'),
+    )
+
+
+def collect_logs(name, logs_dict, logfile_dir=None, beakerlib_plugin=None):
+    """Collect logs from remote hosts
+
+    Calls collect_logs
+
+    :param name: Name under which logs arecollected, e.g. name of the test
+    :param logs_dict: Mapping of host to list of log filnames to collect
+    :param logfile_dir: Directory to log to
+    :param beakerlib_plugin:
+        BeakerLibProcess or BeakerLibPlugin used to collect tests for BeakerLib
+
+    If neither logfile_dir nor beakerlib_plugin is given, no tests are
+    collected.
+    """
+    if logs_dict and (logfile_dir or beakerlib_plugin):
+
+        if logfile_dir:
+            remove_dir = False
+        else:
+            logfile_dir = tempfile.mkdtemp()
+            remove_dir = True
+
+        topdirname = os.path.join(logfile_dir, name)
+
+        for host, logs in logs_dict.items():
+            log.info('Collecting logs from: %s', host.hostname)
+
+            # Tar up the logs on the remote server
+            cmd = host.run_command(['tar', 'cJv'] + logs, log_stdout=False,
+                                   raiseonerr=False)
+            if cmd.returncode:
+                log.warn('Could not collect all requested logs')
+
+            # Unpack on the local side
+            dirname = os.path.join(topdirname, host.hostname)
+            try:
+                os.makedirs(dirname)
+            except OSError:
+                pass
+            tarname = os.path.join(dirname, 'logs.tar.xz')
+            with open(tarname, 'w') as f:
+                f.write(cmd.stdout_text)
+            ipautil.run(['tar', 'xJvf', 'logs.tar.xz'], cwd=dirname,
+                        raiseonerr=False)
+            os.unlink(tarname)
+
+        if beakerlib_plugin:
+            # Use BeakerLib's rlFileSubmit on the indifidual files
+            # The resulting submitted filename will be
+            # $HOSTNAME-$FILENAME (with '/' replaced by '-')
+            beakerlib_plugin.run_beakerlib_command(['pushd', topdirname])
+            try:
+                for dirpath, dirnames, filenames in os.walk(topdirname):
+                    for filename in filenames:
+                        fullname = os.path.relpath(
+                            os.path.join(dirpath, filename), topdirname)
+                        log.debug('Submitting file: %s', fullname)
+                        beakerlib_plugin.run_beakerlib_command(
+                            ['rlFileSubmit', fullname])
+            finally:
+                beakerlib_plugin.run_beakerlib_command(['popd'])
+
+        if remove_dir:
+            if beakerlib_plugin:
+                # The BeakerLib process runs asynchronously, let it clean up
+                # after it's done with the directory
+                beakerlib_plugin.run_beakerlib_command(
+                    ['rm', '-rvf', topdirname])
+            else:
+                shutil.rmtree(topdirname)
+
+        logs_dict.clear()
+
+
+@pytest.fixture(scope='class')
+def class_integration_logs():
+    """Internal fixture providing class-level logs_dict"""
+    return {}
+
+
+@pytest.yield_fixture
+def integration_logs(class_integration_logs, request):
+    """Provides access to test integration logs, and collects after each test
+    """
+    yield class_integration_logs
+    collect_test_logs(request.node, class_integration_logs, request.config)
+
+
 @pytest.yield_fixture(scope='class')
-def integration_config(request):
+def integration_config(request, class_integration_logs):
+    """Integration test Config object
+    """
     cls = request.cls
 
     def get_resources(resource_container, resource_str, num_needed):
@@ -39,7 +163,7 @@ def integration_config(request):
     if not config.domains:
         raise pytest.skip('Integration testing not configured')
 
-    cls.logs_to_collect = {}
+    cls.logs_to_collect = class_integration_logs
 
     cls.domain = config.domains[0]
 
@@ -64,8 +188,13 @@ def integration_config(request):
                           % (missing_extra_roles,
                              available_extra_roles))
 
+    def collect_log(host, filename):
+        log.info('Adding %s:%s to list of logs to collect' %
+                 (host.external_hostname, filename))
+        class_integration_logs.setdefault(host, []).append(filename)
+
     for host in cls.get_all_hosts():
-        host.add_log_collector(cls.collect_log)
+        host.add_log_collector(collect_log)
         cls.prepare_host(host)
 
     try:
@@ -79,6 +208,8 @@ def integration_config(request):
     for host in cls.get_all_hosts():
         host.remove_log_collector(collect_log)
 
+    collect_test_logs(request.node, class_integration_logs, request.config)
+
     try:
         cls.uninstall()
     finally:
diff --git a/ipatests/test_integration/base.py b/ipatests/test_integration/base.py
index 60b0f02a9..c8b98126e 100644
--- a/ipatests/test_integration/base.py
+++ b/ipatests/test_integration/base.py
@@ -30,6 +30,7 @@ log = log_mgr.get_logger(__name__)
 
 @ordered
 @pytest.mark.usefixtures('integration_config')
+@pytest.mark.usefixtures('integration_logs')
 class IntegrationTest(object):
     num_replicas = 0
     num_clients = 0
@@ -83,10 +84,5 @@ class IntegrationTest(object):
         for client in cls.clients:
             tasks.uninstall_client(client)
 
-    @classmethod
-    def collect_log(cls, host, filename):
-        cls.log.info('Adding %s:%s to list of logs to collect' %
-                     (host.external_hostname, filename))
-        cls.logs_to_collect.setdefault(host, []).append(filename)
 
 IntegrationTest.log = log_mgr.get_logger(IntegrationTest())