Integration tests: Port the ordering plugin to pytest

Ordered integration tests may now be run with pytest.

https://fedorahosted.org/freeipa/ticket/4610

Reviewed-By: Tomas Babej <tbabej@redhat.com>

7 files changed, 146 insertions, 108 deletions

diff --git a/freeipa.spec.in b/freeipa.spec.in
index 703ef9e19..d393fee36 100644
--- a/freeipa.spec.in
+++ b/freeipa.spec.in
@@ -306,6 +306,7 @@ Requires: %{name}-python = %{version}-%{release}
 Requires: tar
 Requires: xz
 Requires: python-nose
+Requires: pytest >= 2.6
 Requires: python-paste
 Requires: python-coverage
 Requires: python-polib
diff --git a/ipatests/order_plugin.py b/ipatests/order_plugin.py
deleted file mode 100644
index 7b114a567..000000000
--- a/ipatests/order_plugin.py
+++ /dev/null
@@ -1,106 +0,0 @@
-# Authors:
-#   Petr Viktorin <pviktori@redhat.com>
-#
-# Copyright (C) 2013  Red Hat
-# see file 'COPYING' for use and warranty information
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-"""A Nose plugin that allows ordered test cases"""
-
-import os
-import unittest
-import inspect
-
-from nose.plugins import Plugin
-import nose.loader
-import nose.util
-
-
-def ordered(cls):
-    """Decorator that marks a test class as ordered
-
-    Methods within the marked class will be executed in definition order
-    (or more strictly, in ordered by the line number where they're defined).
-
-    Subclasses of unittest.TestCase can not be ordered.
-
-    Generator methods will not be ordered by this plugin.
-    """
-    cls._order_plugin__ordered = True
-    assert not isinstance(cls, unittest.TestCase), (
-        "A unittest.TestCase may not be ordered.")
-    return cls
-
-
-class OrderTests(Plugin):
-    name = 'ordered-tests'
-
-    def options(self, parser, env=os.environ):
-        super(OrderTests, self).options(parser, env=env)
-
-    def configure(self, options, conf):
-        super(OrderTests, self).configure(options, conf)
-        if not self.enabled:
-            return
-
-    def loadTestsFromTestClass(self, cls):
-        """Sort methods of ordered test cases by co_firstlineno"""
-        if not getattr(cls, '_order_plugin__ordered', False):
-            return
-        loader = nose.loader.TestLoader()
-
-        def wanted(attr):
-            item = getattr(cls, attr, None)
-            if not inspect.ismethod(item):
-                return False
-            if nose.util.isgenerator(item.im_func):
-                return False
-            return loader.selector.wantMethod(item)
-
-        def sort_with_respect_to_overriding(func):
-            """
-            Sorts the methods in respect with the parent classes.
-
-            The methods are sorted with respect to the inheritance chain,
-            methods that were defined in the same class are sorted by the line
-            number on which they were defined.
-            """
-
-            # Check each *ordered* class in MRO for definition of func method
-            for i, parent_class in enumerate(reversed(cls.mro())):
-                if getattr(parent_class, '_order_plugin__ordered', False):
-                    method = getattr(parent_class, func.__name__, None)
-                    if method:
-                        # This sorts methods as tuples  (position of the class
-                        # in the inheritance chain, position of the method
-                        # within that class)
-                        return 0, i, method.func_code.co_firstlineno
-
-            # Weird case fallback
-            # Method name not in any of the classes in MRO, run it last
-            return 1, func.func_code.co_firstlineno
-
-        methods = [getattr(cls, case) for case in dir(cls) if wanted(case)]
-        methods.sort(key=sort_with_respect_to_overriding)
-        cases = [loader.makeTest(m, cls) for m in methods]
-        return cases
-
-    def wantMethod(self, method):
-        """Hide non-TestCase methods from the normal loader"""
-        im_class = getattr(method, 'im_class', None)
-        if im_class and getattr(im_class, '_order_plugin__ordered', False):
-            if nose.util.isgenerator(method.im_func):
-                return True
-            return False
diff --git a/ipatests/pytest.ini b/ipatests/pytest.ini
index d4ff3f00d..fbd6558ab 100644
--- a/ipatests/pytest.ini
+++ b/ipatests/pytest.ini
@@ -8,6 +8,7 @@
 python_classes = test_ Test
 addopts = --doctest-modules
           -p ipatests.pytest_plugins.declarative
+          -p ipatests.pytest_plugins.ordering
             # Ignore files for doc tests.
             # TODO: ideally, these should all use __name__=='__main__' guards
           --ignore=setup.py
diff --git a/ipatests/pytest_plugins/ordering.py b/ipatests/pytest_plugins/ordering.py
new file mode 100644
index 000000000..3af496a88
--- /dev/null
+++ b/ipatests/pytest_plugins/ordering.py
@@ -0,0 +1,88 @@
+# Authors:
+#   Petr Viktorin <pviktori@redhat.com>
+#
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Pytest plugin for IPA
+
+Adds support for the @pytest.mark.source_order decorator which,
+when applied to a class, runs the test methods in source order.
+
+See test_ordering for an example.
+"""
+
+import unittest
+
+import pytest
+
+
+def ordered(cls):
+    """Decorator that marks a test class as ordered
+
+    Methods within the marked class will be executed in definition order
+    (or more strictly, in ordered by the line number where they're defined).
+
+    Subclasses of unittest.TestCase can not be ordered.
+
+    Generator methods will not be ordered by this plugin.
+    """
+    cls._order_plugin__ordered = True
+    assert not isinstance(cls, unittest.TestCase), (
+        "A unittest.TestCase may not be ordered.")
+    cls = pytest.mark.source_order(cls)
+    return cls
+
+
+def decorate_items(items):
+    node_indexes = {}
+    for index, item in enumerate(items):
+        try:
+            func = item.function
+        except AttributeError:
+            yield (index, ), item
+            continue
+
+        key = (index, )
+        for node in reversed(item.listchain()):
+            # Find the corresponding class
+            if isinstance(node, pytest.Class):
+                cls = node.cls
+            else:
+                continue
+            if getattr(cls, '_order_plugin__ordered', False):
+                node_index = node_indexes.setdefault(node, index)
+                # Find first occurence of the method in class hierarchy
+                for i, parent_class in enumerate(reversed(cls.mro())):
+                    if getattr(parent_class, '_order_plugin__ordered', False):
+                        method = getattr(parent_class, func.__name__, None)
+                        if method:
+                            # Sort methods as tuples  (position of the class
+                            # in the inheritance chain, position of the method
+                            # within that class)
+                            key = (node_index, 0,
+                                   i, method.func_code.co_firstlineno, node)
+                            break
+                else:
+                    # Weird case fallback
+                    # Method name not in any of the classes in MRO, run it last
+                    key = node_index, 1, func.func_code.co_firstlineno, node
+                break
+        yield key, item
+
+
+def pytest_collection_modifyitems(session, config, items):
+    items[:] = [item for i, item in sorted(decorate_items(items))]
diff --git a/ipatests/test_integration/base.py b/ipatests/test_integration/base.py
index a24a577d6..b07eab099 100644
--- a/ipatests/test_integration/base.py
+++ b/ipatests/test_integration/base.py
@@ -24,7 +24,7 @@ import nose
 from ipapython.ipa_log_manager import log_mgr
 from ipatests.test_integration.config import get_global_config
 from ipatests.test_integration import tasks
-from ipatests.order_plugin import ordered
+from ipatests.pytest_plugins.ordering import ordered
 
 log = log_mgr.get_logger(__name__)
 
diff --git a/ipatests/test_integration/test_caless.py b/ipatests/test_integration/test_caless.py
index b2af4f7fe..19a425e15 100644
--- a/ipatests/test_integration/test_caless.py
+++ b/ipatests/test_integration/test_caless.py
@@ -31,7 +31,7 @@ from ipaplatform.paths import paths
 from ipapython.dn import DN
 from ipatests.test_integration.base import IntegrationTest
 from ipatests.test_integration import tasks
-from ipatests.order_plugin import ordered
+from ipatests.pytest_plugins.ordering import ordered
 
 _DEFAULT = object()
 
diff --git a/ipatests/test_integration/test_ordering.py b/ipatests/test_integration/test_ordering.py
new file mode 100644
index 000000000..ee224901c
--- /dev/null
+++ b/ipatests/test_integration/test_ordering.py
@@ -0,0 +1,54 @@
+# Authors:
+#   Petr Viktorin <pviktori@redhat.com>
+#
+# Copyright (C) 2014  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the ordering of tests
+
+IPA integration tests, marked with `@ordered`, require tests to be run
+in a specific order:
+- Base classes first
+- Within a class, test methods are ordered according to source line
+"""
+
+from ipatests.pytest_plugins.ordering import ordered
+
+
+@ordered
+class TestBase(object):
+    @classmethod
+    def setup_class(cls):
+        cls.value = 'unchanged'
+
+    def test_d_first(self):
+        type(self).value = 'changed once'
+
+
+class TestChild(TestBase):
+    def test_b_third(self):
+        assert type(self).value == 'changed twice'
+        type(self).value = 'changed thrice'
+
+    def test_a_fourth(self):
+        assert type(self).value == 'changed thrice'
+
+
+def test_c_second(self):
+    assert type(self).value == 'changed once'
+    type(self).value = 'changed twice'
+TestBase.test_c_second = test_c_second
+del test_c_second