Refactor scheduling filters

This changes the scheduling filters to use the code in nova/loadables.py
to locate filtering classes.  This also creates some base functionality
in nova/filters.py to be used by both the host scheduler and the cells
scheduler.

The scheduler_available_filters default has changed to
'nova.scheduler.filters.all_filters' which is better named compared to
the old setting of 'standard_filters'.  The old method is still
supported for those that have put it explicitly in their configs, but
it's marked as deprecated.

DocImpact

Change-Id: I84fdeafdba0275ab4b25f8857563bd7b1494bb69

4 files changed, 289 insertions, 136 deletions

diff --git a/nova/tests/scheduler/test_filter_scheduler.py b/nova/tests/scheduler/test_filter_scheduler.py
index 6add77cbe..b8ab45bc2 100644
--- a/nova/tests/scheduler/test_filter_scheduler.py
+++ b/nova/tests/scheduler/test_filter_scheduler.py
@@ -32,7 +32,7 @@ from nova.tests.scheduler import fakes
 from nova.tests.scheduler import test_scheduler
 
 
-def fake_filter_hosts(hosts, filter_properties):
+def fake_get_filtered_hosts(hosts, filter_properties):
     return list(hosts)
 
 
@@ -155,8 +155,8 @@ class FilterSchedulerTestCase(test_scheduler.SchedulerTestCase):
         fake_context = context.RequestContext('user', 'project',
                 is_admin=True)
 
-        self.stubs.Set(sched.host_manager, 'filter_hosts',
-                fake_filter_hosts)
+        self.stubs.Set(sched.host_manager, 'get_filtered_hosts',
+                fake_get_filtered_hosts)
         self.stubs.Set(least_cost, 'weighted_sum', _fake_weighted_sum)
         fakes.mox_host_manager_db_calls(self.mox, fake_context)
 
diff --git a/nova/tests/scheduler/test_host_filters.py b/nova/tests/scheduler/test_host_filters.py
index f19755b82..1f35ede9b 100644
--- a/nova/tests/scheduler/test_host_filters.py
+++ b/nova/tests/scheduler/test_host_filters.py
@@ -263,32 +263,33 @@ class HostFiltersTestCase(test.TestCase):
         self.json_query = jsonutils.dumps(
                 ['and', ['>=', '$free_ram_mb', 1024],
                         ['>=', '$free_disk_mb', 200 * 1024]])
-        # This has a side effect of testing 'get_filter_classes'
-        # when specifying a method (in this case, our standard filters)
-        classes = filters.get_filter_classes(
-                ['nova.scheduler.filters.standard_filters'])
+        filter_handler = filters.HostFilterHandler()
+        classes = filter_handler.get_matching_classes(
+                ['nova.scheduler.filters.all_filters'])
         self.class_map = {}
         for cls in classes:
             self.class_map[cls.__name__] = cls
 
-    def test_get_filter_classes(self):
-        classes = filters.get_filter_classes(
-                ['nova.tests.scheduler.test_host_filters.TestFilter'])
-        self.assertEqual(len(classes), 1)
-        self.assertEqual(classes[0].__name__, 'TestFilter')
-        # Test a specific class along with our standard filters
-        classes = filters.get_filter_classes(
-                ['nova.tests.scheduler.test_host_filters.TestFilter',
-                 'nova.scheduler.filters.standard_filters'])
-        self.assertEqual(len(classes), 1 + len(self.class_map))
-
-    def test_get_filter_classes_raises_on_invalid_classes(self):
-        self.assertRaises(ImportError,
-                filters.get_filter_classes,
-                ['nova.tests.scheduler.test_host_filters.NoExist'])
-        self.assertRaises(exception.ClassNotFound,
-                filters.get_filter_classes,
-                ['nova.tests.scheduler.test_host_filters.TestBogusFilter'])
+    def test_standard_filters_is_deprecated(self):
+        info = {'called': False}
+
+        def _fake_deprecated(*args, **kwargs):
+            info['called'] = True
+
+        self.stubs.Set(filters.LOG, 'deprecated', _fake_deprecated)
+
+        filter_handler = filters.HostFilterHandler()
+        filter_handler.get_matching_classes(
+                ['nova.scheduler.filters.standard_filters'])
+
+        self.assertTrue(info['called'])
+        self.assertIn('AllHostsFilter', self.class_map)
+        self.assertIn('ComputeFilter', self.class_map)
+
+    def test_all_filters(self):
+        # Double check at least a couple of known filters exist
+        self.assertIn('AllHostsFilter', self.class_map)
+        self.assertIn('ComputeFilter', self.class_map)
 
     def test_all_host_filter(self):
         filt_cls = self.class_map['AllHostsFilter']()
diff --git a/nova/tests/scheduler/test_host_manager.py b/nova/tests/scheduler/test_host_manager.py
index 0984cbf80..d12f1dea5 100644
--- a/nova/tests/scheduler/test_host_manager.py
+++ b/nova/tests/scheduler/test_host_manager.py
@@ -62,34 +62,146 @@ class HostManagerTestCase(test.TestCase):
                 ComputeFilterClass2]
 
         # Test we returns 1 correct function
-        filter_fns = self.host_manager._choose_host_filters(None)
-        self.assertEqual(len(filter_fns), 1)
-        self.assertEqual(filter_fns[0].__func__,
-                ComputeFilterClass2.host_passes.__func__)
-
-    def test_filter_hosts(self):
-        filters = ['fake-filter1', 'fake-filter2']
-        fake_host1 = host_manager.HostState('host1', 'node1')
-        fake_host2 = host_manager.HostState('host2', 'node2')
-        hosts = [fake_host1, fake_host2]
-        filter_properties = {'fake_prop': 'fake_val'}
-
-        self.mox.StubOutWithMock(self.host_manager,
-                '_choose_host_filters')
-        self.mox.StubOutWithMock(fake_host1, 'passes_filters')
-        self.mox.StubOutWithMock(fake_host2, 'passes_filters')
-
-        self.host_manager._choose_host_filters(None).AndReturn(filters)
-        fake_host1.passes_filters(filters, filter_properties).AndReturn(
-                False)
-        fake_host2.passes_filters(filters, filter_properties).AndReturn(
-                True)
+        filter_classes = self.host_manager._choose_host_filters(None)
+        self.assertEqual(len(filter_classes), 1)
+        self.assertEqual(filter_classes[0].__name__, 'ComputeFilterClass2')
+
+    def test_get_filtered_hosts(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1']
+        fake_classes = 'fake_classes'
+        fake_properties = {'moo': 1, 'cow': 2}
+        expected_hosts = set(fake_hosts)
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
 
         self.mox.ReplayAll()
-        filtered_hosts = self.host_manager.filter_hosts(hosts,
-                filter_properties, filters=None)
-        self.assertEqual(len(filtered_hosts), 1)
-        self.assertEqual(filtered_hosts[0], fake_host2)
+
+        result = self.host_manager. get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_specificed_filters(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1']
+        fake_classes = 'fake_classes'
+        fake_properties = {'moo': 1, 'cow': 2}
+        fake_filters = 'fake_filters'
+        expected_hosts = set(fake_hosts)
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(fake_filters).AndReturn(
+                fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties, filter_class_names=fake_filters)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_ignore(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'ignore_hosts': ['fake_host1', 'fake_host3',
+            'fake_host5']}
+        expected_hosts = set(['fake_host2', 'fake_host4'])
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_force_hosts(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'force_hosts': ['fake_host1', 'fake_host3',
+            'fake_host5']}
+        expected_hosts = set(['fake_host1', 'fake_host3'])
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
+
+    def test_get_filtered_hosts_with_no_matching_force_hosts(self):
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'force_hosts': ['fake_host5', 'fake_host6']}
+        expected_result = []
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        # Make sure this is not called.
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, expected_result)
+
+    def test_get_filtered_hosts_with_ignore_and_force(self):
+        """Ensure ignore_hosts processed before force_hosts in host filters"""
+        fake_hosts = ['fake_host1', 'fake_host2', 'fake_host1',
+                'fake_host1', 'fake_host3', 'fake_host4']
+        fake_classes = 'fake_classes'
+        fake_properties = {'force_hosts': ['fake_host3', 'fake_host1'],
+                           'ignore_hosts': ['fake_host1']}
+        expected_hosts = set(['fake_host3'])
+        fake_result = 'fake_result'
+
+        self.mox.StubOutWithMock(self.host_manager, '_choose_host_filters')
+        # Make sure this is not called.
+        self.mox.StubOutWithMock(self.host_manager.filter_handler,
+                'get_filtered_objects')
+        self.host_manager.filter_handler.get_filtered_objects(fake_classes,
+                expected_hosts, fake_properties).AndReturn(fake_result)
+
+        self.host_manager._choose_host_filters(None).AndReturn(fake_classes)
+
+        self.mox.ReplayAll()
+
+        result = self.host_manager.get_filtered_hosts(fake_hosts,
+                fake_properties)
+        self.assertEqual(result, fake_result)
 
     def test_update_service_capabilities(self):
         service_states = self.host_manager.service_states
@@ -190,91 +302,6 @@ class HostStateTestCase(test.TestCase):
     # update_from_compute_node() and consume_from_instance() are tested
     # in HostManagerTestCase.test_get_all_host_states()
 
-    def test_host_state_passes_filters_passes(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        cls1.host_passes(fake_host, filter_properties).AndReturn(True)
-        cls2.host_passes(fake_host, filter_properties).AndReturn(True)
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertTrue(result)
-
-    def test_host_state_passes_filters_passes_with_ignore(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {'ignore_hosts': ['host2']}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        cls1.host_passes(fake_host, filter_properties).AndReturn(True)
-        cls2.host_passes(fake_host, filter_properties).AndReturn(True)
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertTrue(result)
-
-    def test_host_state_passes_filters_fails(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        cls1.host_passes(fake_host, filter_properties).AndReturn(False)
-        # cls2.host_passes() not called because of short circuit
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertFalse(result)
-
-    def test_host_state_passes_filters_fails_from_ignore(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {'ignore_hosts': ['host1']}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        # cls[12].host_passes() not called because of short circuit
-        # with matching host to ignore
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertFalse(result)
-
-    def test_host_state_passes_filters_skipped_from_force(self):
-        fake_host = host_manager.HostState('host1', 'node1')
-        filter_properties = {'force_hosts': ['host1']}
-
-        cls1 = ComputeFilterClass1()
-        cls2 = ComputeFilterClass2()
-        self.mox.StubOutWithMock(cls1, 'host_passes')
-        self.mox.StubOutWithMock(cls2, 'host_passes')
-        filter_fns = [cls1.host_passes, cls2.host_passes]
-
-        # cls[12].host_passes() not called because of short circuit
-        # with matching host to force
-
-        self.mox.ReplayAll()
-        result = fake_host.passes_filters(filter_fns, filter_properties)
-        self.assertTrue(result)
-
     def test_stat_consumption_from_compute_node(self):
         stats = [
             dict(key='num_instances', value='5'),
diff --git a/nova/tests/test_filters.py b/nova/tests/test_filters.py
new file mode 100644
index 000000000..546b13180
--- /dev/null
+++ b/nova/tests/test_filters.py
@@ -0,0 +1,125 @@
+# Copyright 2012 OpenStack LLC.  # All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For Scheduler Host Filters.
+"""
+
+import inspect
+import sys
+
+from nova import filters
+from nova import loadables
+from nova import test
+
+
+class Filter1(filters.BaseFilter):
+    """Test Filter class #1."""
+    pass
+
+
+class Filter2(filters.BaseFilter):
+    """Test Filter class #2."""
+    pass
+
+
+class FiltersTestCase(test.TestCase):
+    def test_filter_all(self):
+        filter_obj_list = ['obj1', 'obj2', 'obj3']
+        filter_properties = 'fake_filter_properties'
+        base_filter = filters.BaseFilter()
+
+        self.mox.StubOutWithMock(base_filter, '_filter_one')
+
+        base_filter._filter_one('obj1', filter_properties).AndReturn(True)
+        base_filter._filter_one('obj2', filter_properties).AndReturn(False)
+        base_filter._filter_one('obj3', filter_properties).AndReturn(True)
+
+        self.mox.ReplayAll()
+
+        result = base_filter.filter_all(filter_obj_list, filter_properties)
+        self.assertTrue(inspect.isgenerator(result))
+        self.assertEqual(list(result), ['obj1', 'obj3'])
+
+    def test_filter_all_recursive_yields(self):
+        """Test filter_all() allows generators from previous filter_all()s."""
+        # filter_all() yields results.  We want to make sure that we can
+        # call filter_all() with generators returned from previous calls
+        # to filter_all().
+        filter_obj_list = ['obj1', 'obj2', 'obj3']
+        filter_properties = 'fake_filter_properties'
+        base_filter = filters.BaseFilter()
+
+        self.mox.StubOutWithMock(base_filter, '_filter_one')
+
+        total_iterations = 200
+
+        # The order that _filter_one is going to get called gets
+        # confusing because we will be recursively yielding things..
+        # We are going to simulate the first call to filter_all()
+        # returning False for 'obj2'.  So, 'obj1' will get yielded
+        # 'total_iterations' number of times before the first filter_all()
+        # call gets to processing 'obj2'.  We then return 'False' for it.
+        # After that, 'obj3' gets yielded 'total_iterations' number of
+        # times.
+        for x in xrange(total_iterations):
+            base_filter._filter_one('obj1', filter_properties).AndReturn(True)
+        base_filter._filter_one('obj2', filter_properties).AndReturn(False)
+        for x in xrange(total_iterations):
+            base_filter._filter_one('obj3', filter_properties).AndReturn(True)
+        self.mox.ReplayAll()
+
+        objs = iter(filter_obj_list)
+        for x in xrange(total_iterations):
+            # Pass in generators returned from previous calls.
+            objs = base_filter.filter_all(objs, filter_properties)
+        self.assertTrue(inspect.isgenerator(objs))
+        self.assertEqual(list(objs), ['obj1', 'obj3'])
+
+    def test_get_filtered_objects(self):
+        filter_objs_initial = ['initial', 'filter1', 'objects1']
+        filter_objs_second = ['second', 'filter2', 'objects2']
+        filter_objs_last = ['last', 'filter3', 'objects3']
+        filter_properties = 'fake_filter_properties'
+
+        def _fake_base_loader_init(*args, **kwargs):
+            pass
+
+        self.stubs.Set(loadables.BaseLoader, '__init__',
+                       _fake_base_loader_init)
+
+        filt1_mock = self.mox.CreateMock(Filter1)
+        filt2_mock = self.mox.CreateMock(Filter2)
+
+        self.mox.StubOutWithMock(sys.modules[__name__], 'Filter1',
+                                 use_mock_anything=True)
+        self.mox.StubOutWithMock(filt1_mock, 'filter_all')
+        self.mox.StubOutWithMock(sys.modules[__name__], 'Filter2',
+                                 use_mock_anything=True)
+        self.mox.StubOutWithMock(filt2_mock, 'filter_all')
+
+        Filter1().AndReturn(filt1_mock)
+        filt1_mock.filter_all(filter_objs_initial,
+                              filter_properties).AndReturn(filter_objs_second)
+        Filter2().AndReturn(filt2_mock)
+        filt2_mock.filter_all(filter_objs_second,
+                              filter_properties).AndReturn(filter_objs_last)
+
+        self.mox.ReplayAll()
+
+        filter_handler = filters.BaseFilterHandler(filters.BaseFilter)
+        filter_classes = [Filter1, Filter2]
+        result = filter_handler.get_filtered_objects(filter_classes,
+                                                     filter_objs_initial,
+                                                     filter_properties)
+        self.assertEqual(result, filter_objs_last)