This is the groundwork for the upcoming distributed scheduler changes. Nothing is actually wired up here, so it shouldn't break any existing code (and all tests pass).

The goals were to:

1. Define the basic distributed scheduler communication mechanism:
   a. call_zone_method - how each zone can communicate with its children

   b. encrypted child-blobs - how child zones an securely and statelessly report back weight and build-plan info

2. Put in hooks for advanced-filtering (hard-requirements, capabilities) as well as preferences (least-cost-scheduling)

3. Create a base set of dist-scheduler tests that we can extend as we add more functionality.

Next up will be to:

1. Add in a filtering driver

2. Add in a cost-scheduler driver

4 files changed, 266 insertions, 4 deletions

diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py
index 5d5799b59..fa2e05033 100644
--- a/nova/tests/api/openstack/test_zones.py
+++ b/nova/tests/api/openstack/test_zones.py
@@ -20,6 +20,8 @@ import json
 
 import nova.db
 from nova import context
+from nova import crypto
+from nova import exception
 from nova import flags
 from nova import test
 from nova.api.openstack import zones
@@ -79,6 +81,18 @@ def zone_capabilities(method, context):
     return dict()
 
 
+GLOBAL_BUILD_PLAN = [
+        dict(name='host1', weight=10, ip='10.0.0.1', zone='zone1'),
+        dict(name='host2', weight=9, ip='10.0.0.2', zone='zone2'),
+        dict(name='host3', weight=8, ip='10.0.0.3', zone='zone3'),
+        dict(name='host4', weight=7, ip='10.0.0.4', zone='zone4'),
+     ]
+
+
+def zone_select(context, specs):
+    return GLOBAL_BUILD_PLAN
+
+
 class ZonesTest(test.TestCase):
     def setUp(self):
         super(ZonesTest, self).setUp()
@@ -190,3 +204,31 @@ class ZonesTest(test.TestCase):
         self.assertEqual(res_dict['zone']['name'], 'darksecret')
         self.assertEqual(res_dict['zone']['cap1'], 'a;b')
         self.assertEqual(res_dict['zone']['cap2'], 'c;d')
+
+    def test_zone_select(self):
+        FLAGS.build_plan_encryption_key = 'c286696d887c9aa0611bbb3e2025a45a'
+        self.stubs.Set(api, 'select', zone_select)
+
+        req = webob.Request.blank('/v1.0/zones/select')
+
+        res = req.get_response(fakes.wsgi_app())
+        res_dict = json.loads(res.body)
+        self.assertEqual(res.status_int, 200)
+
+        self.assertTrue('weights' in res_dict)
+
+        for item in res_dict['weights']:
+            blob = item['blob']
+            decrypt = crypto.decryptor(FLAGS.build_plan_encryption_key)
+            secret_item = json.loads(decrypt(blob))
+            found = False
+            for original_item in GLOBAL_BUILD_PLAN:
+                if original_item['name'] != secret_item['name']:
+                    continue
+                found = True
+                for key in ('weight', 'ip', 'zone'):
+                    self.assertEqual(secret_item[key], original_item[key])
+
+            self.assertTrue(found)
+            self.assertEqual(len(item), 2)
+            self.assertTrue('weight' in item)
diff --git a/nova/tests/test_crypto.py b/nova/tests/test_crypto.py
new file mode 100644
index 000000000..945d78794
--- /dev/null
+++ b/nova/tests/test_crypto.py
@@ -0,0 +1,48 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests for Crypto module.
+"""
+
+from nova import crypto
+from nova import test
+
+
+class SymmetricKeyTestCase(test.TestCase):
+    """Test case for Encrypt/Decrypt"""
+    def test_encrypt_decrypt(self):
+        key = 'c286696d887c9aa0611bbb3e2025a45a'
+        plain_text = "The quick brown fox jumped over the lazy dog."
+
+        # No IV supplied (all 0's)
+        encrypt = crypto.encryptor(key)
+        cipher_text = encrypt(plain_text)
+        self.assertNotEquals(plain_text, cipher_text)
+
+        decrypt = crypto.decryptor(key)
+        plain = decrypt(cipher_text)
+
+        self.assertEquals(plain_text, plain)
+
+        # IV supplied ...
+        iv = '562e17996d093d28ddb3ba695a2e6f58'
+        encrypt = crypto.encryptor(key, iv)
+        cipher_text = encrypt(plain_text)
+        self.assertNotEquals(plain_text, cipher_text)
+
+        decrypt = crypto.decryptor(key, iv)
+        plain = decrypt(cipher_text)
+
+        self.assertEquals(plain_text, plain)
diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py
index 968ef9d6c..54b3f80fb 100644
--- a/nova/tests/test_scheduler.py
+++ b/nova/tests/test_scheduler.py
@@ -912,7 +912,8 @@ class SimpleDriverTestCase(test.TestCase):
 
 
 class FakeZone(object):
-    def __init__(self, api_url, username, password):
+    def __init__(self, id, api_url, username, password):
+        self.id = id
         self.api_url = api_url
         self.username = username
         self.password = password
@@ -920,7 +921,7 @@ class FakeZone(object):
 
 def zone_get_all(context):
     return [
-                FakeZone('http://example.com', 'bob', 'xxx'),
+                FakeZone(1, 'http://example.com', 'bob', 'xxx'),
            ]
 
 
@@ -1037,7 +1038,7 @@ class FakeNovaClient(object):
 
 class DynamicNovaClientTest(test.TestCase):
     def test_issue_novaclient_command_found(self):
-        zone = FakeZone('http://example.com', 'bob', 'xxx')
+        zone = FakeZone(1, 'http://example.com', 'bob', 'xxx')
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeServerCollection()),
                     zone, "servers", "get", 100).a, 10)
@@ -1051,7 +1052,7 @@ class DynamicNovaClientTest(test.TestCase):
                     zone, "servers", "pause", 100), None)
 
     def test_issue_novaclient_command_not_found(self):
-        zone = FakeZone('http://example.com', 'bob', 'xxx')
+        zone = FakeZone(1, 'http://example.com', 'bob', 'xxx')
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeEmptyServerCollection()),
                     zone, "servers", "get", 100), None)
@@ -1063,3 +1064,55 @@ class DynamicNovaClientTest(test.TestCase):
         self.assertEquals(api._issue_novaclient_command(
                     FakeNovaClient(FakeEmptyServerCollection()),
                     zone, "servers", "any", "name"), None)
+
+
+class FakeZonesProxy(object):
+    def do_something(*args, **kwargs):
+        return 42
+
+    def raises_exception(*args, **kwargs):
+        raise Exception('testing')
+
+
+class FakeNovaClientOpenStack(object):
+    def __init__(self, *args, **kwargs):
+        self.zones = FakeZonesProxy()
+
+    def authenticate(self):
+        pass
+
+
+class CallZoneMethodTest(test.TestCase):
+    def setUp(self):
+        super(CallZoneMethodTest, self).setUp()
+        self.stubs = stubout.StubOutForTesting()
+        self.stubs.Set(db, 'zone_get_all', zone_get_all)
+        self.stubs.Set(novaclient, 'OpenStack', FakeNovaClientOpenStack)
+
+    def tearDown(self):
+        self.stubs.UnsetAll()
+        super(CallZoneMethodTest, self).tearDown()
+
+    def test_call_zone_method(self):
+        context = {}
+        method = 'do_something'
+        results = api.call_zone_method(context, method)
+        expected = [(1, 42)]
+        self.assertEqual(expected, results)
+
+    def test_call_zone_method_not_present(self):
+        context = {}
+        method = 'not_present'
+        self.assertRaises(AttributeError, api.call_zone_method,
+                          context, method)
+
+    def test_call_zone_method_generates_exception(self):
+        context = {}
+        method = 'raises_exception'
+        results = api.call_zone_method(context, method)
+
+        # FIXME(sirp): for now the _error_trap code is catching errors and
+        # converting them to a ("ERROR", "string") tuples. The code (and this
+        # test) should eventually handle real exceptions.
+        expected = [(1, ('ERROR', 'testing'))]
+        self.assertEqual(expected, results)
diff --git a/nova/tests/test_zone_aware_scheduler.py b/nova/tests/test_zone_aware_scheduler.py
new file mode 100644
index 000000000..fdcde34c9
--- /dev/null
+++ b/nova/tests/test_zone_aware_scheduler.py
@@ -0,0 +1,119 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+Tests For Zone Aware Scheduler.
+"""
+
+from nova import test
+from nova.scheduler import driver
+from nova.scheduler import zone_aware_scheduler
+from nova.scheduler import zone_manager
+
+
+class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler):
+    def filter_hosts(self, num, specs):
+        # NOTE(sirp): this is returning [(hostname, services)]
+        return self.zone_manager.service_states.items()
+
+    def weigh_hosts(self, num, specs, hosts):
+        fake_weight = 99
+        weighted = []
+        for hostname, caps in hosts:
+            weighted.append(dict(weight=fake_weight, name=hostname))
+        return weighted
+
+
+class FakeZoneManager(zone_manager.ZoneManager):
+    def __init__(self):
+        self.service_states = {
+                        'host1': {
+                            'compute': {'ram': 1000}
+                         },
+                         'host2': {
+                            'compute': {'ram': 2000}
+                         },
+                         'host3': {
+                            'compute': {'ram': 3000}
+                         }
+                     }
+
+
+class FakeEmptyZoneManager(zone_manager.ZoneManager):
+    def __init__(self):
+        self.service_states = {}
+
+
+def fake_empty_call_zone_method(context, method, specs):
+    return []
+
+
+def fake_call_zone_method(context, method, specs):
+    return [
+        ('zone1', [
+            dict(weight=1, blob='AAAAAAA'),
+            dict(weight=111, blob='BBBBBBB'),
+            dict(weight=112, blob='CCCCCCC'),
+            dict(weight=113, blob='DDDDDDD'),
+        ]),
+        ('zone2', [
+            dict(weight=120, blob='EEEEEEE'),
+            dict(weight=2, blob='FFFFFFF'),
+            dict(weight=122, blob='GGGGGGG'),
+            dict(weight=123, blob='HHHHHHH'),
+        ]),
+        ('zone3', [
+            dict(weight=130, blob='IIIIIII'),
+            dict(weight=131, blob='JJJJJJJ'),
+            dict(weight=132, blob='KKKKKKK'),
+            dict(weight=3, blob='LLLLLLL'),
+        ]),
+    ]
+
+
+class ZoneAwareSchedulerTestCase(test.TestCase):
+    """Test case for Zone Aware Scheduler."""
+
+    def test_zone_aware_scheduler(self):
+        """
+        Create a nested set of FakeZones, ensure that a select call returns the
+        appropriate build plan.
+        """
+        sched = FakeZoneAwareScheduler()
+        self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method)
+
+        zm = FakeZoneManager()
+        sched.set_zone_manager(zm)
+
+        fake_context = {}
+        build_plan = sched.select(fake_context, {})
+
+        self.assertEqual(15, len(build_plan))
+
+        hostnames = [plan_item['name']
+                     for plan_item in build_plan if 'name' in plan_item]
+        self.assertEqual(3, len(hostnames))
+
+    def test_empty_zone_aware_scheduler(self):
+        """
+        Ensure empty hosts & child_zones result in NoValidHosts exception.
+        """
+        sched = FakeZoneAwareScheduler()
+        self.stubs.Set(sched, '_call_zone_method', fake_empty_call_zone_method)
+
+        zm = FakeEmptyZoneManager()
+        sched.set_zone_manager(zm)
+
+        fake_context = {}
+        self.assertRaises(driver.NoValidHost, sched.schedule, fake_context, {})